Changeset 362

Show
Ignore:
Timestamp:
06/21/08 18:44:59 (5 months ago)
Author:
FeepingCreature
Message:
  • Fixes, fixes
Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/tools/tools/downloader.d

    r361 r362  
    1111} 
    1212 
    13 void filterLink(ref string addr) { 
     13void filterLink(ref string addr, ref string prepend) { 
    1414  while (addr.length && addr[$-1]==' ') addr=addr[0..$-1]; 
    15   string post, refer; 
     15  string post, refer, cookie; 
    1616  bool old_html=false; 
    1717  if (addr.startsWith("POST=")) { post=addr[0..addr.find(" ")+1]; addr=addr[addr.find(" ")+1..$]; } 
    1818  if (addr.startsWith("REFER=")) { refer=addr[0..addr.find(" ")+1]; addr=addr[addr.find(" ")+1..$]; } 
     19  if (addr.startsWith("COOKIE=")) { cookie=addr[0..addr.find(" ")+1]; addr=addr[addr.find(" ")+1..$]; } 
    1920  if (addr.startsWith("1.0")) { old_html=true; addr=addr[addr.find(" ")+1..$]; } 
    2021   
    2122  addr=addr.replace("&", "&").replace(" ", "%20"); 
    2223   
    23   addr=post~refer~(old_html?"1.0 ":"")~addr; 
    24 } 
     24  prepend = post~refer~cookie~(old_html?"1.0 ":""); 
     25} 
     26 
     27bool quiet = false; 
    2528 
    2629char[] followLink(char[] oldAddr, char[] newAddr) { 
    27   newAddr.filterLink(); 
    28   if (oldAddr.length>7 && oldAddr[0..7]=="http://") oldAddr=oldAddr[7..$]; 
    29   if (newAddr.beginsWith("data://")) return newAddr; 
     30  string new_prepend; newAddr.filterLink(new_prepend); 
     31  string old_prepend; oldAddr.filterLink(old_prepend); 
     32  string[string] tags; 
     33  foreach (entry; old_prepend.split(" ")~new_prepend.split(" ")) { 
     34    if (!entry.length) continue; 
     35    auto sp = entry.split("="); 
     36    tags[sp[0]] = (sp.length>1)?(sp[1..$].join("=")):""; 
     37  } 
    3038  string prepend; 
    31   auto spaceloc=newAddr.rfind(" "); 
    32   if (spaceloc!=-1) { prepend=newAddr[0..spaceloc+1]; newAddr=newAddr[spaceloc+1..$]; } 
    33   if (newAddr.length>7 && newAddr[0..7]=="http://") return prepend~newAddr[7..$]; 
     39  foreach (key, value; tags) prepend ~= key~"="~value~" "; 
     40   
     41  if (auto rest = oldAddr.startsWith("http://")) oldAddr = rest; 
     42  if (newAddr.startsWith("data://")) return new_prepend~newAddr; 
     43  if (auto rest = newAddr.startsWith("http://")) return prepend~rest; 
    3444  if (!newAddr.length) throw new Exception("Empty newAddr found! Aborting followLink."); 
    3545  if (newAddr[0]=='/') return prepend~oldAddr.split("/")[0]~newAddr; 
    …… 
    96106  if (url=="") throw new Exception("URL empty"); 
    97107  if (url.beginsWith("data://")) return "<data>"~url[7 .. $]; 
    98   string post=""; 
     108  string post; 
    99109  if (auto rest=url.startsWith("POST=")) { 
    100110    url=rest; 
    101111    post=url[0..url.find(" ")]; 
    102112    url=url[url.find(" ")+1..$]; 
    103     logln("Posting ", post); 
     113    if (!quiet) logln("Posting ", post); 
    104114  } 
    105115  string refer; 
    …… 
    108118    url=rest[rest.find(" ")+1..$]; 
    109119    if (refer.length<7 || refer[0..7]!="http://") refer="http://"~refer; // referrer needs that 
    110     logln("Refer ", refer); 
     120    if (!quiet) logln("Refer ", refer); 
     121  } 
     122  string cookie; 
     123  if (auto rest=url.startsWith("COOKIE=")) { 
     124    cookie=rest[0..rest.find(" ")]; 
     125    url=rest[rest.find(" ")+1..$]; 
     126    if (!quiet) logln("Cookie ", cookie); 
    111127  } 
    112128  bool http_1_0=false; 
    113129  if (auto rest=url.startsWith("1.0 ")) { 
    114130    url=rest; 
    115     logln("Fallback to HTTP 1.0"); 
     131    if (!quiet) logln("Fallback to HTTP 1.0"); 
    116132    http_1_0=true; 
    117133  } 
    …… 
    177193  if (!retries) throw new Exception("Can't download page."); 
    178194  retries--; 
    179   //logln("Downloading ", folder); 
    180195  if (post.length) sock=openSocket("POST "~folder~" HTTP/"~(http_1_0?"1.0":"1.1")~"\r\n"); 
    181196  else sock=openSocket("GET "~folder~" HTTP/"~(http_1_0?"1.0":"1.1")~"\r\n"); 
    …… 
    189204  if (refer.length) 
    190205    sock.send("Referer: "~refer~"\r\n"); 
     206  if (cookie.length) 
     207    sock.send("Cookie: "~cookie~"\r\n"); 
    191208  sock.send("\r\n"~post); 
    192209  string received; 
    …… 
    202219    if (ifind(header, "\r\ncontent-length:")==-1) { 
    203220      if (ifind(header, "\r\ntransfer-encoding: chunked")!=-1) { 
    204         logln("Recognize chunked mode"); 
     221        if (!quiet) logln("Recognize chunked mode"); 
    205222        chunked=true; 
    206223      } 
    …… 
    216233      if (auto rest = line.startsWith("Location: ")) { 
    217234        string loc = param.followLink(rest); 
    218         logln("Following redirect to ", loc); 
     235        if (!quiet) logln("Following redirect to ", loc); 
    219236        if (redirect) { *redirect=loc; res=null; return true; } 
    220237        void delegate(float) dg=(float f) { download_callbacks.doCbs(param, f); }; 
    …… 
    254271    } 
    255272    auto chars=hex2ulong(line); 
    256     //logln("line ", line, ", chars ", chars, ", received now ", received.length, ", line ends at ", line_end); 
    257273    if (chars) { 
    258274      if (received.length < line_end+2) return true; 
    …… 
    285301    if (chunked) if (receiveChunked) continue; else break; 
    286302    string _res; if (checkProgress(chunked, _res)) return _res; 
    287     if (length && (received.length !< length)) { logln("Received all data"); break; } 
     303    if (length && (received.length !< length)) { if (!quiet) logln("Received all data"); break; } 
    288304  } 
    289305  if (!header.length) { 
    290     logln("Connection closed, but no body found. Presuming invalid persistant connection; retrying ... "); 
     306    if (!quiet) logln("Connection closed, but no body found. Presuming invalid persistant connection; retrying ... "); 
    291307    goto retry; 
    292308  } 

  • trunk/tools/tools/log.d

    r361 r362  
    77int getThread() { foreach (int idx, thr; Thread.getAll) if (thr.isSelf) return idx; throw new Exception("Not in a known thread"); } 
    88 
     9bool log_threads = true; 
     10 
    911// This _needs_ to be outside the function because it's a template func 
    1012Object log_sync=null; 
    1113FILE * logfile=null; 
    1214static ~this() { 
    13   if (lastThread) log("\\", repeat("-", Format("+--[ ", lastThread, " ]------").length-1), "\n"); 
     15  if (log_threads && lastThread) log("\\", repeat("-", Format("+--[ ", lastThread, " ]------").length-1), "\n"); 
    1416  if (logfile) fclose(logfile); 
    1517} 
    …… 
    7981  if (!log_sync) log_sync = new Object; 
    8082  synchronized (log_sync) { 
    81     if (lastThread !is thr) 
     83    if (log_threads && (lastThread !is thr)) 
    8284      _log(prefix, "--[ ", threadname, " ]------", "\n"); 
    83     _log("| ", t, "\n"); 
     85    if (log_threads) _log("| ", t, "\n"); 
     86    else _log(t, "\n"); 
    8487  } 
    8588  lastThread=thr; 

  • trunk/tools/tools/threadpool.d

    r359 r362  
    2222  bool finish=false; 
    2323  MessageMultiChannel!(task, false, false) tasks; 
     24  int getThreadId() { foreach (id, thread; threads) if (thread.isSelf) return id; throw new Exception("Not a thread of this pool!"); } 
    2425  task getTask() { if (finish) return task("ShutdownThreadsBusy", &shutdown_me); else return tasks.get(); } 
    2526  void addTask(string name, proc c) { tasks.put(task(name, c)); }