sys/src/cmd/webfs/http.c

   1 #include <u.h>
   2 #include <libc.h>
   3 #include <ctype.h>
   4 #include <fcall.h>
   5 #include <thread.h>
   6 #include <9p.h>
   7
   8 #include "dat.h"
   9 #include "fns.h"
  10
  11 #include <auth.h>
  12 #include <mp.h>
  13 #include <libsec.h>
  14
  15 typedef struct Hconn Hconn;
  16 typedef struct Hpool Hpool;
  17 typedef struct Hauth Hauth;
  18
  19 struct Hconn
  20 {
  21         Hconn   *next;
  22         long    time;
  23
  24         int     fd;
  25         int     ctl;
  26         int     keep;
  27         int     cancel;
  28         int     tunnel;
  29         int     len;
  30         char    addr[128];
  31         char    buf[8192+2];
  32 };
  33
  34 struct Hpool
  35 {
  36         QLock;
  37
  38         Hconn   *head;
  39         int     active;
  40
  41         int     limit;
  42         int     peer;
  43         int     idle;
  44 };
  45
  46 struct Hauth
  47 {
  48         Hauth   *next;
  49         Url     *url;
  50         char    *auth;
  51 };
  52
  53 static Hpool hpool = {
  54         .limit  = 16,
  55         .peer   = 4,
  56         .idle   = 5,    /* seconds */
  57 };
  58
  59 static QLock authlk;
  60 static Hauth *hauth;
  61
  62 static void hclose(Hconn *h);
  63
  64 static int
  65 tlstrace(char *fmt, ...)
  66 {
  67         int r;
  68         va_list a;
  69         va_start(a, fmt);
  70         r = vfprint(2, fmt, a);
  71         va_end(a);
  72         return r;
  73 }
  74
  75 static int
  76 tlswrap(int fd, char *servername)
  77 {
  78         TLSconn conn;
  79
  80         memset(&conn, 0, sizeof(conn));
  81         if(debug)
  82                 conn.trace = tlstrace;
  83         if(servername != nil)
  84                 conn.serverName = smprint("%N", servername);
  85         if((fd = tlsClient(fd, &conn)) < 0){
  86                 if(debug) fprint(2, "tlsClient: %r\n");
  87                 return -1;
  88         }
  89         free(conn.cert);
  90         free(conn.sessionID);
  91         free(conn.serverName);
  92         return fd;
  93 }
  94
  95 static Hconn*
  96 hdial(Url *u)
  97 {
  98         char addr[128];
  99         Hconn *h, *p;
 100         int fd, ctl;
 101
 102         snprint(addr, sizeof(addr), "tcp!%s!%s", u->host, u->port ? u->port : u->scheme);
 103
 104         qlock(&hpool);
 105         for(p = nil, h = hpool.head; h; p = h, h = h->next){
 106                 if(strcmp(h->addr, addr) == 0){
 107                         if(p)
 108                                 p->next = h->next;
 109                         else
 110                                 hpool.head = h->next;
 111                         h->next = nil;
 112                         qunlock(&hpool);
 113                         return h;
 114                 }
 115         }
 116         hpool.active++;
 117         qunlock(&hpool);
 118
 119         if(debug)
 120                 fprint(2, "hdial [%d] %s\n", hpool.active, addr);
 121
 122         if(proxy)
 123                 snprint(addr, sizeof(addr), "tcp!%s!%s",
 124                         proxy->host, proxy->port ? proxy->port : proxy->scheme);
 125
 126         if((fd = dial(addr, 0, 0, &ctl)) >= 0){
 127                 if(proxy){
 128                         if(strcmp(proxy->scheme, "https") == 0)
 129                                 fd = tlswrap(fd, proxy->host);
 130                 } else {
 131                         if(strcmp(u->scheme, "https") == 0)
 132                                 fd = tlswrap(fd, u->host);
 133                 }
 134         }
 135         if(fd < 0){
 136                 close(ctl);
 137                 return nil;
 138         }
 139
 140         h = emalloc(sizeof(*h));
 141         h->next = nil;
 142         h->time = 0;
 143         h->cancel = 0;
 144         h->tunnel = 0;
 145         h->keep = 1;
 146         h->len = 0;
 147         h->fd = fd;
 148         h->ctl = ctl;
 149
 150         if(proxy){
 151                 h->tunnel = strcmp(u->scheme, "https") == 0;
 152                 snprint(addr, sizeof(addr), "tcp!%s!%s", u->host, u->port ? u->port : u->scheme);
 153         }
 154         nstrcpy(h->addr, addr, sizeof(h->addr));
 155
 156         return h;
 157 }
 158
 159 static void
 160 hcloseall(Hconn *x)
 161 {
 162         Hconn *h;
 163
 164         while(h = x){
 165                 x = h->next;
 166                 h->next = nil;
 167                 h->keep = 0;
 168                 hclose(h);
 169         }
 170 }
 171
 172 static void
 173 hclose(Hconn *h)
 174 {
 175         Hconn *x, *t;
 176         int i, n;
 177
 178         if(h == nil)
 179                 return;
 180
 181         qlock(&hpool);
 182         if(!h->tunnel && h->keep && h->fd >= 0){
 183                 for(n = 0, i = 0, t = nil, x = hpool.head; x; x = x->next){
 184                         if(strcmp(x->addr, h->addr) == 0)
 185                                 if(++n > hpool.peer)
 186                                         break;
 187                         if(++i < hpool.limit)
 188                                 t = x;
 189                 }
 190                 if(x == nil){
 191                         /* return connection to pool */
 192                         h->time = time(0);
 193                         h->next = hpool.head;
 194                         hpool.head = h;
 195
 196                         /* cut off tail */
 197                         if(t){
 198                                 x = t->next;
 199                                 t->next = nil;
 200                         }
 201
 202                         i = h->next != nil;
 203                         qunlock(&hpool);
 204
 205                         /* free the tail */
 206                         hcloseall(x);
 207
 208                         /*
 209                          * if h is first one in pool, spawn proc to close
 210                          * idle connections.
 211                          */
 212                         if(i == 0)
 213                         if(rfork(RFMEM|RFPROC|RFNOWAIT) == 0){
 214                                 do {
 215                                         Hconn **xx;
 216                                         long now;
 217
 218                                         sleep(1000);
 219
 220                                         qlock(&hpool);
 221                                         now = time(0);
 222
 223                                         x = nil;
 224                                         xx = &hpool.head;
 225                                         while(h = *xx){
 226                                                 if((now - h->time) > hpool.idle){
 227                                                         *xx = h->next;
 228
 229                                                         /* link to tail */
 230                                                         h->next = x;
 231                                                         x = h;
 232                                                         continue;
 233                                                 }
 234                                                 xx = &h->next;
 235                                         }
 236
 237                                         i = hpool.head != nil;
 238                                         qunlock(&hpool);
 239
 240                                         /* free the tail */
 241                                         hcloseall(x);
 242                                 } while(i);
 243                                 exits(nil);
 244                         }
 245                         return;
 246                 }
 247         }
 248         hpool.active--;
 249         qunlock(&hpool);
 250
 251         if(debug)
 252                 fprint(2, "hclose [%d] %s\n", hpool.active, h->addr);
 253
 254         if(h->ctl >= 0)
 255                 close(h->ctl);
 256         if(h->fd >= 0)
 257                 close(h->fd);
 258         free(h);
 259 }
 260
 261 static void
 262 hhangup(Hconn *h)
 263 {
 264         if(debug)
 265                 fprint(2, "hangup pc=%p: %r\n", getcallerpc(&h));
 266         h->keep = 0;
 267         if(h->ctl >= 0)
 268                 hangup(h->ctl);
 269 }
 270
 271 static int
 272 hread(Hconn *h, void *data, int len)
 273 {
 274         if(h->len > 0){
 275                 if(len > h->len)
 276                         len = h->len;
 277                 memmove(data, h->buf, len);
 278                 h->len -= len;
 279                 if(h->len > 0)
 280                         memmove(h->buf, h->buf + len, h->len);
 281                 return len;
 282         }
 283         if((len = read(h->fd, data, len)) == 0)
 284                 h->keep = 0;
 285         if(len < 0)
 286                 hhangup(h);
 287         return len;
 288 }
 289
 290 static int
 291 hwrite(Hconn *h, void *data, int len)
 292 {
 293         if(write(h->fd, data, len) != len){
 294                 hhangup(h);
 295                 return -1;
 296         }
 297         return len;
 298 }
 299
 300 static int
 301 hline(Hconn *h, char *data, int len, int cont)
 302 {
 303         char *x, *y, *e;
 304         int n;
 305
 306         data[0] = 0;
 307         for(;;){
 308                 if(h->len > 0){
 309                         while(x = memchr(h->buf, '\n', h->len)){
 310                                 n = x - h->buf;
 311                                 if(n > 0 && x[-1] == '\r')
 312                                         n--;
 313                                 if(n > 0 && cont){
 314                                         e = h->buf + h->len;
 315                                         for(y = x+1; y < e; y++)
 316                                                 if(*y != ' ' && *y != '\t')
 317                                                         break;
 318                                         if(y >= e || *y == 0)
 319                                                 break;
 320                                         if(y > x+1){
 321                                                 if(x > h->buf && x[-1] == '\r')
 322                                                         x--;
 323                                                 memmove(x, y, e - y);
 324                                                 h->len -= y - x;
 325                                                 continue;
 326                                         }
 327                                 }
 328                                 if(n < len)
 329                                         len = n;
 330                                 memmove(data, h->buf, len);
 331                                 data[len] = 0;
 332                                 h->len -= (++x - h->buf);
 333                                 if(h->len > 0)
 334                                         memmove(h->buf, x, h->len);
 335                                 return len;
 336                         }
 337                 }
 338                 n = sizeof(h->buf) - h->len;
 339                 if(n <= 0)
 340                         return 0;
 341                 if(h->tunnel)
 342                         n = 1;  /* do not read beyond header */
 343                 if((n = read(h->fd, h->buf + h->len, n)) <= 0){
 344                         hhangup(h);
 345                         return -1;
 346                 }
 347                 h->len += n;
 348         }
 349 }
 350
 351 int
 352 authenticate(Url *u, Url *ru, char *method, char *s)
 353 {
 354         char *user, *pass, *realm, *nonce, *opaque, *x;
 355         Hauth *a;
 356         Fmt fmt;
 357         int n;
 358
 359         user = u->user;
 360         pass = u->pass;
 361         realm = nonce = opaque = nil;
 362         if(!cistrncmp(s, "Basic ", 6)){
 363                 UserPasswd *up;
 364
 365                 s += 6;
 366                 if(x = cistrstr(s, "realm="))
 367                         realm = unquote(x+6, &s);
 368                 if(realm == nil)
 369                         return -1;
 370                 up = nil;
 371                 if(user == nil || pass == nil){
 372                         fmtstrinit(&fmt);
 373                         fmtprint(&fmt, " realm=%q", realm);
 374                         if(user)
 375                                 fmtprint(&fmt, " user=%q", user);
 376                         if((s = fmtstrflush(&fmt)) == nil)
 377                                 return -1;
 378                         up = auth_getuserpasswd(nil, "proto=pass service=http server=%q%s", u->host, s);
 379                         free(s);
 380                         if(up == nil)
 381                                 return -1;
 382                         user = up->user;
 383                         pass = up->passwd;
 384                 }
 385                 fmtstrinit(&fmt);
 386                 fmtprint(&fmt, "%s:%s", user ? user : "", pass ? pass : "");
 387                 if(up){
 388                         memset(up->user, 0, strlen(up->user));
 389                         memset(up->passwd, 0, strlen(up->passwd));
 390                         free(up);
 391                 }
 392                 if((s = fmtstrflush(&fmt)) == nil)
 393                         return -1;
 394                 n = strlen(s);
 395                 fmtstrinit(&fmt);
 396                 fmtprint(&fmt, "Basic %.*[", n, s);
 397                 memset(s, 0, n);
 398                 free(s);
 399                 u = saneurl(url(".", u));       /* all uris below the requested one */
 400         }else
 401         if(!cistrncmp(s, "Digest ", 7)){
 402                 char chal[1024], ouser[128], resp[2*MD5LEN+1];
 403                 int nchal;
 404
 405                 s += 7;
 406                 if(x = cistrstr(s, "realm="))
 407                         realm = unquote(x+6, &s);
 408                 if(x = cistrstr(s, "nonce="))
 409                         nonce = unquote(x+6, &s);
 410                 if(x = cistrstr(s, "opaque="))
 411                         opaque = unquote(x+7, &s);
 412                 if(realm == nil || nonce == nil)
 413                         return -1;
 414                 fmtstrinit(&fmt);
 415                 fmtprint(&fmt, " realm=%q", realm);
 416                 if(user)
 417                         fmtprint(&fmt, " user=%q", user);
 418                 if((s = fmtstrflush(&fmt)) == nil)
 419                         return -1;
 420                 nchal = snprint(chal, sizeof(chal), "%s %s %U", nonce, method, ru);
 421                 n = auth_respond(chal, nchal, ouser, sizeof ouser, resp, sizeof resp, nil,
 422                         "proto=httpdigest role=client server=%q%s", u->host, s);
 423                 memset(chal, 0, sizeof(chal));
 424                 free(s);
 425                 if(n < 0)
 426                         return -1;
 427                 fmtstrinit(&fmt);
 428                 fmtprint(&fmt, "Digest ");
 429                 fmtprint(&fmt, "username=\"%s\", ", ouser);
 430                 fmtprint(&fmt, "realm=\"%s\", ", realm);
 431                 fmtprint(&fmt, "host=\"%N\", ", u->host);
 432                 fmtprint(&fmt, "uri=\"%U\", ", ru);
 433                 fmtprint(&fmt, "nonce=\"%s\", ", nonce);
 434                 fmtprint(&fmt, "response=\"%s\"", resp);
 435                 if(opaque)
 436                         fmtprint(&fmt, ", opaque=\"%s\"", opaque);
 437                 u = saneurl(url("/", u));       /* BUG: should be the ones in domain= only */
 438         } else
 439                 return -1;
 440         if((s = fmtstrflush(&fmt)) == nil){
 441                 freeurl(u);
 442                 return -1;
 443         }
 444         if(u == nil){
 445                 free(s);
 446                 return -1;
 447         }
 448         a = emalloc(sizeof(*a));
 449         a->url = u;
 450         a->auth = s;
 451         qlock(&authlk);
 452         a->next = hauth;
 453         hauth = a;
 454         qunlock(&authlk);
 455
 456         return 0;
 457 }
 458
 459 int
 460 hauthenticate(Url *u, Url *ru, char *method, char *key, Key *hdr)
 461 {
 462         for(hdr = getkey(hdr, key); hdr != nil; hdr = getkey(hdr->next, key))
 463                 if(authenticate(u, ru, method, hdr->val) == 0)
 464                         return 0;
 465         return -1;
 466 }
 467
 468 void
 469 flushauth(Url *u, char *t)
 470 {
 471         Hauth *a, *p;
 472
 473         qlock(&authlk);
 474 Again:
 475         for(p = nil, a = hauth; a; p = a, a = a->next)
 476                 if(matchurl(u, a->url) && (t == nil || !strcmp(t, a->auth))){
 477                         if(p)
 478                                 p->next = a->next;
 479                         else
 480                                 hauth = a->next;
 481                         if(debug)
 482                                 fprint(2, "flushauth for %U\n", a->url);
 483                         freeurl(a->url);
 484                         memset(a->auth, 0, strlen(a->auth));
 485                         free(a->auth);
 486                         free(a);
 487                         goto Again;
 488                 }
 489         qunlock(&authlk);
 490 }
 491
 492 static void
 493 catch(void *, char *msg)
 494 {
 495         if(strstr("alarm", msg) != nil)
 496                 noted(NCONT);
 497         else
 498                 noted(NDFLT);
 499 }
 500
 501 #define NOLENGTH 0x7fffffffffffffffLL
 502
 503 void
 504 http(char *m, Url *u, Key *shdr, Buq *qbody, Buq *qpost)
 505 {
 506         int i, l, n, try, pid, fd, cfd, needlength, chunked, retry, nobody;
 507         char *s, *x, buf[8192+2], status[256], method[16], *host;
 508         vlong length, offset;
 509         Url ru, tu, *nu;
 510         Key *k, *rhdr;
 511         Hconn *h;
 512         Hauth *a;
 513
 514         incref(qbody);
 515         if(qpost) incref(qpost);
 516         nstrcpy(method, m, sizeof(method));
 517         switch(rfork(RFPROC|RFMEM|RFNOWAIT)){
 518         default:
 519                 return;
 520         case -1:
 521                 buclose(qbody, "can't fork");
 522                 bufree(qbody);
 523                 buclose(qpost, "can't fork");
 524                 bufree(qpost);
 525                 while(k = shdr){
 526                         shdr = k->next;
 527                         free(k);
 528                 }
 529                 freeurl(u);
 530                 return;
 531         case 0:
 532                 break;
 533         }
 534
 535         notify(catch);
 536         if(qpost){
 537                 /* file for spooling the postbody if we need to restart the request */
 538                 snprint(buf, sizeof(buf), "/tmp/http.%d.%d.post", getppid(), getpid());
 539                 fd = create(buf, OEXCL|ORDWR|ORCLOSE, 0600);
 540         } else
 541                 fd = -1;
 542
 543         h = nil;
 544         cfd = -1;
 545         pid = 0;
 546         host = nil;
 547         needlength = 0;
 548         for(try = 0; try < 12; try++){
 549                 strcpy(status, "0 No status");
 550                 if(u == nil || (strcmp(u->scheme, "http") && strcmp(u->scheme, "https"))){
 551                         werrstr("bad url scheme");
 552                         break;
 553                 }
 554
 555                 if(debug)
 556                         fprint(2, "http(%d): %s %U\n", try, method, u);
 557
 558                 /* preemptive authentication from hauth cache */
 559                 qlock(&authlk);
 560                 if(proxy && !lookkey(shdr, "Proxy-Authorization"))
 561                         for(a = hauth; a; a = a->next)
 562                                 if(matchurl(a->url, proxy)){
 563                                         shdr = addkey(shdr, "Proxy-Authorization", a->auth);
 564                                         break;
 565                                 }
 566                 if(!lookkey(shdr, "Authorization"))
 567                         for(a = hauth; a; a = a->next)
 568                                 if(matchurl(a->url, u)){
 569                                         shdr = addkey(shdr, "Authorization", a->auth);
 570                                         break;
 571                                 }
 572                 qunlock(&authlk);
 573
 574                 length = 0;
 575                 chunked = 0;
 576                 if(qpost){
 577                         /* have to read it to temp file to figure out the length */
 578                         if(fd >= 0 && needlength && lookkey(shdr, "Content-Length") == nil){
 579                                 seek(fd, 0, 2);
 580                                 while((n = buread(qpost, buf, sizeof(buf))) > 0)
 581                                         write(fd, buf, n);
 582                                 shdr = delkey(shdr, "Transfer-Encoding");
 583                         }
 584
 585                         qlock(qpost);
 586                         /* wait until buffer is full, most posts are small */
 587                         while(!qpost->closed && qpost->size < qpost->limit && qpost->nwq == 0)
 588                                 rsleep(&qpost->rz);
 589
 590                         if(lookkey(shdr, "Content-Length"))
 591                                 chunked = 0;
 592                         else if(x = lookkey(shdr, "Transfer-Encoding"))
 593                                 chunked = cistrstr(x, "chunked") != nil;
 594                         else if(chunked = !qpost->closed)
 595                                 shdr = addkey(shdr, "Transfer-Encoding", "chunked");
 596                         else if(qpost->closed){
 597                                 if(fd >= 0){
 598                                         length = seek(fd, 0, 2);
 599                                         if(length < 0)
 600                                                 length = 0;
 601                                 }
 602                                 length += qpost->size;
 603                                 snprint(buf, sizeof(buf), "%lld", length);
 604                                 shdr = addkey(shdr, "Content-Length", buf);
 605                         }
 606                         qunlock(qpost);
 607                 }
 608
 609                 /* http requires ascii encoding of host */
 610                 free(host);
 611                 host = smprint("%N", u->host);
 612
 613                 if(proxy && strcmp(u->scheme, "https") != 0){
 614                         ru = *u;
 615                         ru.host = host;
 616                         ru.fragment = nil;
 617                 } else {
 618                         memset(&ru, 0, sizeof(ru));
 619                         ru.path = Upath(u);
 620                         ru.query = u->query;
 621                 }
 622                 n = snprint(buf, sizeof(buf), "%s %U HTTP/1.1\r\nHost: %s%s%s\r\n",
 623                         method, &ru, host, u->port ? ":" : "", u->port ? u->port : "");
 624                 if(n >= sizeof(buf)-64){
 625                         werrstr("request too large");
 626                         break;
 627                 }
 628                 if(h == nil){
 629                         alarm(timeout);
 630                         if((h = hdial(u)) == nil)
 631                                 break;
 632                 }
 633                 if(h->tunnel){
 634                         n = snprint(buf, sizeof(buf), "CONNECT %s:%s HTTP/1.1\r\nHost: %s:%s\r\n",
 635                                 host, u->port ? u->port : "443",
 636                                 host, u->port ? u->port : "443");
 637                 }
 638                 else if((cfd = open("/mnt/webcookies/http", ORDWR)) >= 0){
 639                         /* only scheme, host and path are relevant for cookies */
 640                         memset(&tu, 0, sizeof(tu));
 641                         tu.scheme = u->scheme;
 642                         tu.host = host;
 643                         tu.path = Upath(u);
 644                         fprint(cfd, "%U", &tu);
 645                         for(;;){
 646                                 if(n >= sizeof(buf)-2){
 647                                         if(debug)
 648                                                 fprint(2, "-> %.*s", n, buf);
 649                                         if(hwrite(h, buf, n) != n)
 650                                                 goto Badflush;
 651                                         n = 0;
 652                                 }
 653                                 if((l = read(cfd, buf+n, sizeof(buf)-2 - n)) == 0)
 654                                         break;
 655                                 if(l < 0){
 656                                         close(cfd);
 657                                         cfd = -1;
 658                                         break;
 659                                 }
 660                                 n += l;
 661                         }
 662                 }
 663
 664                 for(k = shdr; k; k = k->next){
 665                         /* only send proxy headers when establishing tunnel */
 666                         if(h->tunnel && cistrncmp(k->key, "Proxy-", 6) != 0)
 667                                 continue;
 668                         n += snprint(buf+n, sizeof(buf)-2 - n, "%s: %s\r\n", k->key, k->val);
 669                 }
 670                 n += snprint(buf+n, sizeof(buf)-n, "\r\n");
 671                 if(debug)
 672                         fprint(2, "-> %.*s", n, buf);
 673                 if(hwrite(h, buf, n) != n){
 674                 Badflush:
 675                         alarm(0);
 676                         goto Retry;
 677                 }
 678
 679                 if(qpost && !h->tunnel){
 680                         h->cancel = 0;
 681                         if((pid = rfork(RFMEM|RFPROC)) <= 0){
 682                                 int ifd;
 683
 684                                 if((ifd = fd) >= 0)
 685                                         seek(ifd, 0, 0);
 686                                 while(!h->cancel){
 687                                         alarm(0);
 688                                         if((ifd < 0) || ((n = read(ifd, buf, sizeof(buf)-2)) <= 0)){
 689                                                 ifd = -1;
 690                                                 if((n = buread(qpost, buf, sizeof(buf)-2)) <= 0)
 691                                                         break;
 692                                                 if(fd >= 0)
 693                                                         if(write(fd, buf, n) != n)
 694                                                                 break;
 695                                         }
 696                                         alarm(timeout);
 697                                         if(chunked){
 698                                                 char tmp[32];
 699                                                 if(hwrite(h, tmp, snprint(tmp, sizeof(tmp), "%x\r\n", n)) < 0)
 700                                                         break;
 701                                                 buf[n++] = '\r';
 702                                                 buf[n++] = '\n';
 703                                         }
 704                                         if(hwrite(h, buf, n) != n)
 705                                                 break;
 706                                 }
 707                                 if(chunked){
 708                                         alarm(timeout);
 709                                         hwrite(h, "0\r\n\r\n", 5);
 710                                 }else
 711                                         h->keep = 0;
 712                                 if(pid == 0)
 713                                         exits(nil);
 714                         }
 715                         /* no timeout when posting */
 716                         alarm(0);
 717                 } else
 718                         alarm(timeout);
 719
 720                 Cont:
 721                 rhdr = 0;
 722                 retry = 0;
 723                 chunked = 0;
 724                 offset = 0;
 725                 length = NOLENGTH;
 726                 for(l = 0; hline(h, s = buf, sizeof(buf)-1, 1) > 0; l++){
 727                         if(debug)
 728                                 fprint(2, "<- %s\n", s);
 729                         if(l == 0){
 730                                 if(x = strchr(s, ' '))
 731                                         while(*x == ' ')
 732                                                 *x++ = 0;
 733                                 if(cistrncmp(s, "HTTP", 4)){
 734                                         h->keep = 0;
 735                                         if(cistrcmp(s, "ICY"))
 736                                                 break;
 737                                 }
 738                                 if(x[0])
 739                                         nstrcpy(status, x, sizeof(status));
 740                                 continue;
 741                         }
 742                         if((k = parsehdr(s)) == nil)
 743                                 continue;
 744                         if(!cistrcmp(k->key, "Connection")){
 745                                 if(cistrstr(k->val, "close"))
 746                                         h->keep = 0;
 747                         }
 748                         else if(!cistrcmp(k->key, "Content-Length"))
 749                                 length = atoll(k->val);
 750                         else if(!cistrcmp(k->key, "Transfer-Encoding")){
 751                                 if(cistrstr(k->val, "chunked"))
 752                                         chunked = 1;
 753                         }
 754                         else if(!cistrcmp(k->key, "Set-Cookie") ||
 755                                 !cistrcmp(k->key, "Set-Cookie2")){
 756                                 if(cfd >= 0)
 757                                         fprint(cfd, "Set-Cookie: %s\n", k->val);
 758                                 free(k);
 759                                 continue;
 760                         }
 761                         k->next = rhdr;
 762                         rhdr = k;
 763                 }
 764                 alarm(0);
 765                 if(cfd >= 0){
 766                         close(cfd);
 767                         cfd = -1;
 768                 }
 769
 770                 nobody = !cistrcmp(method, "HEAD");
 771                 if((i = atoi(status)) < 0)
 772                         i = 0;
 773                 Status:
 774                 switch(i){
 775                 default:
 776                         if(i % 100){
 777                                 i -= (i % 100);
 778                                 goto Status;
 779                         }
 780                 case 100:       /* Continue */
 781                 case 101:       /* Switching Protocols */
 782                         while(k = rhdr){
 783                                 rhdr = k->next;
 784                                 free(k);
 785                         }
 786                         goto Cont;
 787                 case 304:       /* Not Modified */
 788                         nobody = 1;
 789                 case 305:       /* Use Proxy */
 790                 case 400:       /* Bad Request */
 791                 case 402:       /* Payment Required */
 792                 case 403:       /* Forbidden */
 793                 case 404:       /* Not Found */
 794                 case 405:       /* Method Not Allowed */
 795                 case 406:       /* Not Acceptable */
 796                 case 408:       /* Request Timeout */
 797                 case 409:       /* Conflict */
 798                 case 410:       /* Gone */
 799                         goto Error;
 800                 case 411:       /* Length Required */
 801                         if(qpost){
 802                                 needlength = 1;
 803                                 h->cancel = 1;
 804                                 retry = 1;
 805                                 break;
 806                         }
 807                 case 412:       /* Precondition Failed */
 808                 case 413:       /* Request Entity Too Large */
 809                 case 414:       /* Request URI Too Large */
 810                 case 415:       /* Unsupported Media Type */
 811                 case 416:       /* Requested Range Not Satisfiable */
 812                 case 417:       /* Expectation Failed */
 813                 case 500:       /* Internal server error */
 814                 case 501:       /* Not implemented */
 815                 case 502:       /* Bad gateway */
 816                 case 503:       /* Service unavailable */
 817                 case 504:       /* Gateway Timeout */
 818                 case 505:       /* HTTP Version not Supported */
 819                 Error:
 820                         h->cancel = 1;
 821                         buclose(qbody, status);
 822                         buclose(qpost, status);
 823                         break;
 824                 case 300:       /* Multiple choices */
 825                 case 302:       /* Found */
 826                 case 303:       /* See Other */
 827                         if(qpost){
 828                                 if(pid > 0){
 829                                         waitpid();
 830                                         pid = 0;
 831                                 }
 832                                 buclose(qpost, 0);
 833                                 bufree(qpost);
 834                                 qpost = nil;
 835                         }
 836                         shdr = delkey(shdr, "Content-Length");
 837                         shdr = delkey(shdr, "Content-Type");
 838                         shdr = delkey(shdr, "Transfer-Encoding");
 839                         if(cistrcmp(method, "HEAD"))
 840                                 nstrcpy(method, "GET", sizeof(method));
 841                 case 301:       /* Moved Permanently */
 842                 case 307:       /* Temporary Redirect */
 843                 case 308:       /* Resume Incomplete */
 844                         if((x = lookkey(rhdr, "Location")) == nil)
 845                                 goto Error;
 846                         if((nu = saneurl(url(x, u))) == nil)
 847                                 goto Error;
 848                         freeurl(u);
 849                         u = nu;
 850                         if(0){
 851                 case 401:       /* Unauthorized */
 852                         if(x = lookkey(shdr, "Authorization"))
 853                                 flushauth(nil, x);
 854                         if(hauthenticate(u, &ru, method, "WWW-Authenticate", rhdr) < 0)
 855                                 goto Error;
 856                         }
 857                         if(0){
 858                 case 407:       /* Proxy Auth */
 859                         if(proxy == nil)
 860                                 goto Error;
 861                         if(x = lookkey(shdr, "Proxy-Authorization"))
 862                                 flushauth(proxy, x);
 863                         if(hauthenticate(proxy, proxy, method, "Proxy-Authenticate", rhdr) < 0)
 864                                 goto Error;
 865                         }
 866                 case 0:         /* No status */
 867                         if(qpost && fd < 0){
 868                                 if(i > 0)
 869                                         goto Error;
 870                                 break;
 871                         }
 872                         h->cancel = 1;
 873                         retry = 1;
 874                         break;
 875                 case 204:       /* No Content */
 876                 case 205:       /* Reset Content */
 877                         nobody = 1;
 878                 case 200:       /* OK */
 879                 case 201:       /* Created */
 880                 case 202:       /* Accepted */
 881                 case 203:       /* Non-Authoritative Information */
 882                 case 206:       /* Partial Content */
 883                         if(h->tunnel)
 884                                 break;
 885                         qbody->url = u; u = nil;
 886                         qbody->hdr = rhdr; rhdr = nil;
 887                         if(nobody)
 888                                 buclose(qbody, 0);
 889                         break;
 890                 }
 891
 892                 while(k = rhdr){
 893                         rhdr = k->next;
 894                         free(k);
 895                 }
 896
 897                 /*
 898                  * remove authorization headers so on the next round, we use
 899                  * the hauth cache (wich checks the scope url). this makes
 900                  * sure we wont send credentials to the wrong url after
 901                  * a redirect.
 902                  */
 903                 shdr = delkey(shdr, "Proxy-Authorization");
 904                 shdr = delkey(shdr, "Authorization");
 905
 906                 /*
 907                  * when 2xx response is given for the CONNECT request
 908                  * then the proxy server has established the connection.
 909                  */
 910                 if(h->tunnel && !retry && (i/100) == 2){
 911                         if((h->fd = tlswrap(h->fd, host)) < 0)
 912                                 break;
 913
 914                         /* proceed to the original request */
 915                         h->tunnel = 0;
 916                         continue;
 917                 }
 918
 919                 if(!chunked && length == NOLENGTH)
 920                         h->keep = 0;
 921
 922                 /*
 923                  * read the response body (if any). retry means we'r just
 924                  * skipping the error page so we wont touch qbody.
 925                  */
 926                 while(!nobody){
 927                         if((qbody->closed || retry) && !h->keep)
 928                                 break;
 929                         if(chunked){
 930                                 if(hline(h, buf, sizeof(buf)-1, 0) <= 0)
 931                                         break;
 932                                 length = strtoll(buf, nil, 16);
 933                                 offset = 0;
 934                         }
 935                         while(offset < length){
 936                                 l = sizeof(buf);
 937                                 if(l > (length - offset))
 938                                         l = (length - offset);
 939                                 if((n = hread(h, buf, l)) <= 0)
 940                                         break;
 941                                 offset += n;
 942                                 if(!retry)
 943                                         if(buwrite(qbody, buf, n) != n)
 944                                                 break;
 945                         }
 946                         if(offset != length){
 947                                 h->keep = 0;
 948                                 if(length != NOLENGTH)
 949                                         break;
 950                         }
 951                         if(chunked){
 952                                 while(hline(h, buf, sizeof(buf)-1, 1) > 0){
 953                                         if(debug)
 954                                                 fprint(2, "<= %s\n", buf);
 955                                         if(!retry)
 956                                                 if(k = parsehdr(buf)){
 957                                                         k->next = qbody->hdr;
 958                                                         qbody->hdr = k;
 959                                                 }
 960                                 }
 961                                 if(length > 0)
 962                                         continue;
 963                         }
 964                         if(!retry)
 965                                 buclose(qbody, 0);
 966                         break;
 967                 }
 968
 969                 if(!retry)
 970                         break;
 971                 Retry:
 972                 if(cfd >= 0)
 973                         close(cfd);
 974                 if(pid > 0){
 975                         waitpid();
 976                         pid = 0;
 977                 }
 978                 hclose(h);
 979                 h = nil;
 980         }
 981         alarm(0);
 982         snprint(buf, sizeof(buf), "%s %r", status);
 983         buclose(qbody, buf);
 984         bufree(qbody);
 985
 986         if(qpost){
 987                 if(pid > 0)
 988                         waitpid();
 989                 buclose(qpost, buf);
 990                 bufree(qpost);
 991         }
 992         if(fd >= 0)
 993                 close(fd);
 994
 995         hclose(h);
 996         freeurl(u);
 997         free(host);
 998
 999         while(k = shdr){
1000                 shdr = k->next;
1001                 free(k);
1002         }
1003         exits(nil);
1004 }