sys/src/cmd/webfs/http.c

   1 #include <u.h>
   2 #include <libc.h>
   3 #include <ctype.h>
   4 #include <fcall.h>
   5 #include <thread.h>
   6 #include <9p.h>
   7
   8 #include "dat.h"
   9 #include "fns.h"
  10
  11 #include <auth.h>
  12 #include <mp.h>
  13 #include <libsec.h>
  14
  15 typedef struct Hconn Hconn;
  16 typedef struct Hpool Hpool;
  17 typedef struct Hauth Hauth;
  18
  19 struct Hconn
  20 {
  21         Hconn   *next;
  22         long    time;
  23
  24         int     fd;
  25         int     ctl;
  26         int     keep;
  27         int     cancel;
  28         int     tunnel;
  29         int     len;
  30         char    addr[128];
  31         char    buf[8192+2];
  32 };
  33
  34 struct Hpool
  35 {
  36         QLock;
  37
  38         Hconn   *head;
  39         int     active;
  40
  41         int     limit;
  42         int     peer;
  43         int     idle;
  44 };
  45
  46 struct Hauth
  47 {
  48         Hauth   *next;
  49         Url     *url;
  50         char    *auth;
  51 };
  52
  53 static Hpool hpool = {
  54         .limit  = 16,
  55         .peer   = 4,
  56         .idle   = 5,    /* seconds */
  57 };
  58
  59 static QLock authlk;
  60 static Hauth *hauth;
  61
  62 static void hclose(Hconn *h);
  63
  64 static int
  65 tlstrace(char *fmt, ...)
  66 {
  67         int r;
  68         va_list a;
  69         va_start(a, fmt);
  70         r = vfprint(2, fmt, a);
  71         va_end(a);
  72         return r;
  73 }
  74
  75 static int
  76 tlswrap(int fd, char *servername)
  77 {
  78         TLSconn conn;
  79
  80         memset(&conn, 0, sizeof(conn));
  81         if(debug)
  82                 conn.trace = tlstrace;
  83         if(servername != nil)
  84                 conn.serverName = smprint("%N", servername);
  85         if((fd = tlsClient(fd, &conn)) < 0){
  86                 if(debug) fprint(2, "tlsClient: %r\n");
  87         }
  88         free(conn.cert);
  89         free(conn.sessionID);
  90         free(conn.serverName);
  91         return fd;
  92 }
  93
  94 static Hconn*
  95 hdial(Url *u, int cached)
  96 {
  97         char addr[128];
  98         Hconn *h, *p;
  99         int fd, ctl;
 100
 101         snprint(addr, sizeof(addr), "tcp!%s!%s", u->host, u->port ? u->port : u->scheme);
 102
 103         qlock(&hpool);
 104         if(cached){
 105                 for(p = nil, h = hpool.head; h; p = h, h = h->next){
 106                         if(strcmp(h->addr, addr) == 0){
 107                                 if(p)
 108                                         p->next = h->next;
 109                                 else
 110                                         hpool.head = h->next;
 111                                 h->next = nil;
 112                                 qunlock(&hpool);
 113                                 return h;
 114                         }
 115                 }
 116         }
 117         hpool.active++;
 118         qunlock(&hpool);
 119
 120         if(debug)
 121                 fprint(2, "hdial [%d] %s\n", hpool.active, addr);
 122
 123         if(proxy)
 124                 snprint(addr, sizeof(addr), "tcp!%s!%s",
 125                         proxy->host, proxy->port ? proxy->port : proxy->scheme);
 126
 127         if((fd = dial(addr, 0, 0, &ctl)) < 0)
 128                 return nil;
 129
 130         if(proxy){
 131                 if(strcmp(proxy->scheme, "https") == 0)
 132                         fd = tlswrap(fd, proxy->host);
 133         } else {
 134                 if(strcmp(u->scheme, "https") == 0)
 135                         fd = tlswrap(fd, u->host);
 136         }
 137         if(fd < 0){
 138                 close(ctl);
 139                 return nil;
 140         }
 141
 142         h = emalloc(sizeof(*h));
 143         h->next = nil;
 144         h->time = 0;
 145         h->cancel = 0;
 146         h->tunnel = 0;
 147         h->keep = cached;
 148         h->len = 0;
 149         h->fd = fd;
 150         h->ctl = ctl;
 151
 152         if(proxy){
 153                 h->tunnel = strcmp(u->scheme, "https") == 0;
 154                 snprint(addr, sizeof(addr), "tcp!%s!%s", u->host, u->port ? u->port : u->scheme);
 155         }
 156         nstrcpy(h->addr, addr, sizeof(h->addr));
 157
 158         return h;
 159 }
 160
 161 static void
 162 hcloseall(Hconn *x)
 163 {
 164         Hconn *h;
 165
 166         while(h = x){
 167                 x = h->next;
 168                 h->next = nil;
 169                 h->keep = 0;
 170                 hclose(h);
 171         }
 172 }
 173
 174 static void
 175 hclose(Hconn *h)
 176 {
 177         Hconn *x, *t;
 178         int i, n;
 179
 180         if(h == nil)
 181                 return;
 182
 183         qlock(&hpool);
 184         if(!h->tunnel && h->keep && h->fd >= 0){
 185                 for(n = 0, i = 0, t = nil, x = hpool.head; x; x = x->next){
 186                         if(strcmp(x->addr, h->addr) == 0)
 187                                 if(++n > hpool.peer)
 188                                         break;
 189                         if(++i < hpool.limit)
 190                                 t = x;
 191                 }
 192                 if(x == nil){
 193                         /* return connection to pool */
 194                         h->time = time(0);
 195                         h->next = hpool.head;
 196                         hpool.head = h;
 197
 198                         /* cut off tail */
 199                         if(t){
 200                                 x = t->next;
 201                                 t->next = nil;
 202                         }
 203
 204                         i = h->next != nil;
 205                         qunlock(&hpool);
 206
 207                         /* free the tail */
 208                         hcloseall(x);
 209
 210                         /*
 211                          * if h is first one in pool, spawn proc to close
 212                          * idle connections.
 213                          */
 214                         if(i == 0)
 215                         if(rfork(RFMEM|RFPROC|RFNOWAIT) == 0){
 216                                 do {
 217                                         Hconn **xx;
 218                                         long now;
 219
 220                                         sleep(1000);
 221
 222                                         qlock(&hpool);
 223                                         now = time(0);
 224
 225                                         x = nil;
 226                                         xx = &hpool.head;
 227                                         while(h = *xx){
 228                                                 if((now - h->time) > hpool.idle){
 229                                                         *xx = h->next;
 230
 231                                                         /* link to tail */
 232                                                         h->next = x;
 233                                                         x = h;
 234                                                         continue;
 235                                                 }
 236                                                 xx = &h->next;
 237                                         }
 238
 239                                         i = hpool.head != nil;
 240                                         qunlock(&hpool);
 241
 242                                         /* free the tail */
 243                                         hcloseall(x);
 244                                 } while(i);
 245                                 exits(nil);
 246                         }
 247                         return;
 248                 }
 249         }
 250         hpool.active--;
 251         qunlock(&hpool);
 252
 253         if(debug)
 254                 fprint(2, "hclose [%d] %s\n", hpool.active, h->addr);
 255
 256         if(h->ctl >= 0)
 257                 close(h->ctl);
 258         if(h->fd >= 0)
 259                 close(h->fd);
 260         free(h);
 261 }
 262
 263 static void
 264 hhangup(Hconn *h)
 265 {
 266         if(debug)
 267                 fprint(2, "hangup pc=%p: %r\n", getcallerpc(&h));
 268         h->keep = 0;
 269         if(h->ctl >= 0)
 270                 hangup(h->ctl);
 271 }
 272
 273 static int
 274 hread(Hconn *h, void *data, int len)
 275 {
 276         if(h->len > 0){
 277                 if(len > h->len)
 278                         len = h->len;
 279                 memmove(data, h->buf, len);
 280                 h->len -= len;
 281                 if(h->len > 0)
 282                         memmove(h->buf, h->buf + len, h->len);
 283                 return len;
 284         }
 285         if((len = read(h->fd, data, len)) == 0)
 286                 h->keep = 0;
 287         if(len < 0)
 288                 hhangup(h);
 289         return len;
 290 }
 291
 292 static int
 293 hwrite(Hconn *h, void *data, int len)
 294 {
 295         if(write(h->fd, data, len) != len){
 296                 hhangup(h);
 297                 return -1;
 298         }
 299         return len;
 300 }
 301
 302 static int
 303 hline(Hconn *h, char *data, int len, int cont)
 304 {
 305         char *x, *y, *e;
 306         int n;
 307
 308         data[0] = 0;
 309         for(;;){
 310                 if(h->len > 0){
 311                         while(x = memchr(h->buf, '\n', h->len)){
 312                                 n = x - h->buf;
 313                                 if(n > 0 && x[-1] == '\r')
 314                                         n--;
 315                                 if(n > 0 && cont){
 316                                         e = h->buf + h->len;
 317                                         for(y = x+1; y < e; y++)
 318                                                 if(*y != ' ' && *y != '\t')
 319                                                         break;
 320                                         if(y >= e || *y == 0)
 321                                                 break;
 322                                         if(y > x+1){
 323                                                 if(x > h->buf && x[-1] == '\r')
 324                                                         x--;
 325                                                 memmove(x, y, e - y);
 326                                                 h->len -= y - x;
 327                                                 continue;
 328                                         }
 329                                 }
 330                                 if(n < len)
 331                                         len = n;
 332                                 memmove(data, h->buf, len);
 333                                 data[len] = 0;
 334                                 h->len -= (++x - h->buf);
 335                                 if(h->len > 0)
 336                                         memmove(h->buf, x, h->len);
 337                                 return len;
 338                         }
 339                 }
 340                 n = sizeof(h->buf) - h->len;
 341                 if(n <= 0)
 342                         return 0;
 343                 if(h->tunnel)
 344                         n = 1;  /* do not read beyond header */
 345                 if((n = read(h->fd, h->buf + h->len, n)) <= 0){
 346                         hhangup(h);
 347                         return -1;
 348                 }
 349                 h->len += n;
 350         }
 351 }
 352
 353 static int
 354 hauthgetkey(char *params)
 355 {
 356         if(debug)
 357                 fprint(2, "hauthgetkey %s\n", params);
 358         werrstr("needkey %s", params);
 359         return -1;
 360 }
 361
 362 int
 363 authenticate(Url *u, Url *ru, char *method, char *s)
 364 {
 365         char oerr[ERRMAX], *user, *pass, *realm, *nonce, *opaque, *x;
 366         Hauth *a;
 367         Fmt fmt;
 368         int n;
 369
 370         snprint(oerr, sizeof(oerr), "authentification failed");
 371         errstr(oerr, sizeof(oerr));
 372
 373         user = u->user;
 374         pass = u->pass;
 375         realm = nonce = opaque = nil;
 376         if(!cistrncmp(s, "Basic ", 6)){
 377                 UserPasswd *up;
 378
 379                 s += 6;
 380                 if(x = cistrstr(s, "realm="))
 381                         realm = unquote(x+6, &s);
 382                 if(realm == nil)
 383                         return -1;
 384                 up = nil;
 385                 if(user == nil || pass == nil){
 386                         fmtstrinit(&fmt);
 387                         fmtprint(&fmt, " realm=%q", realm);
 388                         if(user)
 389                                 fmtprint(&fmt, " user=%q", user);
 390                         if((s = fmtstrflush(&fmt)) == nil)
 391                                 return -1;
 392                         up = auth_getuserpasswd(hauthgetkey,
 393                                 "proto=pass service=http server=%q%s", u->host, s);
 394                         free(s);
 395                         if(up == nil)
 396                                 return -1;
 397                         user = up->user;
 398                         pass = up->passwd;
 399                 }
 400                 fmtstrinit(&fmt);
 401                 fmtprint(&fmt, "%s:%s", user ? user : "", pass ? pass : "");
 402                 if(up){
 403                         memset(up->user, 0, strlen(up->user));
 404                         memset(up->passwd, 0, strlen(up->passwd));
 405                         free(up);
 406                 }
 407                 if((s = fmtstrflush(&fmt)) == nil)
 408                         return -1;
 409                 n = strlen(s);
 410                 fmtstrinit(&fmt);
 411                 fmtprint(&fmt, "Basic %.*[", n, s);
 412                 memset(s, 0, n);
 413                 free(s);
 414                 u = saneurl(url(".", u));       /* all uris below the requested one */
 415         }else
 416         if(!cistrncmp(s, "Digest ", 7)){
 417                 char chal[1024], ouser[128], resp[2*MD5LEN+1];
 418                 int nchal;
 419
 420                 s += 7;
 421                 if(x = cistrstr(s, "realm="))
 422                         realm = unquote(x+6, &s);
 423                 if(x = cistrstr(s, "nonce="))
 424                         nonce = unquote(x+6, &s);
 425                 if(x = cistrstr(s, "opaque="))
 426                         opaque = unquote(x+7, &s);
 427                 if(realm == nil || nonce == nil)
 428                         return -1;
 429                 fmtstrinit(&fmt);
 430                 fmtprint(&fmt, " realm=%q", realm);
 431                 if(user)
 432                         fmtprint(&fmt, " user=%q", user);
 433                 if((s = fmtstrflush(&fmt)) == nil)
 434                         return -1;
 435                 nchal = snprint(chal, sizeof(chal), "%s %s %U", nonce, method, ru);
 436                 n = auth_respond(chal, nchal, ouser, sizeof ouser, resp, sizeof resp, hauthgetkey,
 437                         "proto=httpdigest role=client server=%q%s", u->host, s);
 438                 memset(chal, 0, sizeof(chal));
 439                 free(s);
 440                 if(n < 0)
 441                         return -1;
 442                 fmtstrinit(&fmt);
 443                 fmtprint(&fmt, "Digest ");
 444                 fmtprint(&fmt, "username=\"%s\", ", ouser);
 445                 fmtprint(&fmt, "realm=\"%s\", ", realm);
 446                 fmtprint(&fmt, "host=\"%N\", ", u->host);
 447                 fmtprint(&fmt, "uri=\"%U\", ", ru);
 448                 fmtprint(&fmt, "nonce=\"%s\", ", nonce);
 449                 fmtprint(&fmt, "response=\"%s\"", resp);
 450                 if(opaque)
 451                         fmtprint(&fmt, ", opaque=\"%s\"", opaque);
 452                 u = saneurl(url("/", u));       /* BUG: should be the ones in domain= only */
 453         } else
 454                 return -1;
 455         if((s = fmtstrflush(&fmt)) == nil){
 456                 freeurl(u);
 457                 return -1;
 458         }
 459         if(u == nil){
 460                 free(s);
 461                 return -1;
 462         }
 463
 464         a = emalloc(sizeof(*a));
 465         a->url = u;
 466         a->auth = s;
 467         qlock(&authlk);
 468         a->next = hauth;
 469         hauth = a;
 470         qunlock(&authlk);
 471
 472         errstr(oerr, sizeof(oerr));
 473         return 0;
 474 }
 475
 476 int
 477 hauthenticate(Url *u, Url *ru, char *method, char *key, Key *hdr)
 478 {
 479         for(hdr = getkey(hdr, key); hdr != nil; hdr = getkey(hdr->next, key))
 480                 if(authenticate(u, ru, method, hdr->val) == 0)
 481                         return 0;
 482         return -1;
 483 }
 484
 485 void
 486 flushauth(Url *u, char *t)
 487 {
 488         Hauth *a, *p;
 489
 490         qlock(&authlk);
 491 Again:
 492         for(p = nil, a = hauth; a; p = a, a = a->next)
 493                 if(matchurl(u, a->url) && (t == nil || !strcmp(t, a->auth))){
 494                         if(p)
 495                                 p->next = a->next;
 496                         else
 497                                 hauth = a->next;
 498                         if(debug)
 499                                 fprint(2, "flushauth for %U\n", a->url);
 500                         freeurl(a->url);
 501                         memset(a->auth, 0, strlen(a->auth));
 502                         free(a->auth);
 503                         free(a);
 504                         goto Again;
 505                 }
 506         qunlock(&authlk);
 507 }
 508
 509 static void
 510 catch(void *, char *msg)
 511 {
 512         if(strstr("alarm", msg) != nil)
 513                 noted(NCONT);
 514         else
 515                 noted(NDFLT);
 516 }
 517
 518 #define NOLENGTH 0x7fffffffffffffffLL
 519
 520 void
 521 http(char *m, Url *u, Key *shdr, Buq *qbody, Buq *qpost)
 522 {
 523         int i, l, n, try, pid, fd, cfd, needlength, chunked, retry, nobody, badauth;
 524         char *s, *x, buf[8192+2], status[256], method[16], *host;
 525         vlong length, offset;
 526         Url ru, tu, *nu;
 527         Key *k, *rhdr;
 528         Hconn *h;
 529         Hauth *a;
 530
 531         incref(qbody);
 532         if(qpost) incref(qpost);
 533         nstrcpy(method, m, sizeof(method));
 534         switch(rfork(RFPROC|RFMEM|RFNOWAIT)){
 535         default:
 536                 return;
 537         case -1:
 538                 buclose(qbody, "can't fork");
 539                 bufree(qbody);
 540                 buclose(qpost, "can't fork");
 541                 bufree(qpost);
 542                 while(k = shdr){
 543                         shdr = k->next;
 544                         free(k);
 545                 }
 546                 freeurl(u);
 547                 return;
 548         case 0:
 549                 break;
 550         }
 551
 552         notify(catch);
 553         if(qpost){
 554                 /* file for spooling the postbody if we need to restart the request */
 555                 snprint(buf, sizeof(buf), "/tmp/http.%d.%d.post", getppid(), getpid());
 556                 fd = create(buf, OEXCL|ORDWR|ORCLOSE, 0600);
 557         } else
 558                 fd = -1;
 559
 560         h = nil;
 561         cfd = -1;
 562         pid = 0;
 563         host = nil;
 564         needlength = 0;
 565         badauth = 0;
 566         for(try = 0; try < 12; try++){
 567                 strcpy(status, "0 No status");
 568                 if(u == nil || (strcmp(u->scheme, "http") && strcmp(u->scheme, "https"))){
 569                         werrstr("bad url scheme");
 570                         break;
 571                 }
 572
 573                 if(debug)
 574                         fprint(2, "http(%d): %s %U\n", try, method, u);
 575
 576                 /* preemptive authentication from hauth cache */
 577                 qlock(&authlk);
 578                 if(proxy && !lookkey(shdr, "Proxy-Authorization"))
 579                         for(a = hauth; a; a = a->next)
 580                                 if(matchurl(a->url, proxy)){
 581                                         shdr = addkey(shdr, "Proxy-Authorization", a->auth);
 582                                         break;
 583                                 }
 584                 if(!lookkey(shdr, "Authorization"))
 585                         for(a = hauth; a; a = a->next)
 586                                 if(matchurl(a->url, u)){
 587                                         shdr = addkey(shdr, "Authorization", a->auth);
 588                                         break;
 589                                 }
 590                 qunlock(&authlk);
 591
 592                 length = 0;
 593                 chunked = 0;
 594                 if(qpost){
 595                         /* have to read it to temp file to figure out the length */
 596                         if(fd >= 0 && needlength && lookkey(shdr, "Content-Length") == nil){
 597                                 seek(fd, 0, 2);
 598                                 while((n = buread(qpost, buf, sizeof(buf))) > 0)
 599                                         write(fd, buf, n);
 600                                 shdr = delkey(shdr, "Transfer-Encoding");
 601                         }
 602
 603                         qlock(qpost);
 604                         /* wait until buffer is full, most posts are small */
 605                         while(!qpost->closed && qpost->size < qpost->limit && qpost->nwq == 0)
 606                                 rsleep(&qpost->rz);
 607
 608                         if(lookkey(shdr, "Content-Length"))
 609                                 chunked = 0;
 610                         else if(x = lookkey(shdr, "Transfer-Encoding"))
 611                                 chunked = cistrstr(x, "chunked") != nil;
 612                         else if(chunked = !qpost->closed)
 613                                 shdr = addkey(shdr, "Transfer-Encoding", "chunked");
 614                         else if(qpost->closed){
 615                                 if(fd >= 0){
 616                                         length = seek(fd, 0, 2);
 617                                         if(length < 0)
 618                                                 length = 0;
 619                                 }
 620                                 length += qpost->size;
 621                                 snprint(buf, sizeof(buf), "%lld", length);
 622                                 shdr = addkey(shdr, "Content-Length", buf);
 623                         }
 624                         qunlock(qpost);
 625                 }
 626
 627                 /* http requires ascii encoding of host */
 628                 free(host);
 629                 host = smprint("%N", u->host);
 630
 631                 if(proxy && strcmp(u->scheme, "https") != 0){
 632                         ru = *u;
 633                         ru.host = host;
 634                         ru.fragment = nil;
 635                 } else {
 636                         memset(&ru, 0, sizeof(ru));
 637                         ru.path = Upath(u);
 638                         ru.query = u->query;
 639                 }
 640                 n = snprint(buf, sizeof(buf), "%s %U HTTP/1.1\r\nHost: %]%s%s\r\n",
 641                         method, &ru, host, u->port ? ":" : "", u->port ? u->port : "");
 642                 if(n >= sizeof(buf)-64){
 643                         werrstr("request too large");
 644                         break;
 645                 }
 646                 if(h == nil){
 647                         alarm(timeout);
 648                         if((h = hdial(u, qpost==nil)) == nil)
 649                                 break;
 650                 }
 651                 if(h->tunnel){
 652                         n = snprint(buf, sizeof(buf), "CONNECT %]:%s HTTP/1.1\r\nHost: %]:%s\r\n",
 653                                 host, u->port ? u->port : "443",
 654                                 host, u->port ? u->port : "443");
 655                 }
 656                 else if((cfd = open("/mnt/webcookies/http", ORDWR)) >= 0){
 657                         /* only scheme, host and path are relevant for cookies */
 658                         memset(&tu, 0, sizeof(tu));
 659                         tu.scheme = u->scheme;
 660                         tu.host = host;
 661                         tu.path = Upath(u);
 662                         fprint(cfd, "%U", &tu);
 663                         for(;;){
 664                                 if(n >= sizeof(buf)-2){
 665                                         if(debug)
 666                                                 fprint(2, "-> %.*s", utfnlen(buf, n), buf);
 667                                         if(hwrite(h, buf, n) != n)
 668                                                 goto Badflush;
 669                                         n = 0;
 670                                 }
 671                                 if((l = read(cfd, buf+n, sizeof(buf)-2 - n)) == 0)
 672                                         break;
 673                                 if(l < 0){
 674                                         close(cfd);
 675                                         cfd = -1;
 676                                         break;
 677                                 }
 678                                 n += l;
 679                         }
 680                 }
 681
 682                 for(k = shdr; k; k = k->next){
 683                         /* only send proxy headers when establishing tunnel */
 684                         if(h->tunnel && cistrncmp(k->key, "Proxy-", 6) != 0)
 685                                 continue;
 686                         if(n > 0){
 687                                 if(debug)
 688                                         fprint(2, "-> %.*s", utfnlen(buf, n), buf);
 689                                 if(hwrite(h, buf, n) != n)
 690                                         goto Badflush;
 691                         }
 692                         n = snprint(buf, sizeof(buf)-2, "%s: %s\r\n", k->key, k->val);
 693                 }
 694                 n += snprint(buf+n, sizeof(buf)-n, "\r\n");
 695                 if(debug)
 696                         fprint(2, "-> %.*s", utfnlen(buf, n), buf);
 697                 if(hwrite(h, buf, n) != n){
 698                 Badflush:
 699                         alarm(0);
 700                         goto Retry;
 701                 }
 702
 703                 if(qpost && !h->tunnel){
 704                         h->cancel = 0;
 705                         if((pid = rfork(RFMEM|RFPROC)) <= 0){
 706                                 int ifd;
 707
 708                                 if((ifd = fd) >= 0)
 709                                         seek(ifd, 0, 0);
 710                                 while(!h->cancel){
 711                                         alarm(0);
 712                                         if((ifd < 0) || ((n = read(ifd, buf, sizeof(buf)-2)) <= 0)){
 713                                                 ifd = -1;
 714                                                 if((n = buread(qpost, buf, sizeof(buf)-2)) <= 0)
 715                                                         break;
 716                                                 if(fd >= 0)
 717                                                         if(write(fd, buf, n) != n)
 718                                                                 break;
 719                                         }
 720                                         alarm(timeout);
 721                                         if(chunked){
 722                                                 char tmp[32];
 723                                                 if(hwrite(h, tmp, snprint(tmp, sizeof(tmp), "%x\r\n", n)) < 0)
 724                                                         break;
 725                                                 buf[n++] = '\r';
 726                                                 buf[n++] = '\n';
 727                                         }
 728                                         if(hwrite(h, buf, n) != n)
 729                                                 break;
 730                                 }
 731                                 if(chunked){
 732                                         alarm(timeout);
 733                                         hwrite(h, "0\r\n\r\n", 5);
 734                                 }else
 735                                         h->keep = 0;
 736                                 if(pid == 0)
 737                                         exits(nil);
 738                         }
 739                         /* no timeout when posting */
 740                         alarm(0);
 741                 } else
 742                         alarm(timeout);
 743
 744                 Cont:
 745                 rhdr = 0;
 746                 retry = 0;
 747                 chunked = 0;
 748                 offset = 0;
 749                 length = NOLENGTH;
 750                 for(l = 0; hline(h, s = buf, sizeof(buf)-1, 1) > 0; l++){
 751                         if(debug)
 752                                 fprint(2, "<- %s\n", s);
 753                         if(l == 0){
 754                                 if(x = strchr(s, ' '))
 755                                         while(*x == ' ')
 756                                                 *x++ = 0;
 757                                 if(cistrncmp(s, "HTTP", 4)){
 758                                         h->keep = 0;
 759                                         if(cistrcmp(s, "ICY"))
 760                                                 break;
 761                                 }
 762                                 if(x[0])
 763                                         nstrcpy(status, x, sizeof(status));
 764                                 continue;
 765                         }
 766                         if((k = parsehdr(s)) == nil)
 767                                 continue;
 768                         if(!cistrcmp(k->key, "Connection")){
 769                                 if(cistrstr(k->val, "close"))
 770                                         h->keep = 0;
 771                         }
 772                         else if(!cistrcmp(k->key, "Content-Length"))
 773                                 length = atoll(k->val);
 774                         else if(!cistrcmp(k->key, "Transfer-Encoding")){
 775                                 if(cistrstr(k->val, "chunked"))
 776                                         chunked = 1;
 777                         }
 778                         else if(!cistrcmp(k->key, "Set-Cookie") ||
 779                                 !cistrcmp(k->key, "Set-Cookie2")){
 780                                 if(cfd >= 0)
 781                                         fprint(cfd, "Set-Cookie: %s\n", k->val);
 782                                 free(k);
 783                                 continue;
 784                         }
 785                         k->next = rhdr;
 786                         rhdr = k;
 787                 }
 788                 alarm(0);
 789                 if(cfd >= 0){
 790                         close(cfd);
 791                         cfd = -1;
 792                 }
 793
 794                 nobody = !cistrcmp(method, "HEAD");
 795                 if((i = atoi(status)) < 0)
 796                         i = 0;
 797                 Status:
 798                 switch(i){
 799                 default:
 800                         if(i % 100){
 801                                 i -= (i % 100);
 802                                 goto Status;
 803                         }
 804                         goto Error;
 805                 case 100:       /* Continue */
 806                 case 101:       /* Switching Protocols */
 807                 case 102:       /* Processing */
 808                 case 103:       /* Early Hints */
 809                         while(k = rhdr){
 810                                 rhdr = k->next;
 811                                 free(k);
 812                         }
 813                         strcpy(status, "0 No status");
 814                         goto Cont;
 815                 case 304:       /* Not Modified */
 816                         nobody = 1;
 817                 case 305:       /* Use Proxy */
 818                 case 400:       /* Bad Request */
 819                 case 402:       /* Payment Required */
 820                 case 403:       /* Forbidden */
 821                 case 404:       /* Not Found */
 822                 case 405:       /* Method Not Allowed */
 823                 case 406:       /* Not Acceptable */
 824                 case 408:       /* Request Timeout */
 825                 case 409:       /* Conflict */
 826                 case 410:       /* Gone */
 827                         goto Error;
 828                 case 411:       /* Length Required */
 829                         if(qpost){
 830                                 needlength = 1;
 831                                 h->cancel = 1;
 832                                 retry = 1;
 833                                 break;
 834                         }
 835                 case 412:       /* Precondition Failed */
 836                 case 413:       /* Request Entity Too Large */
 837                 case 414:       /* Request URI Too Large */
 838                 case 415:       /* Unsupported Media Type */
 839                 case 416:       /* Requested Range Not Satisfiable */
 840                 case 417:       /* Expectation Failed */
 841                 case 500:       /* Internal server error */
 842                 case 501:       /* Not implemented */
 843                 case 502:       /* Bad gateway */
 844                 case 503:       /* Service unavailable */
 845                 case 504:       /* Gateway Timeout */
 846                 case 505:       /* HTTP Version not Supported */
 847                 Error:
 848                         h->cancel = 1;
 849                         buclose(qbody, status);
 850                         buclose(qpost, status);
 851                         break;
 852                 case 300:       /* Multiple choices */
 853                 case 302:       /* Found */
 854                 case 303:       /* See Other */
 855                         if(qpost){
 856                                 if(pid > 0){
 857                                         waitpid();
 858                                         pid = 0;
 859                                 }
 860                                 buclose(qpost, 0);
 861                                 bufree(qpost);
 862                                 qpost = nil;
 863                         }
 864                         shdr = delkey(shdr, "Content-Length");
 865                         shdr = delkey(shdr, "Content-Type");
 866                         shdr = delkey(shdr, "Transfer-Encoding");
 867                         if(cistrcmp(method, "HEAD"))
 868                                 nstrcpy(method, "GET", sizeof(method));
 869                 case 301:       /* Moved Permanently */
 870                 case 307:       /* Temporary Redirect */
 871                 case 308:       /* Resume Incomplete */
 872                         if((x = lookkey(rhdr, "Location")) == nil)
 873                                 goto Error;
 874                         if((nu = saneurl(url(x, u))) == nil)
 875                                 goto Error;
 876                         freeurl(u);
 877                         u = nu;
 878                 if(0){
 879                 case 401:       /* Unauthorized */
 880                         if(x = lookkey(shdr, "Authorization")){
 881                                 flushauth(nil, x);
 882                                 if(badauth++)
 883                                         goto Error;
 884                         }
 885                         if(hauthenticate(u, &ru, method, "WWW-Authenticate", rhdr) < 0){
 886                         Autherror:
 887                                 h->cancel = 1;
 888                                 rerrstr(buf, sizeof(buf));
 889                                 buclose(qbody, buf);
 890                                 buclose(qpost, buf);
 891                                 break;
 892                         }
 893                 }
 894                 if(0){
 895                 case 407:       /* Proxy Auth */
 896                         if(proxy == nil)
 897                                 goto Error;
 898                         if(x = lookkey(shdr, "Proxy-Authorization")){
 899                                 flushauth(proxy, x);
 900                                 if(badauth++)
 901                                         goto Error;
 902                         }
 903                         if(hauthenticate(proxy, proxy, method, "Proxy-Authenticate", rhdr) < 0)
 904                                 goto Autherror;
 905                 }
 906                 case 0:         /* No status */
 907                         if(qpost && fd < 0){
 908                                 if(i > 0)
 909                                         goto Error;
 910                                 break;
 911                         }
 912                         h->cancel = 1;
 913                         retry = 1;
 914                         break;
 915                 case 204:       /* No Content */
 916                 case 205:       /* Reset Content */
 917                         nobody = 1;
 918                 case 200:       /* OK */
 919                 case 201:       /* Created */
 920                 case 202:       /* Accepted */
 921                 case 203:       /* Non-Authoritative Information */
 922                 case 206:       /* Partial Content */
 923                         if(h->tunnel)
 924                                 break;
 925                         qbody->url = u; u = nil;
 926                         qbody->hdr = rhdr; rhdr = nil;
 927                         if(nobody)
 928                                 buclose(qbody, 0);
 929                         break;
 930                 }
 931
 932                 while(k = rhdr){
 933                         rhdr = k->next;
 934                         free(k);
 935                 }
 936
 937                 /*
 938                  * remove authorization headers so on the next round, we use
 939                  * the hauth cache (wich checks the scope url). this makes
 940                  * sure we wont send credentials to the wrong url after
 941                  * a redirect.
 942                  */
 943                 shdr = delkey(shdr, "Proxy-Authorization");
 944                 shdr = delkey(shdr, "Authorization");
 945
 946                 /*
 947                  * when 2xx response is given for the CONNECT request
 948                  * then the proxy server has established the connection.
 949                  */
 950                 if(h->tunnel && !retry && (i/100) == 2){
 951                         if((h->fd = tlswrap(h->fd, host)) < 0)
 952                                 break;
 953
 954                         /* proceed to the original request */
 955                         h->tunnel = 0;
 956                         continue;
 957                 }
 958
 959                 if(!chunked && length == NOLENGTH)
 960                         h->keep = 0;
 961
 962                 /*
 963                  * read the response body (if any). retry means we'r just
 964                  * skipping the error page so we wont touch qbody.
 965                  */
 966                 while(!nobody){
 967                         if((qbody->closed || retry) && !h->keep)
 968                                 break;
 969                         if(chunked){
 970                                 if(hline(h, buf, sizeof(buf)-1, 0) <= 0)
 971                                         break;
 972                                 length = strtoll(buf, nil, 16);
 973                                 offset = 0;
 974                         }
 975                         while(offset < length){
 976                                 l = sizeof(buf);
 977                                 if(l > (length - offset))
 978                                         l = (length - offset);
 979                                 if((n = hread(h, buf, l)) <= 0)
 980                                         break;
 981                                 offset += n;
 982                                 if(!retry)
 983                                         if(buwrite(qbody, buf, n) != n)
 984                                                 break;
 985                         }
 986                         if(offset != length){
 987                                 h->keep = 0;
 988                                 if(length != NOLENGTH)
 989                                         break;
 990                         }
 991                         if(chunked){
 992                                 while(hline(h, buf, sizeof(buf)-1, 1) > 0){
 993                                         if(debug)
 994                                                 fprint(2, "<= %s\n", buf);
 995                                         if(!retry)
 996                                                 if(k = parsehdr(buf)){
 997                                                         k->next = qbody->hdr;
 998                                                         qbody->hdr = k;
 999                                                 }
1000                                 }
1001                                 if(length > 0)
1002                                         continue;
1003                         }
1004                         if(!retry)
1005                                 buclose(qbody, 0);
1006                         break;
1007                 }
1008
1009                 if(!retry)
1010                         break;
1011                 Retry:
1012                 if(cfd >= 0)
1013                         close(cfd);
1014                 if(pid > 0){
1015                         waitpid();
1016                         pid = 0;
1017                 }
1018                 hclose(h);
1019                 h = nil;
1020         }
1021         alarm(0);
1022         snprint(buf, sizeof(buf), "%s %r", status);
1023         buclose(qbody, buf);
1024         bufree(qbody);
1025
1026         if(qpost){
1027                 if(pid > 0)
1028                         waitpid();
1029                 buclose(qpost, buf);
1030                 bufree(qpost);
1031         }
1032         if(fd >= 0)
1033                 close(fd);
1034
1035         hclose(h);
1036         freeurl(u);
1037         free(host);
1038
1039         while(k = shdr){
1040                 shdr = k->next;
1041                 free(k);
1042         }
1043         exits(nil);
1044 }