sys/src/cmd/ip/httpfile.c

   1 /* contributed by 20h@r-36.net, September 2005 */
   2
   3 #include <u.h>
   4 #include <libc.h>
   5 #include <bio.h>
   6 #include <ndb.h>
   7 #include <thread.h>
   8 #include <fcall.h>
   9 #include <9p.h>
  10 #include <mp.h>
  11 #include <libsec.h>
  12
  13 enum
  14 {
  15         Blocksize = 64*1024,
  16         Stacksize = 8192,
  17 };
  18
  19 char *host;
  20 char *file;
  21 char *port;
  22 char *url;
  23 char *get;
  24 char *user;
  25 char *net = "net";
  26
  27 vlong size;
  28 int usetls;
  29 int debug;
  30 int ncache;
  31 int mcache;
  32
  33 void
  34 usage(void)
  35 {
  36         fprint(2, "usage: httpfile [-Dd] [-c count] [-f file] [-m mtpt] [-s srvname] [-x net] url\n");
  37         exits("usage");
  38 }
  39
  40 enum
  41 {
  42         Qroot,
  43         Qfile,
  44 };
  45
  46 #define PATH(type, n)           ((type)|((n)<<8))
  47 #define TYPE(path)                      ((int)(path) & 0xFF)
  48 #define NUM(path)                       ((uint)(path)>>8)
  49
  50 Channel *reqchan;
  51 Channel *httpchan;
  52 Channel *finishchan;
  53 ulong time0;
  54
  55 typedef struct Block Block;
  56 struct Block
  57 {
  58         uchar *p;
  59         vlong off;
  60         vlong len;
  61         Block *link;
  62         long lastuse;
  63         Req *rq;
  64         Req **erq;
  65 };
  66
  67 typedef struct Blocklist Blocklist;
  68 struct Blocklist
  69 {
  70         Block *first;
  71         Block **end;
  72 };
  73
  74 Blocklist cache;
  75 Blocklist inprogress;
  76
  77 void
  78 queuereq(Block *b, Req *r)
  79 {
  80         if(b->rq==nil)
  81                 b->erq = &b->rq;
  82         *b->erq = r;
  83         r->aux = nil;
  84         b->erq = (Req**)&r->aux;
  85 }
  86
  87 void
  88 addblock(Blocklist *l, Block *b)
  89 {
  90         if(debug)
  91                 print("adding: %p %lld\n", b, b->off);
  92
  93         if(l->first == nil)
  94                 l->end = &l->first;
  95         *l->end = b;
  96         b->link = nil;
  97         l->end = &b->link;
  98         b->lastuse = time(0);
  99 }
 100
 101 void
 102 delreq(Block *b, Req *r)
 103 {
 104         Req **l;
 105
 106         for(l = &b->rq; *l; l = (Req**)&(*l)->aux){
 107                 if(*l == r){
 108                         *l = r->aux;
 109                         if(*l == nil)
 110                                 b->erq = l;
 111                         free(r);
 112                         return;
 113                 }
 114         }
 115 }
 116
 117 void
 118 evictblock(Blocklist *cache)
 119 {
 120         Block **l, **oldest, *b;
 121
 122         if(cache->first == nil)
 123                 return;
 124
 125         oldest = nil;
 126         for(l=&cache->first; *l; l=&(*l)->link)
 127                 if(oldest == nil || (*oldest)->lastuse > (*l)->lastuse)
 128                         oldest = l;
 129
 130         b = *oldest;
 131         *oldest = (*oldest)->link;
 132         if(*oldest == nil)
 133                 cache->end = oldest;
 134         free(b->p);
 135         free(b);
 136         ncache--;
 137 }
 138
 139 Block *
 140 findblock(Blocklist *s, vlong off)
 141 {
 142         Block *b;
 143
 144         for(b = s->first; b != nil; b = b->link){
 145                 if(b->off <= off && off < b->off + Blocksize){
 146                         if(debug)
 147                                 print("found: %lld -> %lld\n", off, b->off);
 148                         b->lastuse = time(0);
 149                         return b;
 150                 }
 151         }
 152
 153         return nil;
 154 }
 155
 156 void
 157 readfrom(Req *r, Block *b)
 158 {
 159         int d, n;
 160
 161         b->lastuse = time(0);
 162
 163         n = r->ifcall.count;
 164         d = r->ifcall.offset - b->off;
 165         if(b->off + d + n > b->off + b->len)
 166                 n = b->len - d;
 167         if(debug)
 168                 print("Reading from: %p %d %d\n", b->p, d, n);
 169         memmove(r->ofcall.data, b->p + d, n);
 170         r->ofcall.count = n;
 171
 172         respond(r, nil);
 173 }
 174
 175 void
 176 hangupclient(Srv*)
 177 {
 178         if(debug)
 179                 print("Hangup.\n");
 180
 181         threadexitsall("done");
 182 }
 183
 184 int
 185 dotls(int fd)
 186 {
 187         TLSconn conn;
 188
 189         if((fd=tlsClient(fd, &conn)) < 0)
 190                 sysfatal("tlsclient: %r");
 191
 192         if(conn.cert != nil)
 193                 free(conn.cert);
 194
 195         return fd;
 196 }
 197
 198 char*
 199 nocr(char *s)
 200 {
 201         char *r, *w;
 202
 203         for(r=w=s; *r; r++)
 204                 if(*r != '\r')
 205                         *w++ = *r;
 206         *w = 0;
 207         return s;
 208 }
 209
 210 char*
 211 readhttphdr(Biobuf *netbio, vlong *size)
 212 {
 213         char *s, *stat;
 214
 215         stat = nil;
 216         while((s = Brdstr(netbio, '\n', 1)) != nil && s[0] != '\r'
 217                         && s[0] != '\0'){
 218                 if(stat == nil)
 219                         stat = estrdup9p(s);
 220                 if(strncmp(s, "Content-Length: ", 16) == 0 && size != nil)
 221                         *size = atoll(s + 16);
 222                 free(s);
 223         }
 224         if(stat)
 225                 nocr(stat);
 226
 227         return stat;
 228 }
 229
 230 int
 231 dialhttp(Biobuf *netbio)
 232 {
 233         int netfd;
 234
 235         netfd = dial(netmkaddr(host, net, port), 0, 0, 0);
 236         if(netfd < 0)
 237                 sysfatal("dial: %r");
 238         if(usetls)
 239                 netfd = dotls(netfd);
 240         Binit(netbio, netfd, OREAD);
 241
 242         return netfd;
 243 }
 244
 245 uchar*
 246 getrange(Block *b)
 247 {
 248         uchar *data;
 249         char *status;
 250         int netfd;
 251         static Biobuf netbio;
 252
 253         b->len = Blocksize;
 254         if(b->off + b->len > size)
 255                 b->len = size - b->off;
 256
 257         if(debug)
 258                 print("getrange: %lld %lld\n", b->off, b->len);
 259
 260         netfd = dialhttp(&netbio);
 261
 262         fprint(netfd,
 263                 "GET %s HTTP/1.1\r\n"
 264                 "Host: %s\r\n"
 265                 "Accept-Encoding:\r\n"
 266                 "Range: bytes=%lld-%lld\r\n"
 267                 "\r\n",
 268                 get, host, b->off, b->off+b->len);
 269         Bflush(&netbio);
 270
 271         status = readhttphdr(&netbio, nil);
 272         if(status == nil)
 273                 return nil;
 274
 275         /*
 276          * Some servers (e.g., www.google.com) return 200 OK
 277          * when you ask for the entire page in one range.
 278          */
 279         if(strstr(status, "206 Partial Content")==nil
 280         && (b->off!=0 || b->len!=size || strstr(status, "200 OK")==nil)){
 281                 free(status);
 282                 close(netfd);
 283                 werrstr("did not get requested range");
 284                 return nil;
 285         }
 286         free(status);
 287
 288         data = emalloc9p(b->len);
 289         if(Bread(&netbio, data, b->len) != b->len){
 290                 free(data);
 291                 close(netfd);
 292                 werrstr("not enough bytes read");
 293                 return nil;
 294         }
 295
 296         b->p = data;
 297
 298         close(netfd);
 299         return data;
 300 }
 301
 302 void
 303 httpfilereadproc(void*)
 304 {
 305         Block *b;
 306
 307         threadsetname("httpfilereadproc");
 308
 309         for(;;){
 310                 b = recvp(httpchan);
 311                 if(b == nil)
 312                         continue;
 313                 if(getrange(b) == nil)
 314                         sysfatal("getrange: %r");
 315                 sendp(finishchan, b);
 316         }
 317 }
 318
 319 typedef struct Tab Tab;
 320 struct Tab
 321 {
 322         char *name;
 323         ulong mode;
 324 };
 325
 326 Tab tab[] =
 327 {
 328         "/",            DMDIR|0555,
 329         nil,            0444,
 330 };
 331
 332 static void
 333 fillstat(Dir *d, uvlong path)
 334 {
 335         Tab *t;
 336
 337         memset(d, 0, sizeof(*d));
 338         d->uid = estrdup9p(user);
 339         d->gid = estrdup9p(user);
 340         d->qid.path = path;
 341         d->atime = d->mtime = time0;
 342         t = &tab[TYPE(path)];
 343         d->name = estrdup9p(t->name);
 344         d->length = size;
 345         d->qid.type = t->mode>>24;
 346         d->mode = t->mode;
 347 }
 348
 349 static void
 350 fsattach(Req *r)
 351 {
 352         if(r->ifcall.aname && r->ifcall.aname[0]){
 353                 respond(r, "invalid attach specifier");
 354                 return;
 355         }
 356         r->fid->qid.path = PATH(Qroot, 0);
 357         r->fid->qid.type = QTDIR;
 358         r->fid->qid.vers = 0;
 359         r->ofcall.qid = r->fid->qid;
 360         respond(r, nil);
 361 }
 362
 363 static void
 364 fsstat(Req *r)
 365 {
 366         fillstat(&r->d, r->fid->qid.path);
 367         respond(r, nil);
 368 }
 369
 370 static int
 371 rootgen(int i, Dir *d, void*)
 372 {
 373         i += Qroot + 1;
 374         if(i <= Qfile){
 375                 fillstat(d, i);
 376                 return 0;
 377         }
 378         return -1;
 379 }
 380
 381 static char*
 382 fswalk1(Fid *fid, char *name, Qid *qid)
 383 {
 384         int i;
 385         ulong path;
 386
 387         path = fid->qid.path;
 388         if(!(fid->qid.type & QTDIR))
 389                 return "walk in non-directory";
 390
 391         if(strcmp(name, "..") == 0){
 392                 switch(TYPE(path)){
 393                 case Qroot:
 394                         return nil;
 395                 default:
 396                         return "bug in fswalk1";
 397                 }
 398         }
 399
 400         i = TYPE(path) + 1;
 401         while(i < nelem(tab)){
 402                 if(strcmp(name, tab[i].name) == 0){
 403                         qid->path = PATH(i, NUM(path));
 404                         qid->type = tab[i].mode>>24;
 405                         return nil;
 406                 }
 407                 if(tab[i].mode & DMDIR)
 408                         break;
 409                 i++;
 410         }
 411         return "directory entry not found";
 412 }
 413
 414 vlong
 415 getfilesize(void)
 416 {
 417         char *status;
 418         vlong size;
 419         int netfd;
 420         static Biobuf netbio;
 421
 422         netfd = dialhttp(&netbio);
 423
 424         fprint(netfd,
 425                 "HEAD %s HTTP/1.1\r\n"
 426                 "Host: %s\r\n"
 427                 "Accept-Encoding:\r\n"
 428                 "\r\n",
 429                 get, host);
 430
 431         status = readhttphdr(&netbio, &size);
 432         if(strstr(status, "200 OK") == nil){
 433                 werrstr("%s", status);
 434                 size = -1;
 435         }
 436         free(status);
 437
 438         close(netfd);
 439         return size;
 440 }
 441
 442 void
 443 fileread(Req *r)
 444 {
 445         Block *b;
 446
 447         if(r->ifcall.offset > size){
 448                 respond(r, nil);
 449                 return;
 450         }
 451
 452         if((b = findblock(&cache, r->ifcall.offset)) != nil){
 453                 readfrom(r, b);
 454                 return;
 455         }
 456         if((b = findblock(&inprogress, r->ifcall.offset)) == nil){
 457                 b = emalloc9p(sizeof(Block));
 458                 b->off = r->ifcall.offset - (r->ifcall.offset % Blocksize);
 459                 addblock(&inprogress, b);
 460                 if(inprogress.first == b)
 461                         sendp(httpchan, b);
 462         }
 463         queuereq(b, r);
 464 }
 465
 466 static void
 467 fsopen(Req *r)
 468 {
 469         if(r->ifcall.mode != OREAD){
 470                 respond(r, "permission denied");
 471                 return;
 472         }
 473         respond(r, nil);
 474 }
 475
 476 void
 477 finishthread(void*)
 478 {
 479         Block *b;
 480         Req *r, *nextr;
 481
 482         threadsetname("finishthread");
 483
 484         for(;;){
 485                 b = recvp(finishchan);
 486                 assert(b == inprogress.first);
 487                 inprogress.first = b->link;
 488                 ncache++;
 489                 if(ncache >= mcache)
 490                         evictblock(&cache);
 491                 addblock(&cache, b);
 492                 for(r=b->rq; r; r=nextr){
 493                         nextr = r->aux;
 494                         readfrom(r, b);
 495                 }
 496                 b->rq = nil;
 497                 if(inprogress.first)
 498                         sendp(httpchan, inprogress.first);
 499         }
 500 }
 501
 502 void
 503 fsnetproc(void*)
 504 {
 505         Req *r;
 506         Block *b;
 507
 508         threadcreate(finishthread, nil, 8192);
 509
 510         threadsetname("fsnetproc");
 511
 512         for(;;){
 513                 r = recvp(reqchan);
 514                 switch(r->ifcall.type){
 515                 case Tflush:
 516                         b = findblock(&inprogress, r->ifcall.offset);
 517                         delreq(b, r->oldreq);
 518                         respond(r->oldreq, "interrupted");
 519                         respond(r, nil);
 520                         break;
 521                 case Tread:
 522                         fileread(r);
 523                         break;
 524                 default:
 525                         respond(r, "bug in fsthread");
 526                         break;
 527                 }
 528         }
 529 }
 530
 531 static void
 532 fsflush(Req *r)
 533 {
 534         sendp(reqchan, r);
 535 }
 536
 537 static void
 538 fsread(Req *r)
 539 {
 540         char e[ERRMAX];
 541         ulong path;
 542
 543         path = r->fid->qid.path;
 544         switch(TYPE(path)){
 545         case Qroot:
 546                 dirread9p(r, rootgen, nil);
 547                 respond(r, nil);
 548                 break;
 549         case Qfile:
 550                 sendp(reqchan, r);
 551                 break;
 552         default:
 553                 snprint(e, sizeof(e), "bug in fsread path=%lux", path);
 554                 respond(r, e);
 555                 break;
 556         }
 557 }
 558
 559 Srv fs =
 560 {
 561 .attach=                fsattach,
 562 .walk1=         fswalk1,
 563 .open=          fsopen,
 564 .read=          fsread,
 565 .stat=          fsstat,
 566 .flush=         fsflush,
 567 .end=           hangupclient,
 568 };
 569
 570 void
 571 threadmain(int argc, char **argv)
 572 {
 573         char *defport, *mtpt, *srvname, *p;
 574
 575         mtpt = nil;
 576         srvname = nil;
 577         ARGBEGIN{
 578         case 'D':
 579                 chatty9p++;
 580                 break;
 581         case 'd':
 582                 debug++;
 583                 break;
 584         case 's':
 585                 srvname = EARGF(usage());
 586                 break;
 587         case 'm':
 588                 mtpt = EARGF(usage());
 589                 break;
 590         case 'c':
 591                 mcache = atoi(EARGF(usage()));
 592                 break;
 593         case 'f':
 594                 file = EARGF(usage());
 595                 break;
 596         case 'x':
 597                 net = smprint("%s/net", EARGF(usage()));
 598                 break;
 599         default:
 600                 usage();
 601         }ARGEND;
 602
 603         if(srvname == nil && mtpt == nil)
 604                 mtpt = ".";
 605
 606         if(argc < 1)
 607                 usage();
 608         if(mcache <= 0)
 609                 mcache = 32;
 610
 611         time0 = time(0);
 612         host = url = estrdup9p(argv[0]);
 613
 614         defport = nil;
 615         if(!cistrncmp(url, "https://", 8)){
 616                 host += 8;
 617                 usetls = 1;
 618                 defport = "https";
 619         }else if(!cistrncmp(url, "http://", 7)){
 620                 host += 7;
 621                 defport = "http";
 622         }else
 623                 sysfatal("unsupported url: %s", url);
 624
 625         if((p = strchr(host, '/')) != nil){
 626                 get = estrdup9p(p);
 627                 *p = '\0';
 628         }else
 629                 get = "/";
 630
 631         port = strchr(host, ':');
 632         if(port != nil)
 633                 *port++ = '\0';
 634         else
 635                 port = defport;
 636
 637         if(file == nil){
 638                 file = strrchr(get, '/')+1;
 639                 if(*file == 0)
 640                         file = "index";
 641         }
 642
 643         tab[Qfile].name = file;
 644         user = getuser();
 645         size = getfilesize();
 646         if(size < 0)
 647                 sysfatal("getfilesize: %r");
 648
 649         reqchan = chancreate(sizeof(Req*), 0);
 650         httpchan = chancreate(sizeof(Block*), 0);
 651         finishchan = chancreate(sizeof(Block*), 0);
 652
 653         procrfork(fsnetproc, nil, Stacksize, RFNAMEG|RFNOTEG);
 654         procrfork(httpfilereadproc, nil, Stacksize, RFNAMEG|RFNOTEG);
 655
 656         threadpostmountsrv(&fs, srvname, mtpt, MBEFORE);
 657         threadexits(0);
 658 }