sys/src/cmd/ip/httpfile.c

   1 /* contributed by 20h@r-36.net, September 2005 */
   2
   3 #include <u.h>
   4 #include <libc.h>
   5 #include <bio.h>
   6 #include <ndb.h>
   7 #include <thread.h>
   8 #include <fcall.h>
   9 #include <9p.h>
  10 #include <mp.h>
  11 #include <libsec.h>
  12
  13 enum
  14 {
  15         Blocksize = 64*1024,
  16         Stacksize = 8192,
  17 };
  18
  19 char *host;
  20 char *file;
  21 char *port;
  22 char *url;
  23 char *get;
  24 char *user;
  25 char *net = "net";
  26
  27 vlong size;
  28 int usetls;
  29 int debug;
  30 int ncache;
  31 int mcache;
  32
  33 void
  34 usage(void)
  35 {
  36         fprint(2, "usage: httpfile [-Dd] [-c count] [-f file] [-m mtpt] [-s srvname] [-x net] url\n");
  37         exits("usage");
  38 }
  39
  40 enum
  41 {
  42         Qroot,
  43         Qfile,
  44 };
  45
  46 #define PATH(type, n)           ((type)|((n)<<8))
  47 #define TYPE(path)                      ((int)(path) & 0xFF)
  48 #define NUM(path)                       ((uint)(path)>>8)
  49
  50 Channel *reqchan;
  51 Channel *httpchan;
  52 Channel *finishchan;
  53 ulong time0;
  54
  55 typedef struct Block Block;
  56 struct Block
  57 {
  58         uchar *p;
  59         vlong off;
  60         vlong len;
  61         Block *link;
  62         long lastuse;
  63         Req *rq;
  64         Req **erq;
  65 };
  66
  67 typedef struct Blocklist Blocklist;
  68 struct Blocklist
  69 {
  70         Block *first;
  71         Block **end;
  72 };
  73
  74 Blocklist cache;
  75 Blocklist inprogress;
  76
  77 void
  78 queuereq(Block *b, Req *r)
  79 {
  80         if(b->rq==nil)
  81                 b->erq = &b->rq;
  82         *b->erq = r;
  83         r->aux = nil;
  84         b->erq = (Req**)&r->aux;
  85 }
  86
  87 void
  88 addblock(Blocklist *l, Block *b)
  89 {
  90         if(debug)
  91                 print("adding: %p %lld\n", b, b->off);
  92
  93         if(l->first == nil)
  94                 l->end = &l->first;
  95         *l->end = b;
  96         b->link = nil;
  97         l->end = &b->link;
  98         b->lastuse = time(0);
  99 }
 100
 101 void
 102 delreq(Block *b, Req *r)
 103 {
 104         Req **l;
 105
 106         for(l = &b->rq; *l; l = (Req**)&(*l)->aux){
 107                 if(*l == r){
 108                         *l = r->aux;
 109                         if(*l == nil)
 110                                 b->erq = l;
 111                         free(r);
 112                         return;
 113                 }
 114         }
 115 }
 116
 117 void
 118 evictblock(Blocklist *cache)
 119 {
 120         Block **l, **oldest, *b;
 121
 122         if(cache->first == nil)
 123                 return;
 124
 125         oldest = nil;
 126         for(l=&cache->first; *l; l=&(*l)->link)
 127                 if(oldest == nil || (*oldest)->lastuse > (*l)->lastuse)
 128                         oldest = l;
 129
 130         b = *oldest;
 131         *oldest = (*oldest)->link;
 132         if(*oldest == nil)
 133                 cache->end = oldest;
 134         free(b->p);
 135         free(b);
 136         ncache--;
 137 }
 138
 139 Block *
 140 findblock(Blocklist *s, vlong off)
 141 {
 142         Block *b;
 143
 144         for(b = s->first; b != nil; b = b->link){
 145                 if(b->off <= off && off < b->off + Blocksize){
 146                         if(debug)
 147                                 print("found: %lld -> %lld\n", off, b->off);
 148                         b->lastuse = time(0);
 149                         return b;
 150                 }
 151         }
 152
 153         return nil;
 154 }
 155
 156 void
 157 readfrom(Req *r, Block *b)
 158 {
 159         int d, n;
 160
 161         b->lastuse = time(0);
 162
 163         n = r->ifcall.count;
 164         d = r->ifcall.offset - b->off;
 165         if(b->off + d + n > b->off + b->len)
 166                 n = b->len - d;
 167         if(debug)
 168                 print("Reading from: %p %d %d\n", b->p, d, n);
 169         memmove(r->ofcall.data, b->p + d, n);
 170         r->ofcall.count = n;
 171
 172         respond(r, nil);
 173 }
 174
 175 void
 176 hangupclient(Srv*)
 177 {
 178         if(debug)
 179                 print("Hangup.\n");
 180
 181         threadexitsall("done");
 182 }
 183
 184 int
 185 dotls(int fd)
 186 {
 187         TLSconn conn;
 188
 189         memset(&conn, 0, sizeof(conn));
 190         if((fd=tlsClient(fd, &conn)) < 0)
 191                 sysfatal("tlsclient: %r");
 192         free(conn.cert);
 193         free(conn.sessionID);
 194         return fd;
 195 }
 196
 197 char*
 198 nocr(char *s)
 199 {
 200         char *r, *w;
 201
 202         for(r=w=s; *r; r++)
 203                 if(*r != '\r')
 204                         *w++ = *r;
 205         *w = 0;
 206         return s;
 207 }
 208
 209 char*
 210 readhttphdr(Biobuf *netbio, vlong *size)
 211 {
 212         char *s, *stat;
 213
 214         stat = nil;
 215         while((s = Brdstr(netbio, '\n', 1)) != nil && s[0] != '\r'
 216                         && s[0] != '\0'){
 217                 if(stat == nil)
 218                         stat = estrdup9p(s);
 219                 if(strncmp(s, "Content-Length: ", 16) == 0 && size != nil)
 220                         *size = atoll(s + 16);
 221                 free(s);
 222         }
 223         if(stat)
 224                 nocr(stat);
 225
 226         return stat;
 227 }
 228
 229 int
 230 dialhttp(Biobuf *netbio)
 231 {
 232         int netfd;
 233
 234         netfd = dial(netmkaddr(host, net, port), 0, 0, 0);
 235         if(netfd < 0)
 236                 sysfatal("dial: %r");
 237         if(usetls)
 238                 netfd = dotls(netfd);
 239         Binit(netbio, netfd, OREAD);
 240
 241         return netfd;
 242 }
 243
 244 uchar*
 245 getrange(Block *b)
 246 {
 247         uchar *data;
 248         char *status;
 249         int netfd;
 250         static Biobuf netbio;
 251
 252         b->len = Blocksize;
 253         if(b->off + b->len > size)
 254                 b->len = size - b->off;
 255
 256         if(debug)
 257                 print("getrange: %lld %lld\n", b->off, b->len);
 258
 259         netfd = dialhttp(&netbio);
 260
 261         fprint(netfd,
 262                 "GET %s HTTP/1.1\r\n"
 263                 "Host: %s\r\n"
 264                 "Accept-Encoding:\r\n"
 265                 "Range: bytes=%lld-%lld\r\n"
 266                 "\r\n",
 267                 get, host, b->off, b->off+b->len-1);
 268         Bflush(&netbio);
 269
 270         status = readhttphdr(&netbio, nil);
 271         if(status == nil)
 272                 return nil;
 273
 274         /*
 275          * Some servers (e.g., www.google.com) return 200 OK
 276          * when you ask for the entire page in one range.
 277          */
 278         if(strstr(status, "206 Partial Content")==nil
 279         && (b->off!=0 || b->len!=size || strstr(status, "200 OK")==nil)){
 280                 free(status);
 281                 close(netfd);
 282                 werrstr("did not get requested range");
 283                 return nil;
 284         }
 285         free(status);
 286
 287         data = emalloc9p(b->len);
 288         if(Bread(&netbio, data, b->len) != b->len){
 289                 free(data);
 290                 close(netfd);
 291                 werrstr("not enough bytes read");
 292                 return nil;
 293         }
 294
 295         b->p = data;
 296
 297         close(netfd);
 298         return data;
 299 }
 300
 301 void
 302 httpfilereadproc(void*)
 303 {
 304         Block *b;
 305
 306         threadsetname("httpfilereadproc");
 307
 308         for(;;){
 309                 b = recvp(httpchan);
 310                 if(b == nil)
 311                         continue;
 312                 if(getrange(b) == nil)
 313                         sysfatal("getrange: %r");
 314                 sendp(finishchan, b);
 315         }
 316 }
 317
 318 typedef struct Tab Tab;
 319 struct Tab
 320 {
 321         char *name;
 322         ulong mode;
 323 };
 324
 325 Tab tab[] =
 326 {
 327         "/",            DMDIR|0555,
 328         nil,            0444,
 329 };
 330
 331 static void
 332 fillstat(Dir *d, uvlong path)
 333 {
 334         Tab *t;
 335
 336         memset(d, 0, sizeof(*d));
 337         d->uid = estrdup9p(user);
 338         d->gid = estrdup9p(user);
 339         d->qid.path = path;
 340         d->atime = d->mtime = time0;
 341         t = &tab[TYPE(path)];
 342         d->name = estrdup9p(t->name);
 343         d->length = size;
 344         d->qid.type = t->mode>>24;
 345         d->mode = t->mode;
 346 }
 347
 348 static void
 349 fsattach(Req *r)
 350 {
 351         if(r->ifcall.aname && r->ifcall.aname[0]){
 352                 respond(r, "invalid attach specifier");
 353                 return;
 354         }
 355         r->fid->qid.path = PATH(Qroot, 0);
 356         r->fid->qid.type = QTDIR;
 357         r->fid->qid.vers = 0;
 358         r->ofcall.qid = r->fid->qid;
 359         respond(r, nil);
 360 }
 361
 362 static void
 363 fsstat(Req *r)
 364 {
 365         fillstat(&r->d, r->fid->qid.path);
 366         respond(r, nil);
 367 }
 368
 369 static int
 370 rootgen(int i, Dir *d, void*)
 371 {
 372         i += Qroot + 1;
 373         if(i <= Qfile){
 374                 fillstat(d, i);
 375                 return 0;
 376         }
 377         return -1;
 378 }
 379
 380 static char*
 381 fswalk1(Fid *fid, char *name, Qid *qid)
 382 {
 383         int i;
 384         ulong path;
 385
 386         path = fid->qid.path;
 387         if(!(fid->qid.type & QTDIR))
 388                 return "walk in non-directory";
 389
 390         if(strcmp(name, "..") == 0){
 391                 switch(TYPE(path)){
 392                 case Qroot:
 393                         return nil;
 394                 default:
 395                         return "bug in fswalk1";
 396                 }
 397         }
 398
 399         i = TYPE(path) + 1;
 400         while(i < nelem(tab)){
 401                 if(strcmp(name, tab[i].name) == 0){
 402                         qid->path = PATH(i, NUM(path));
 403                         qid->type = tab[i].mode>>24;
 404                         return nil;
 405                 }
 406                 if(tab[i].mode & DMDIR)
 407                         break;
 408                 i++;
 409         }
 410         return "directory entry not found";
 411 }
 412
 413 vlong
 414 getfilesize(void)
 415 {
 416         char *status;
 417         vlong size;
 418         int netfd;
 419         static Biobuf netbio;
 420
 421         netfd = dialhttp(&netbio);
 422
 423         fprint(netfd,
 424                 "HEAD %s HTTP/1.1\r\n"
 425                 "Host: %s\r\n"
 426                 "Accept-Encoding:\r\n"
 427                 "\r\n",
 428                 get, host);
 429
 430         status = readhttphdr(&netbio, &size);
 431         if(strstr(status, "200 OK") == nil){
 432                 werrstr("%s", status);
 433                 size = -1;
 434         }
 435         free(status);
 436
 437         close(netfd);
 438         return size;
 439 }
 440
 441 void
 442 fileread(Req *r)
 443 {
 444         Block *b;
 445
 446         if(r->ifcall.offset > size){
 447                 respond(r, nil);
 448                 return;
 449         }
 450
 451         if((b = findblock(&cache, r->ifcall.offset)) != nil){
 452                 readfrom(r, b);
 453                 return;
 454         }
 455         if((b = findblock(&inprogress, r->ifcall.offset)) == nil){
 456                 b = emalloc9p(sizeof(Block));
 457                 b->off = r->ifcall.offset - (r->ifcall.offset % Blocksize);
 458                 addblock(&inprogress, b);
 459                 if(inprogress.first == b)
 460                         sendp(httpchan, b);
 461         }
 462         queuereq(b, r);
 463 }
 464
 465 static void
 466 fsopen(Req *r)
 467 {
 468         if(r->ifcall.mode != OREAD){
 469                 respond(r, "permission denied");
 470                 return;
 471         }
 472         respond(r, nil);
 473 }
 474
 475 void
 476 finishthread(void*)
 477 {
 478         Block *b;
 479         Req *r, *nextr;
 480
 481         threadsetname("finishthread");
 482
 483         for(;;){
 484                 b = recvp(finishchan);
 485                 assert(b == inprogress.first);
 486                 inprogress.first = b->link;
 487                 ncache++;
 488                 if(ncache >= mcache)
 489                         evictblock(&cache);
 490                 addblock(&cache, b);
 491                 for(r=b->rq; r; r=nextr){
 492                         nextr = r->aux;
 493                         readfrom(r, b);
 494                 }
 495                 b->rq = nil;
 496                 if(inprogress.first)
 497                         sendp(httpchan, inprogress.first);
 498         }
 499 }
 500
 501 void
 502 fsnetproc(void*)
 503 {
 504         Req *r;
 505         Block *b;
 506
 507         threadcreate(finishthread, nil, 8192);
 508
 509         threadsetname("fsnetproc");
 510
 511         for(;;){
 512                 r = recvp(reqchan);
 513                 switch(r->ifcall.type){
 514                 case Tflush:
 515                         b = findblock(&inprogress, r->ifcall.offset);
 516                         delreq(b, r->oldreq);
 517                         respond(r->oldreq, "interrupted");
 518                         respond(r, nil);
 519                         break;
 520                 case Tread:
 521                         fileread(r);
 522                         break;
 523                 default:
 524                         respond(r, "bug in fsthread");
 525                         break;
 526                 }
 527         }
 528 }
 529
 530 static void
 531 fsflush(Req *r)
 532 {
 533         sendp(reqchan, r);
 534 }
 535
 536 static void
 537 fsread(Req *r)
 538 {
 539         char e[ERRMAX];
 540         ulong path;
 541
 542         path = r->fid->qid.path;
 543         switch(TYPE(path)){
 544         case Qroot:
 545                 dirread9p(r, rootgen, nil);
 546                 respond(r, nil);
 547                 break;
 548         case Qfile:
 549                 sendp(reqchan, r);
 550                 break;
 551         default:
 552                 snprint(e, sizeof(e), "bug in fsread path=%lux", path);
 553                 respond(r, e);
 554                 break;
 555         }
 556 }
 557
 558 Srv fs =
 559 {
 560 .attach=                fsattach,
 561 .walk1=         fswalk1,
 562 .open=          fsopen,
 563 .read=          fsread,
 564 .stat=          fsstat,
 565 .flush=         fsflush,
 566 .end=           hangupclient,
 567 };
 568
 569 void
 570 threadmain(int argc, char **argv)
 571 {
 572         char *defport, *mtpt, *srvname, *p;
 573
 574         mtpt = nil;
 575         srvname = nil;
 576         ARGBEGIN{
 577         case 'D':
 578                 chatty9p++;
 579                 break;
 580         case 'd':
 581                 debug++;
 582                 break;
 583         case 's':
 584                 srvname = EARGF(usage());
 585                 break;
 586         case 'm':
 587                 mtpt = EARGF(usage());
 588                 break;
 589         case 'c':
 590                 mcache = atoi(EARGF(usage()));
 591                 break;
 592         case 'f':
 593                 file = EARGF(usage());
 594                 break;
 595         case 'x':
 596                 net = smprint("%s/net", EARGF(usage()));
 597                 break;
 598         default:
 599                 usage();
 600         }ARGEND;
 601
 602         if(srvname == nil && mtpt == nil)
 603                 mtpt = ".";
 604
 605         if(argc < 1)
 606                 usage();
 607         if(mcache <= 0)
 608                 mcache = 32;
 609
 610         time0 = time(0);
 611         host = url = estrdup9p(argv[0]);
 612
 613         defport = nil;
 614         if(!cistrncmp(url, "https://", 8)){
 615                 host += 8;
 616                 usetls = 1;
 617                 defport = "https";
 618         }else if(!cistrncmp(url, "http://", 7)){
 619                 host += 7;
 620                 defport = "http";
 621         }else
 622                 sysfatal("unsupported url: %s", url);
 623
 624         if((p = strchr(host, '/')) != nil){
 625                 get = estrdup9p(p);
 626                 *p = '\0';
 627         }else
 628                 get = "/";
 629
 630         port = strchr(host, ':');
 631         if(port != nil)
 632                 *port++ = '\0';
 633         else
 634                 port = defport;
 635
 636         if(file == nil){
 637                 file = strrchr(get, '/')+1;
 638                 if(*file == 0)
 639                         file = "index";
 640         }
 641
 642         tab[Qfile].name = file;
 643         user = getuser();
 644         size = getfilesize();
 645         if(size < 0)
 646                 sysfatal("getfilesize: %r");
 647
 648         reqchan = chancreate(sizeof(Req*), 0);
 649         httpchan = chancreate(sizeof(Block*), 0);
 650         finishchan = chancreate(sizeof(Block*), 0);
 651
 652         procrfork(fsnetproc, nil, Stacksize, RFNAMEG|RFNOTEG);
 653         procrfork(httpfilereadproc, nil, Stacksize, RFNAMEG|RFNOTEG);
 654
 655         threadpostmountsrv(&fs, srvname, mtpt, MBEFORE);
 656         threadexits(0);
 657 }