sys/src/cmd/vac/vac.c

   1 #include "stdinc.h"
   2 #include "vac.h"
   3 #include "dat.h"
   4 #include "fns.h"
   5
   6 // TODO: qids
   7
   8 void
   9 usage(void)
  10 {
  11         fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
  12         threadexitsall("usage");
  13 }
  14
  15 enum
  16 {
  17         BlockSize = 8*1024,
  18 };
  19
  20 struct
  21 {
  22         int nfile;
  23         int ndir;
  24         vlong data;
  25         vlong skipdata;
  26         int skipfiles;
  27 } stats;
  28
  29 int qdiff;
  30 int merge;
  31 int verbose;
  32 char *host;
  33 VtConn *z;
  34 VacFs *fs;
  35 char *archivefile;
  36 char *vacfile;
  37
  38 int vacmerge(VacFile*, char*);
  39 void vac(VacFile*, VacFile*, char*, Dir*);
  40 void vacstdin(VacFile*, char*);
  41 VacFile *recentarchive(VacFs*, char*);
  42
  43 static u64int unittoull(char*);
  44 static void warn(char *fmt, ...);
  45 static void removevacfile(void);
  46
  47 void
  48 threadmain(int argc, char **argv)
  49 {
  50         int i, j, fd, n, printstats;
  51         Dir *d;
  52         char *s;
  53         uvlong u;
  54         VacFile *f, *fdiff;
  55         VacFs *fsdiff;
  56         int blocksize;
  57         int outfd;
  58         char *stdinname;
  59         char *diffvac;
  60         uvlong qid;
  61
  62
  63         fmtinstall('F', vtfcallfmt);
  64         fmtinstall('H', encodefmt);
  65         fmtinstall('V', vtscorefmt);
  66
  67         blocksize = BlockSize;
  68         stdinname = nil;
  69         printstats = 0;
  70         fsdiff = nil;
  71         diffvac = nil;
  72
  73         ARGBEGIN{
  74         case 'V':
  75                 chattyventi++;
  76                 break;
  77         case 'a':
  78                 archivefile = EARGF(usage());
  79                 break;
  80         case 'b':
  81                 u = unittoull(EARGF(usage()));
  82                 if(u < 512)
  83                         u = 512;
  84                 if(u > VtMaxLumpSize)
  85                         u = VtMaxLumpSize;
  86                 blocksize = u;
  87                 break;
  88         case 'd':
  89                 diffvac = EARGF(usage());
  90                 break;
  91         case 'e':
  92                 excludepattern(EARGF(usage()));
  93                 break;
  94         case 'f':
  95                 vacfile = EARGF(usage());
  96                 break;
  97         case 'h':
  98                 host = EARGF(usage());
  99                 break;
 100         case 'i':
 101                 stdinname = EARGF(usage());
 102                 break;
 103         case 'm':
 104                 merge++;
 105                 break;
 106         case 'q':
 107                 qdiff++;
 108                 break;
 109         case 's':
 110                 printstats++;
 111                 break;
 112         case 'v':
 113                 verbose++;
 114                 break;
 115         case 'x':
 116                 loadexcludefile(EARGF(usage()));
 117                 break;
 118         default:
 119                 usage();
 120         }ARGEND
 121
 122         if(argc == 0 && !stdinname)
 123                 usage();
 124
 125         if(archivefile && (vacfile || diffvac)){
 126                 fprint(2, "cannot use -a with -f, -d\n");
 127                 usage();
 128         }
 129
 130         z = vtdial(host);
 131         if(z == nil)
 132                 sysfatal("could not connect to server: %r");
 133         if(vtconnect(z) < 0)
 134                 sysfatal("vtconnect: %r");
 135
 136         // Setup:
 137         //      fs is the output vac file system
 138         //      f is directory in output vac to write new files
 139         //      fdiff is corresponding directory in existing vac
 140         if(archivefile){
 141                 VacFile *fp;
 142                 char yyyy[5];
 143                 char mmdd[10];
 144                 char oldpath[40];
 145                 Tm tm;
 146
 147                 fdiff = nil;
 148                 if((outfd = open(archivefile, ORDWR)) < 0){
 149                         if(access(archivefile, 0) >= 0)
 150                                 sysfatal("open %s: %r", archivefile);
 151                         if((outfd = create(archivefile, OWRITE, 0666)) < 0)
 152                                 sysfatal("create %s: %r", archivefile);
 153                         atexit(removevacfile);  // because it is new
 154                         if((fs = vacfscreate(z, blocksize, 512)) == nil)
 155                                 sysfatal("vacfscreate: %r");
 156                 }else{
 157                         if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
 158                                 sysfatal("vacfsopen %s: %r", archivefile);
 159                         if((fdiff = recentarchive(fs, oldpath)) != nil){
 160                                 if(verbose)
 161                                         fprint(2, "diff %s\n", oldpath);
 162                         }else
 163                                 if(verbose)
 164                                         fprint(2, "no recent archive to diff against\n");
 165                 }
 166
 167                 // Create yyyy/mmdd.
 168                 tm = *localtime(time(0));
 169                 snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
 170                 fp = vacfsgetroot(fs);
 171                 if((f = vacfilewalk(fp, yyyy)) == nil
 172                 && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
 173                         sysfatal("vacfscreate %s: %r", yyyy);
 174                 vacfiledecref(fp);
 175                 fp = f;
 176
 177                 snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
 178                 n = 0;
 179                 while((f = vacfilewalk(fp, mmdd)) != nil){
 180                         vacfiledecref(f);
 181                         n++;
 182                         snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
 183                 }
 184                 f = vacfilecreate(fp, mmdd, ModeDir|0555);
 185                 if(f == nil)
 186                         sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
 187                 vacfiledecref(fp);
 188
 189                 if(verbose)
 190                         fprint(2, "archive %s/%s\n", yyyy, mmdd);
 191         }else{
 192                 if(vacfile == nil)
 193                         outfd = 1;
 194                 else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
 195                         sysfatal("create %s: %r", vacfile);
 196                 atexit(removevacfile);
 197                 if((fs = vacfscreate(z, blocksize, 512)) == nil)
 198                         sysfatal("vacfscreate: %r");
 199                 f = vacfsgetroot(fs);
 200
 201                 fdiff = nil;
 202                 if(diffvac){
 203                         if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
 204                                 warn("vacfsopen %s: %r", diffvac);
 205                         else
 206                                 fdiff = vacfsgetroot(fsdiff);
 207                 }
 208         }
 209
 210         if(stdinname)
 211                 vacstdin(f, stdinname);
 212         for(i=0; i<argc; i++){
 213                 // We can't use / and . and .. and ../.. as valid archive
 214                 // names, so expand to the list of files in the directory.
 215                 if(argv[i][0] == 0){
 216                         warn("empty string given as command-line argument");
 217                         continue;
 218                 }
 219                 cleanname(argv[i]);
 220                 if(strcmp(argv[i], "/") == 0
 221                 || strcmp(argv[i], ".") == 0
 222                 || strcmp(argv[i], "..") == 0
 223                 || (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){
 224                         if((fd = open(argv[i], OREAD)) < 0){
 225                                 warn("open %s: %r", argv[i]);
 226                                 continue;
 227                         }
 228                         while((n = dirread(fd, &d)) > 0){
 229                                 for(j=0; j<n; j++){
 230                                         s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1);
 231                                         strcpy(s, argv[i]);
 232                                         strcat(s, "/");
 233                                         strcat(s, d[j].name);
 234                                         cleanname(s);
 235                                         vac(f, fdiff, s, &d[j]);
 236                                 }
 237                                 free(d);
 238                         }
 239                         close(fd);
 240                         continue;
 241                 }
 242                 if((d = dirstat(argv[i])) == nil){
 243                         warn("stat %s: %r", argv[i]);
 244                         continue;
 245                 }
 246                 vac(f, fdiff, argv[i], d);
 247                 free(d);
 248         }
 249         if(fdiff)
 250                 vacfiledecref(fdiff);
 251
 252         /*
 253          * Record the maximum qid so that vacs can be merged
 254          * without introducing overlapping qids.  Older versions
 255          * of vac arranged that the root would have the largest
 256          * qid in the file system, but we can't do that anymore
 257          * (the root gets created first!).
 258          */
 259         if(_vacfsnextqid(fs, &qid) >= 0)
 260                 vacfilesetqidspace(f, 0, qid);
 261         vacfiledecref(f);
 262
 263         /*
 264          * Copy fsdiff's root block score into fs's slot for that,
 265          * so that vacfssync will copy it into root.prev for us.
 266          * Just nice documentation, no effect.
 267          */
 268         if(fsdiff)
 269                 memmove(fs->score, fsdiff->score, VtScoreSize);
 270         if(vacfssync(fs) < 0)
 271                 fprint(2, "vacfssync: %r\n");
 272
 273         fprint(outfd, "vac:%V\n", fs->score);
 274         atexitdont(removevacfile);
 275         vacfsclose(fs);
 276         vthangup(z);
 277
 278         if(printstats){
 279                 fprint(2,
 280                         "%d files, %d files skipped, %d directories\n"
 281                         "%lld data bytes written, %lld data bytes skipped\n",
 282                         stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata);
 283                 dup(2, 1);
 284                 packetstats();
 285         }
 286         threadexitsall(0);
 287 }
 288
 289 VacFile*
 290 recentarchive(VacFs *fs, char *path)
 291 {
 292         VacFile *fp, *f;
 293         VacDirEnum *de;
 294         VacDir vd;
 295         char buf[10];
 296         int year, mmdd, nn, n, n1;
 297         char *p;
 298
 299         fp = vacfsgetroot(fs);
 300         de = vdeopen(fp);
 301         year = 0;
 302         if(de){
 303                 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
 304                         if(strlen(vd.elem) != 4)
 305                                 continue;
 306                         if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
 307                                 continue;
 308                         if(year < n)
 309                                 year = n;
 310                 }
 311         }
 312         vdeclose(de);
 313         if(year == 0){
 314                 vacfiledecref(fp);
 315                 return nil;
 316         }
 317         snprint(buf, sizeof buf, "%04d", year);
 318         if((f = vacfilewalk(fp, buf)) == nil){
 319                 fprint(2, "warning: dirread %s but cannot walk", buf);
 320                 vacfiledecref(fp);
 321                 return nil;
 322         }
 323         fp = f;
 324
 325         de = vdeopen(fp);
 326         mmdd = 0;
 327         nn = 0;
 328         if(de){
 329                 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
 330                         if(strlen(vd.elem) < 4)
 331                                 continue;
 332                         if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
 333                                 continue;
 334                         if(*p == '.'){
 335                                 if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
 336                                         continue;
 337                         }else{
 338                                 if(*p != 0)
 339                                         continue;
 340                                 n1 = 0;
 341                         }
 342                         if(n < mmdd || (n == mmdd && n1 < nn))
 343                                 continue;
 344                         mmdd = n;
 345                         nn = n1;
 346                 }
 347         }
 348         vdeclose(de);
 349         if(mmdd == 0){
 350                 vacfiledecref(fp);
 351                 return nil;
 352         }
 353         if(nn == 0)
 354                 snprint(buf, sizeof buf, "%04d", mmdd);
 355         else
 356                 snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
 357         if((f = vacfilewalk(fp, buf)) == nil){
 358                 fprint(2, "warning: dirread %s but cannot walk", buf);
 359                 vacfiledecref(fp);
 360                 return nil;
 361         }
 362         vacfiledecref(fp);
 363
 364         sprint(path, "%04d/%s", year, buf);
 365         return f;
 366 }
 367
 368 static void
 369 removevacfile(void)
 370 {
 371         if(vacfile)
 372                 remove(vacfile);
 373 }
 374
 375 void
 376 plan9tovacdir(VacDir *vd, Dir *dir)
 377 {
 378         memset(vd, 0, sizeof *vd);
 379
 380         vd->elem = dir->name;
 381         vd->uid = dir->uid;
 382         vd->gid = dir->gid;
 383         vd->mid = dir->muid;
 384         if(vd->mid == nil)
 385                 vd->mid = "";
 386         vd->mtime = dir->mtime;
 387         vd->mcount = 0;
 388         vd->ctime = dir->mtime;         /* ctime: not available on plan 9 */
 389         vd->atime = dir->atime;
 390         vd->size = dir->length;
 391
 392         vd->mode = dir->mode & 0777;
 393         if(dir->mode & DMDIR)
 394                 vd->mode |= ModeDir;
 395         if(dir->mode & DMAPPEND)
 396                 vd->mode |= ModeAppend;
 397         if(dir->mode & DMEXCL)
 398                 vd->mode |= ModeExclusive;
 399
 400         vd->plan9 = 1;
 401         vd->p9path = dir->qid.path;
 402         vd->p9version = dir->qid.vers;
 403 }
 404
 405
 406 /*
 407  * Archive the file named name, which has stat info d,
 408  * into the vac directory fp (p = parent).
 409  *
 410  * If we're doing a vac -d against another archive, the
 411  * equivalent directory to fp in that archive is diffp.
 412  */
 413 void
 414 vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
 415 {
 416         char *elem, *s;
 417         static char buf[65536];
 418         int fd, i, n, bsize;
 419         vlong off;
 420         Dir *dk;        // kids
 421         VacDir vd, vddiff;
 422         VacFile *f, *fdiff;
 423         VtEntry e;
 424
 425         if(!includefile(name)){
 426                 warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
 427                 return;
 428         }
 429
 430         if(d->mode&DMDIR)
 431                 stats.ndir++;
 432         else
 433                 stats.nfile++;
 434
 435         if(merge && vacmerge(fp, name) >= 0)
 436                 return;
 437
 438         if(verbose)
 439                 fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : "");
 440
 441         if((fd = open(name, OREAD)) < 0){
 442                 warn("open %s: %r", name);
 443                 return;
 444         }
 445
 446         elem = strrchr(name, '/');
 447         if(elem)
 448                 elem++;
 449         else
 450                 elem = name;
 451
 452         plan9tovacdir(&vd, d);
 453         if((f = vacfilecreate(fp, elem, vd.mode)) == nil){
 454                 warn("vacfilecreate %s: %r", name);
 455                 return;
 456         }
 457         if(diffp)
 458                 fdiff = vacfilewalk(diffp, elem);
 459         else
 460                 fdiff = nil;
 461
 462         if(vacfilesetdir(f, &vd) < 0)
 463                 warn("vacfilesetdir %s: %r", name);
 464
 465         if(d->mode&DMDIR){
 466                 while((n = dirread(fd, &dk)) > 0){
 467                         for(i=0; i<n; i++){
 468                                 s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1);
 469                                 strcpy(s, name);
 470                                 strcat(s, "/");
 471                                 strcat(s, dk[i].name);
 472                                 vac(f, fdiff, s, &dk[i]);
 473                                 free(s);
 474                         }
 475                         free(dk);
 476                 }
 477         }else{
 478                 off = 0;
 479                 bsize = fs->bsize;
 480                 if(fdiff){
 481                         /*
 482                          * Copy fdiff's contents into f by moving the score.
 483                          * We'll diff and update below.
 484                          */
 485                         if(vacfilegetentries(fdiff, &e, nil) >= 0)
 486                         if(vacfilesetentries(f, &e, nil) >= 0){
 487                                 bsize = e.dsize;
 488
 489                                 /*
 490                                  * Or if -q is set, and the metadata looks the same,
 491                                  * don't even bother reading the file.
 492                                  */
 493                                 if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){
 494                                         if(vddiff.mtime == vd.mtime)
 495                                         if(vddiff.size == vd.size)
 496                                         if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){
 497                                                 stats.skipfiles++;
 498                                                 stats.nfile--;
 499                                                 vdcleanup(&vddiff);
 500                                                 goto Out;
 501                                         }
 502
 503                                         /*
 504                                          * Skip over presumably-unchanged prefix
 505                                          * of an append-only file.
 506                                          */
 507                                         if(vd.mode&ModeAppend)
 508                                         if(vddiff.size < vd.size)
 509                                         if(vddiff.plan9 && vd.plan9)
 510                                         if(vddiff.p9path == vd.p9path){
 511                                                 off = vd.size/bsize*bsize;
 512                                                 if(seek(fd, off, 0) >= 0)
 513                                                         stats.skipdata += off;
 514                                                 else{
 515                                                         seek(fd, 0, 0); // paranoia
 516                                                         off = 0;
 517                                                 }
 518                                         }
 519
 520                                         vdcleanup(&vddiff);
 521                                         // XXX different verbose chatty prints for kaminsky?
 522                                 }
 523                         }
 524                 }
 525                 if(qdiff && verbose)
 526                         fprint(2, "+%s\n", name);
 527                 while((n = readn(fd, buf, bsize)) > 0){
 528                         if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){
 529                                 off += n;
 530                                 stats.skipdata += n;
 531                                 continue;
 532                         }
 533                         if(vacfilewrite(f, buf, n, off) < 0){
 534                                 warn("venti write %s: %r", name);
 535                                 goto Out;
 536                         }
 537                         stats.data += n;
 538                         off += n;
 539                 }
 540                 /*
 541                  * Since we started with fdiff's contents,
 542                  * set the size in case fdiff was bigger.
 543                  */
 544                 if(fdiff && vacfilesetsize(f, off) < 0)
 545                         warn("vtfilesetsize %s: %r", name);
 546         }
 547
 548 Out:
 549         vacfileflush(f, 1);
 550         vacfiledecref(f);
 551         if(fdiff)
 552                 vacfiledecref(fdiff);
 553         close(fd);
 554 }
 555
 556 void
 557 vacstdin(VacFile *fp, char *name)
 558 {
 559         vlong off;
 560         VacFile *f;
 561         static char buf[8192];
 562         int n;
 563
 564         if((f = vacfilecreate(fp, name, 0666)) == nil){
 565                 warn("vacfilecreate %s: %r", name);
 566                 return;
 567         }
 568
 569         off = 0;
 570         while((n = read(0, buf, sizeof buf)) > 0){
 571                 if(vacfilewrite(f, buf, n, off) < 0){
 572                         warn("venti write %s: %r", name);
 573                         vacfiledecref(f);
 574                         return;
 575                 }
 576                 off += n;
 577         }
 578         vacfileflush(f, 1);
 579         vacfiledecref(f);
 580 }
 581
 582 /*
 583  * fp is the directory we're writing.
 584  * mp is the directory whose contents we're merging in.
 585  * d is the directory entry of the file from mp that we want to add to fp.
 586  * vacfile is the name of the .vac file, for error messages.
 587  * offset is the qid that qid==0 in mp should correspond to.
 588  * max is the maximum qid we expect to see (not really needed).
 589  */
 590 int
 591 vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile,
 592         vlong offset, vlong max)
 593 {
 594         VtEntry ed, em;
 595         VacFile *mf;
 596         VacFile *f;
 597
 598         mf = vacfilewalk(mp, d->elem);
 599         if(mf == nil){
 600                 warn("could not walk %s in %s", d->elem, vacfile);
 601                 return -1;
 602         }
 603         if(vacfilegetentries(mf, &ed, &em) < 0){
 604                 warn("could not get entries for %s in %s", d->elem, vacfile);
 605                 vacfiledecref(mf);
 606                 return -1;
 607         }
 608
 609         if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){
 610                 warn("vacfilecreate %s: %r", d->elem);
 611                 vacfiledecref(mf);
 612                 return -1;
 613         }
 614         if(d->qidspace){
 615                 d->qidoffset += offset;
 616                 d->qidmax += offset;
 617         }else{
 618                 d->qidspace = 1;
 619                 d->qidoffset = offset;
 620                 d->qidmax = max;
 621         }
 622         if(vacfilesetdir(f, d) < 0
 623         || vacfilesetentries(f, &ed, &em) < 0
 624         || vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){
 625                 warn("vacmergefile %s: %r", d->elem);
 626                 vacfiledecref(mf);
 627                 vacfiledecref(f);
 628                 return -1;
 629         }
 630
 631         vacfiledecref(mf);
 632         vacfiledecref(f);
 633         return 0;
 634 }
 635
 636 int
 637 vacmerge(VacFile *fp, char *name)
 638 {
 639         VacFs *mfs;
 640         VacDir vd;
 641         VacDirEnum *de;
 642         VacFile *mp;
 643         uvlong maxqid, offset;
 644
 645         if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
 646                 return -1;
 647         if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
 648                 return -1;
 649         if(verbose)
 650                 fprint(2, "merging %s\n", name);
 651
 652         mp = vacfsgetroot(mfs);
 653         de = vdeopen(mp);
 654         if(de){
 655                 offset = 0;
 656                 if(vacfsgetmaxqid(mfs, &maxqid) >= 0){
 657                         _vacfsnextqid(fs, &offset);
 658                         vacfsjumpqid(fs, maxqid+1);
 659                 }
 660                 while(vderead(de, &vd) > 0){
 661                         if(vd.qid > maxqid){
 662                                 warn("vacmerge %s: maxqid=%lld but %s has %lld",
 663                                         name, maxqid, vd.elem, vd.qid);
 664                                 vacfsjumpqid(fs, vd.qid - maxqid);
 665                                 maxqid = vd.qid;
 666                         }
 667                         vacmergefile(fp, mp, &vd, name,
 668                                 offset, maxqid);
 669                         vdcleanup(&vd);
 670                 }
 671                 vdeclose(de);
 672         }
 673         vacfiledecref(mp);
 674         vacfsclose(mfs);
 675         return 0;
 676 }
 677
 678 #define TWID64  ((u64int)~(u64int)0)
 679
 680 static u64int
 681 unittoull(char *s)
 682 {
 683         char *es;
 684         u64int n;
 685
 686         if(s == nil)
 687                 return TWID64;
 688         n = strtoul(s, &es, 0);
 689         if(*es == 'k' || *es == 'K'){
 690                 n *= 1024;
 691                 es++;
 692         }else if(*es == 'm' || *es == 'M'){
 693                 n *= 1024*1024;
 694                 es++;
 695         }else if(*es == 'g' || *es == 'G'){
 696                 n *= 1024*1024*1024;
 697                 es++;
 698         }
 699         if(*es != '\0')
 700                 return TWID64;
 701         return n;
 702 }
 703
 704 static void
 705 warn(char *fmt, ...)
 706 {
 707         va_list arg;
 708
 709         va_start(arg, fmt);
 710         fprint(2, "vac: ");
 711         vfprint(2, fmt, arg);
 712         fprint(2, "\n");
 713         va_end(arg);
 714 }
 715