sys/src/9/port/page.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "../port/error.h"
   7
   8 #define pghash(daddr)   palloc.hash[(daddr>>PGSHIFT)&(PGHSIZE-1)]
   9
  10 struct  Palloc palloc;
  11
  12 void
  13 pageinit(void)
  14 {
  15         int color, i, j;
  16         Page *p;
  17         Pallocmem *pm;
  18         ulong m, np, k, vkb, pkb;
  19
  20         np = 0;
  21         for(i=0; i<nelem(palloc.mem); i++){
  22                 pm = &palloc.mem[i];
  23                 np += pm->npage;
  24         }
  25         palloc.pages = xalloc(np*sizeof(Page));
  26         if(palloc.pages == 0)
  27                 panic("pageinit");
  28
  29         color = 0;
  30         palloc.head = palloc.pages;
  31         p = palloc.head;
  32         for(i=0; i<nelem(palloc.mem); i++){
  33                 pm = &palloc.mem[i];
  34                 for(j=0; j<pm->npage; j++){
  35                         p->prev = p-1;
  36                         p->next = p+1;
  37                         p->pa = pm->base+j*BY2PG;
  38                         p->color = color;
  39                         palloc.freecount++;
  40                         color = (color+1)%NCOLOR;
  41                         p++;
  42                 }
  43         }
  44         palloc.tail = p - 1;
  45         palloc.head->prev = 0;
  46         palloc.tail->next = 0;
  47
  48         palloc.user = p - palloc.pages;
  49         pkb = palloc.user*BY2PG/1024;
  50         vkb = pkb + (conf.nswap*BY2PG)/1024;
  51
  52         /* Paging numbers */
  53         swapalloc.highwater = (palloc.user*5)/100;
  54         swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4);
  55
  56         m = 0;
  57         for(i=0; i<nelem(conf.mem); i++)
  58                 if(conf.mem[i].npage)
  59                         m += conf.mem[i].npage*BY2PG;
  60         k = PGROUND(end - (char*)KTZERO);
  61         print("%ldM memory: ", (m+k+1024*1024-1)/(1024*1024));
  62         print("%ldM kernel data, ", (m+k-pkb*1024+1024*1024-1)/(1024*1024));
  63         print("%ldM user, ", pkb/1024);
  64         print("%ldM swap\n", vkb/1024);
  65 }
  66
  67 static void
  68 pageunchain(Page *p)
  69 {
  70         if(canlock(&palloc))
  71                 panic("pageunchain (palloc %p)", &palloc);
  72         if(p->prev)
  73                 p->prev->next = p->next;
  74         else
  75                 palloc.head = p->next;
  76         if(p->next)
  77                 p->next->prev = p->prev;
  78         else
  79                 palloc.tail = p->prev;
  80         p->prev = p->next = nil;
  81         palloc.freecount--;
  82 }
  83
  84 void
  85 pagechaintail(Page *p)
  86 {
  87         if(canlock(&palloc))
  88                 panic("pagechaintail");
  89         if(palloc.tail) {
  90                 p->prev = palloc.tail;
  91                 palloc.tail->next = p;
  92         }
  93         else {
  94                 palloc.head = p;
  95                 p->prev = 0;
  96         }
  97         palloc.tail = p;
  98         p->next = 0;
  99         palloc.freecount++;
 100 }
 101
 102 void
 103 pagechainhead(Page *p)
 104 {
 105         if(canlock(&palloc))
 106                 panic("pagechainhead");
 107         if(palloc.head) {
 108                 p->next = palloc.head;
 109                 palloc.head->prev = p;
 110         }
 111         else {
 112                 palloc.tail = p;
 113                 p->next = 0;
 114         }
 115         palloc.head = p;
 116         p->prev = 0;
 117         palloc.freecount++;
 118 }
 119
 120 Page*
 121 newpage(int clear, Segment **s, ulong va)
 122 {
 123         Page *p;
 124         KMap *k;
 125         uchar ct;
 126         int i, hw, color;
 127
 128         lock(&palloc);
 129         color = getpgcolor(va);
 130         hw = swapalloc.highwater;
 131         for(;;) {
 132                 if(palloc.freecount > hw)
 133                         break;
 134                 if(up->kp && palloc.freecount > 0)
 135                         break;
 136
 137                 unlock(&palloc);
 138                 if(s)
 139                         qunlock(&((*s)->lk));
 140
 141                 if(!waserror()){
 142                         eqlock(&palloc.pwait);  /* Hold memory requesters here */
 143
 144                         if(!waserror()){
 145                                 kickpager();
 146                                 tsleep(&palloc.r, ispages, 0, 1000);
 147                                 poperror();
 148                         }
 149
 150                         qunlock(&palloc.pwait);
 151
 152                         poperror();
 153                 }
 154
 155                 /*
 156                  * If called from fault and we lost the segment from
 157                  * underneath don't waste time allocating and freeing
 158                  * a page. Fault will call newpage again when it has
 159                  * reacquired the segment locks
 160                  */
 161                 if(s){
 162                         *s = 0;
 163                         return 0;
 164                 }
 165
 166                 lock(&palloc);
 167         }
 168
 169         /* First try for our colour */
 170         for(p = palloc.head; p; p = p->next)
 171                 if(p->color == color)
 172                         break;
 173
 174         ct = PG_NOFLUSH;
 175         if(p == 0) {
 176                 p = palloc.head;
 177                 p->color = color;
 178                 ct = PG_NEWCOL;
 179         }
 180
 181         pageunchain(p);
 182
 183         lock(p);
 184         if(p->ref != 0)
 185                 panic("newpage: p->ref %d != 0", p->ref);
 186
 187         uncachepage(p);
 188         p->ref++;
 189         p->va = va;
 190         p->modref = 0;
 191         for(i = 0; i < MAXMACH; i++)
 192                 p->cachectl[i] = ct;
 193         unlock(p);
 194         unlock(&palloc);
 195
 196         if(clear) {
 197                 k = kmap(p);
 198                 memset((void*)VA(k), 0, BY2PG);
 199                 kunmap(k);
 200         }
 201
 202         return p;
 203 }
 204
 205 int
 206 ispages(void*)
 207 {
 208         return palloc.freecount >= swapalloc.highwater;
 209 }
 210
 211 void
 212 putpage(Page *p)
 213 {
 214         if(onswap(p)) {
 215                 putswap(p);
 216                 return;
 217         }
 218
 219         lock(&palloc);
 220         lock(p);
 221
 222         if(p->ref == 0)
 223                 panic("putpage");
 224
 225         if(--p->ref > 0) {
 226                 unlock(p);
 227                 unlock(&palloc);
 228                 return;
 229         }
 230
 231         if(p->image && p->image != &swapimage)
 232                 pagechaintail(p);
 233         else
 234                 pagechainhead(p);
 235
 236         if(palloc.r.p != 0)
 237                 wakeup(&palloc.r);
 238
 239         unlock(p);
 240         unlock(&palloc);
 241 }
 242
 243 Page*
 244 auxpage(void)
 245 {
 246         Page *p;
 247
 248         lock(&palloc);
 249         p = palloc.head;
 250         if(palloc.freecount < swapalloc.highwater) {
 251                 unlock(&palloc);
 252                 return 0;
 253         }
 254         pageunchain(p);
 255
 256         lock(p);
 257         if(p->ref != 0)
 258                 panic("auxpage");
 259         p->ref++;
 260         uncachepage(p);
 261         unlock(p);
 262         unlock(&palloc);
 263
 264         return p;
 265 }
 266
 267 void
 268 duppage(Page *p)                                /* Always call with p locked */
 269 {
 270         Page *np;
 271         int color;
 272
 273         /*
 274          *  normal lock ordering is to call
 275          *  lock(&palloc) before lock(p).
 276          *  To avoid deadlock, we have to drop
 277          *  our locks and try again. as the page
 278          *  is from the image cache, this might
 279          *  let someone else come in and grab it
 280          *  so we check page ref below.
 281          */
 282         if(!canlock(&palloc)){
 283                 unlock(p);
 284                 lock(&palloc);
 285                 lock(p);
 286         }
 287
 288         /* don't dup pages that are shared or have no image */
 289         if(p->ref != 1 || p->image == nil || p->image->notext){
 290                 unlock(&palloc);
 291                 return;
 292         }
 293
 294         /* No freelist cache when memory is very low */
 295         if(palloc.freecount < swapalloc.highwater) {
 296                 unlock(&palloc);
 297                 uncachepage(p);
 298                 return;
 299         }
 300
 301         color = getpgcolor(p->va);
 302         for(np = palloc.head; np; np = np->next)
 303                 if(np->color == color)
 304                         break;
 305
 306         /* No page of the correct color */
 307         if(np == 0) {
 308                 unlock(&palloc);
 309                 uncachepage(p);
 310                 return;
 311         }
 312
 313         pageunchain(np);
 314         pagechaintail(np);
 315 /*
 316 * XXX - here's a bug? - np is on the freelist but it's not really free.
 317 * when we unlock palloc someone else can come in, decide to
 318 * use np, and then try to lock it.  they succeed after we've
 319 * run copypage and cachepage and unlock(np).  then what?
 320 * they call pageunchain before locking(np), so it's removed
 321 * from the freelist, but still in the cache because of
 322 * cachepage below.  if someone else looks in the cache
 323 * before they remove it, the page will have a nonzero ref
 324 * once they finally lock(np). This does not happen because
 325 * newpage, auxpage, duppage and lookpage all lock(&palloc)
 326 * so while they hold it nobody is going to grab anything
 327 * from the cache.
 328 */
 329         lock(np);
 330         if(np->ref != 0)        /* should never happen */
 331                 panic("duppage: np->ref %d != 0", np->ref);
 332         unlock(&palloc);
 333
 334         /* Cache the new version */
 335         uncachepage(np);
 336         np->va = p->va;
 337         np->daddr = p->daddr;
 338         copypage(p, np);
 339         cachepage(np, p->image);
 340         unlock(np);
 341         uncachepage(p);
 342 }
 343
 344 void
 345 copypage(Page *f, Page *t)
 346 {
 347         KMap *ks, *kd;
 348
 349         ks = kmap(f);
 350         kd = kmap(t);
 351         memmove((void*)VA(kd), (void*)VA(ks), BY2PG);
 352         kunmap(ks);
 353         kunmap(kd);
 354 }
 355
 356 void
 357 uncachepage(Page *p)                    /* Always called with a locked page */
 358 {
 359         Page **l, *f;
 360
 361         if(p->image == 0)
 362                 return;
 363
 364         lock(&palloc.hashlock);
 365         l = &pghash(p->daddr);
 366         for(f = *l; f; f = f->hash) {
 367                 if(f == p) {
 368                         *l = p->hash;
 369                         break;
 370                 }
 371                 l = &f->hash;
 372         }
 373         unlock(&palloc.hashlock);
 374         putimage(p->image);
 375         p->image = 0;
 376         p->daddr = 0;
 377 }
 378
 379 void
 380 cachepage(Page *p, Image *i)
 381 {
 382         Page **l;
 383
 384         /* If this ever happens it should be fixed by calling
 385          * uncachepage instead of panic. I think there is a race
 386          * with pio in which this can happen. Calling uncachepage is
 387          * correct - I just wanted to see if we got here.
 388          */
 389         if(p->image)
 390                 panic("cachepage");
 391
 392         incref(i);
 393         lock(&palloc.hashlock);
 394         p->image = i;
 395         l = &pghash(p->daddr);
 396         p->hash = *l;
 397         *l = p;
 398         unlock(&palloc.hashlock);
 399 }
 400
 401 void
 402 cachedel(Image *i, ulong daddr)
 403 {
 404         Page *f;
 405
 406 retry:
 407         lock(&palloc.hashlock);
 408         for(f = pghash(daddr); f; f = f->hash) {
 409                 if(f->image == i && f->daddr == daddr) {
 410                         unlock(&palloc.hashlock);
 411
 412                         lock(f);
 413                         if(f->image != i || f->daddr != daddr) {
 414                                 unlock(f);
 415                                 goto retry;
 416                         }
 417                         uncachepage(f);
 418                         unlock(f);
 419
 420                         return;
 421                 }
 422         }
 423         unlock(&palloc.hashlock);
 424 }
 425
 426 Page *
 427 lookpage(Image *i, ulong daddr)
 428 {
 429         Page *f;
 430
 431 retry:
 432         lock(&palloc.hashlock);
 433         for(f = pghash(daddr); f; f = f->hash) {
 434                 if(f->image == i && f->daddr == daddr) {
 435                         unlock(&palloc.hashlock);
 436
 437                         lock(&palloc);
 438                         lock(f);
 439                         if(f->image != i || f->daddr != daddr) {
 440                                 unlock(f);
 441                                 unlock(&palloc);
 442                                 goto retry;
 443                         }
 444                         if(++f->ref == 1)
 445                                 pageunchain(f);
 446                         unlock(&palloc);
 447                         unlock(f);
 448
 449                         return f;
 450                 }
 451         }
 452         unlock(&palloc.hashlock);
 453
 454         return 0;
 455 }
 456
 457 Pte*
 458 ptecpy(Pte *old)
 459 {
 460         Pte *new;
 461         Page **src, **dst;
 462
 463         new = ptealloc();
 464         dst = &new->pages[old->first-old->pages];
 465         new->first = dst;
 466         for(src = old->first; src <= old->last; src++, dst++)
 467                 if(*src) {
 468                         if(onswap(*src))
 469                                 dupswap(*src);
 470                         else {
 471                                 lock(*src);
 472                                 (*src)->ref++;
 473                                 unlock(*src);
 474                         }
 475                         new->last = dst;
 476                         *dst = *src;
 477                 }
 478
 479         return new;
 480 }
 481
 482 Pte*
 483 ptealloc(void)
 484 {
 485         Pte *new;
 486
 487         new = smalloc(sizeof(Pte));
 488         new->first = &new->pages[PTEPERTAB];
 489         new->last = new->pages;
 490         return new;
 491 }
 492
 493 void
 494 freepte(Segment *s, Pte *p)
 495 {
 496         int ref;
 497         void (*fn)(Page*);
 498         Page *pt, **pg, **ptop;
 499
 500         switch(s->type&SG_TYPE) {
 501         case SG_PHYSICAL:
 502                 fn = s->pseg->pgfree;
 503                 ptop = &p->pages[PTEPERTAB];
 504                 if(fn) {
 505                         for(pg = p->pages; pg < ptop; pg++) {
 506                                 if(*pg == 0)
 507                                         continue;
 508                                 (*fn)(*pg);
 509                                 *pg = 0;
 510                         }
 511                         break;
 512                 }
 513                 for(pg = p->pages; pg < ptop; pg++) {
 514                         pt = *pg;
 515                         if(pt == 0)
 516                                 continue;
 517                         lock(pt);
 518                         ref = --pt->ref;
 519                         unlock(pt);
 520                         if(ref == 0)
 521                                 free(pt);
 522                 }
 523                 break;
 524         default:
 525                 for(pg = p->first; pg <= p->last; pg++)
 526                         if(*pg) {
 527                                 putpage(*pg);
 528                                 *pg = 0;
 529                         }
 530         }
 531         free(p);
 532 }
 533
 534 ulong
 535 pagenumber(Page *p)
 536 {
 537         return p-palloc.pages;
 538 }
 539
 540 void
 541 checkpagerefs(void)
 542 {
 543         int s;
 544         ulong i, np, nwrong;
 545         ulong *ref;
 546
 547         np = palloc.user;
 548         ref = malloc(np*sizeof ref[0]);
 549         if(ref == nil){
 550                 print("checkpagerefs: out of memory\n");
 551                 return;
 552         }
 553
 554         /*
 555          * This may not be exact if there are other processes
 556          * holding refs to pages on their stacks.  The hope is
 557          * that if you run it on a quiescent system it will still
 558          * be useful.
 559          */
 560         s = splhi();
 561         lock(&palloc);
 562         countpagerefs(ref, 0);
 563         portcountpagerefs(ref, 0);
 564         nwrong = 0;
 565         for(i=0; i<np; i++){
 566                 if(palloc.pages[i].ref != ref[i]){
 567                         iprint("page %#.8lux ref %d actual %lud\n",
 568                                 palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
 569                         ref[i] = 1;
 570                         nwrong++;
 571                 }else
 572                         ref[i] = 0;
 573         }
 574         countpagerefs(ref, 1);
 575         portcountpagerefs(ref, 1);
 576         iprint("%lud mistakes found\n", nwrong);
 577         unlock(&palloc);
 578         splx(s);
 579 }
 580
 581 void
 582 portcountpagerefs(ulong *ref, int print)
 583 {
 584         ulong i, j, k, ns, n;
 585         Page **pg, *entry;
 586         Proc *p;
 587         Pte *pte;
 588         Segment *s;
 589
 590         /*
 591          * Pages in segments.  s->mark avoids double-counting.
 592          */
 593         n = 0;
 594         ns = 0;
 595         for(i=0; i<conf.nproc; i++){
 596                 p = proctab(i);
 597                 for(j=0; j<NSEG; j++){
 598                         s = p->seg[j];
 599                         if(s)
 600                                 s->mark = 0;
 601                 }
 602         }
 603         for(i=0; i<conf.nproc; i++){
 604                 p = proctab(i);
 605                 for(j=0; j<NSEG; j++){
 606                         s = p->seg[j];
 607                         if(s == nil || s->mark++)
 608                                 continue;
 609                         ns++;
 610                         for(k=0; k<s->mapsize; k++){
 611                                 pte = s->map[k];
 612                                 if(pte == nil)
 613                                         continue;
 614                                 for(pg = pte->first; pg <= pte->last; pg++){
 615                                         entry = *pg;
 616                                         if(pagedout(entry))
 617                                                 continue;
 618                                         if(print){
 619                                                 if(ref[pagenumber(entry)])
 620                                                         iprint("page %#.8lux in segment %#p\n", entry->pa, s);
 621                                                 continue;
 622                                         }
 623                                         if(ref[pagenumber(entry)]++ == 0)
 624                                                 n++;
 625                                 }
 626                         }
 627                 }
 628         }
 629         if(!print){
 630                 iprint("%lud pages in %lud segments\n", n, ns);
 631                 for(i=0; i<conf.nproc; i++){
 632                         p = proctab(i);
 633                         for(j=0; j<NSEG; j++){
 634                                 s = p->seg[j];
 635                                 if(s == nil)
 636                                         continue;
 637                                 if(s->ref != s->mark){
 638                                         iprint("segment %#p (used by proc %lud pid %lud) has bad ref count %lud actual %lud\n",
 639                                                 s, i, p->pid, s->ref, s->mark);
 640                                 }
 641                         }
 642                 }
 643         }
 644 }
 645