sys/src/9/port/page.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "../port/error.h"
   7
   8 #define pghash(daddr)   palloc.hash[(daddr>>PGSHIFT)&(PGHSIZE-1)]
   9
  10 struct  Palloc palloc;
  11
  12 void
  13 pageinit(void)
  14 {
  15         int color, i, j;
  16         Page *p;
  17         Pallocmem *pm;
  18         ulong m, np, k, vkb, pkb;
  19
  20         np = 0;
  21         for(i=0; i<nelem(palloc.mem); i++){
  22                 pm = &palloc.mem[i];
  23                 np += pm->npage;
  24         }
  25         palloc.pages = xalloc(np*sizeof(Page));
  26         if(palloc.pages == 0)
  27                 panic("pageinit");
  28
  29         color = 0;
  30         palloc.head = palloc.pages;
  31         p = palloc.head;
  32         for(i=0; i<nelem(palloc.mem); i++){
  33                 pm = &palloc.mem[i];
  34                 for(j=0; j<pm->npage; j++){
  35                         p->prev = p-1;
  36                         p->next = p+1;
  37                         p->pa = pm->base+j*BY2PG;
  38                         p->color = color;
  39                         palloc.freecount++;
  40                         color = (color+1)%NCOLOR;
  41                         p++;
  42                 }
  43         }
  44         palloc.tail = p - 1;
  45         palloc.head->prev = 0;
  46         palloc.tail->next = 0;
  47
  48         palloc.user = p - palloc.pages;
  49         pkb = palloc.user*BY2PG/1024;
  50         vkb = pkb + (conf.nswap*BY2PG)/1024;
  51
  52         /* Paging numbers */
  53         swapalloc.highwater = (palloc.user*5)/100;
  54         swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4);
  55
  56         m = 0;
  57         for(i=0; i<nelem(conf.mem); i++)
  58                 if(conf.mem[i].npage)
  59                         m += conf.mem[i].npage*BY2PG;
  60         k = PGROUND(end - (char*)KTZERO);
  61         print("%ldM memory: ", (m+k+1024*1024-1)/(1024*1024));
  62         print("%ldM kernel data, ", (m+k-pkb*1024+1024*1024-1)/(1024*1024));
  63         print("%ldM user, ", pkb/1024);
  64         print("%ldM swap\n", vkb/1024);
  65 }
  66
  67 static void
  68 pageunchain(Page *p)
  69 {
  70         if(canlock(&palloc))
  71                 panic("pageunchain (palloc %p)", &palloc);
  72         if(p->prev)
  73                 p->prev->next = p->next;
  74         else
  75                 palloc.head = p->next;
  76         if(p->next)
  77                 p->next->prev = p->prev;
  78         else
  79                 palloc.tail = p->prev;
  80         p->prev = p->next = nil;
  81         palloc.freecount--;
  82 }
  83
  84 void
  85 pagechaintail(Page *p)
  86 {
  87         if(canlock(&palloc))
  88                 panic("pagechaintail");
  89         if(palloc.tail) {
  90                 p->prev = palloc.tail;
  91                 palloc.tail->next = p;
  92         }
  93         else {
  94                 palloc.head = p;
  95                 p->prev = 0;
  96         }
  97         palloc.tail = p;
  98         p->next = 0;
  99         palloc.freecount++;
 100 }
 101
 102 void
 103 pagechainhead(Page *p)
 104 {
 105         if(canlock(&palloc))
 106                 panic("pagechainhead");
 107         if(palloc.head) {
 108                 p->next = palloc.head;
 109                 palloc.head->prev = p;
 110         }
 111         else {
 112                 palloc.tail = p;
 113                 p->next = 0;
 114         }
 115         palloc.head = p;
 116         p->prev = 0;
 117         palloc.freecount++;
 118 }
 119
 120 Page*
 121 newpage(int clear, Segment **s, ulong va)
 122 {
 123         Page *p;
 124         KMap *k;
 125         uchar ct;
 126         int i, hw, color;
 127
 128         lock(&palloc);
 129         color = getpgcolor(va);
 130         hw = swapalloc.highwater;
 131         for(;;) {
 132                 if(palloc.freecount > hw)
 133                         break;
 134                 if(up->kp && palloc.freecount > 0)
 135                         break;
 136
 137                 unlock(&palloc);
 138                 if(s)
 139                         qunlock(&((*s)->lk));
 140
 141                 if(!waserror()){
 142                         eqlock(&palloc.pwait);  /* Hold memory requesters here */
 143
 144                         if(!waserror()){
 145                                 kickpager();
 146                                 tsleep(&palloc.r, ispages, 0, 1000);
 147                                 poperror();
 148                         }
 149
 150                         qunlock(&palloc.pwait);
 151
 152                         poperror();
 153                 }
 154
 155                 /*
 156                  * If called from fault and we lost the segment from
 157                  * underneath don't waste time allocating and freeing
 158                  * a page. Fault will call newpage again when it has
 159                  * reacquired the segment locks
 160                  */
 161                 if(s){
 162                         *s = 0;
 163                         return 0;
 164                 }
 165
 166                 lock(&palloc);
 167         }
 168
 169         /* First try for our colour */
 170         for(p = palloc.head; p; p = p->next)
 171                 if(p->color == color)
 172                         break;
 173
 174         ct = PG_NOFLUSH;
 175         if(p == 0) {
 176                 p = palloc.head;
 177                 p->color = color;
 178                 ct = PG_NEWCOL;
 179         }
 180
 181         pageunchain(p);
 182
 183         lock(p);
 184         if(p->ref != 0)
 185                 panic("newpage: p->ref %d != 0", p->ref);
 186
 187         uncachepage(p);
 188         p->ref++;
 189         p->va = va;
 190         p->modref = 0;
 191         for(i = 0; i < MAXMACH; i++)
 192                 p->cachectl[i] = ct;
 193         unlock(p);
 194         unlock(&palloc);
 195
 196         if(clear) {
 197                 k = kmap(p);
 198                 memset((void*)VA(k), 0, BY2PG);
 199                 kunmap(k);
 200         }
 201
 202         return p;
 203 }
 204
 205 int
 206 ispages(void*)
 207 {
 208         return palloc.freecount >= swapalloc.highwater;
 209 }
 210
 211 void
 212 putpage(Page *p)
 213 {
 214         if(onswap(p)) {
 215                 putswap(p);
 216                 return;
 217         }
 218
 219         lock(&palloc);
 220         lock(p);
 221
 222         if(p->ref == 0)
 223                 panic("putpage");
 224
 225         if(--p->ref > 0) {
 226                 unlock(p);
 227                 unlock(&palloc);
 228                 return;
 229         }
 230
 231         if(p->image && p->image != &swapimage)
 232                 pagechaintail(p);
 233         else
 234                 pagechainhead(p);
 235
 236         if(palloc.r.p != 0)
 237                 wakeup(&palloc.r);
 238
 239         unlock(p);
 240         unlock(&palloc);
 241 }
 242
 243 Page*
 244 auxpage(void)
 245 {
 246         Page *p;
 247
 248         lock(&palloc);
 249         p = palloc.head;
 250         if(palloc.freecount < swapalloc.highwater) {
 251                 unlock(&palloc);
 252                 return 0;
 253         }
 254         pageunchain(p);
 255
 256         lock(p);
 257         if(p->ref != 0)
 258                 panic("auxpage");
 259         p->ref++;
 260         uncachepage(p);
 261         unlock(p);
 262         unlock(&palloc);
 263
 264         return p;
 265 }
 266
 267 static int dupretries = 15000;
 268
 269 void
 270 duppage(Page *p)                                /* Always call with p locked */
 271 {
 272         Page *np;
 273         int color;
 274         int retries;
 275
 276         retries = 0;
 277 retry:
 278         /* don't dup pages that are shared or have no image */
 279         if(p->ref != 1 || p->image == nil || p->image->notext)
 280                 return;
 281
 282         if(retries++ > dupretries){
 283                 print("duppage %d, up %p\n", retries, up);
 284                 dupretries += 100;
 285                 if(dupretries > 100000)
 286                         panic("duppage");
 287                 uncachepage(p);
 288                 return;
 289         }
 290
 291         /*
 292          *  normal lock ordering is to call
 293          *  lock(&palloc) before lock(p).
 294          *  To avoid deadlock, we have to drop
 295          *  our locks and try again. as the page
 296          *  is from the image cache, this might
 297          *  let someone else come in and grab it
 298          *  so we check page ref above.
 299          */
 300         if(!canlock(&palloc)){
 301                 unlock(p);
 302                 if(up)
 303                         sched();
 304                 lock(p);
 305                 goto retry;
 306         }
 307
 308         /* No freelist cache when memory is very low */
 309         if(palloc.freecount < swapalloc.highwater) {
 310                 unlock(&palloc);
 311                 uncachepage(p);
 312                 return;
 313         }
 314
 315         color = getpgcolor(p->va);
 316         for(np = palloc.head; np; np = np->next)
 317                 if(np->color == color)
 318                         break;
 319
 320         /* No page of the correct color */
 321         if(np == 0) {
 322                 unlock(&palloc);
 323                 uncachepage(p);
 324                 return;
 325         }
 326
 327         pageunchain(np);
 328         pagechaintail(np);
 329 /*
 330 * XXX - here's a bug? - np is on the freelist but it's not really free.
 331 * when we unlock palloc someone else can come in, decide to
 332 * use np, and then try to lock it.  they succeed after we've
 333 * run copypage and cachepage and unlock(np).  then what?
 334 * they call pageunchain before locking(np), so it's removed
 335 * from the freelist, but still in the cache because of
 336 * cachepage below.  if someone else looks in the cache
 337 * before they remove it, the page will have a nonzero ref
 338 * once they finally lock(np).
 339 */
 340         lock(np);
 341         if(np->ref != 0)        /* should never happen */
 342                 panic("duppage: np->ref %d != 0", np->ref);
 343         unlock(&palloc);
 344
 345         /* Cache the new version */
 346         uncachepage(np);
 347         np->va = p->va;
 348         np->daddr = p->daddr;
 349         copypage(p, np);
 350         cachepage(np, p->image);
 351         unlock(np);
 352         uncachepage(p);
 353 }
 354
 355 void
 356 copypage(Page *f, Page *t)
 357 {
 358         KMap *ks, *kd;
 359
 360         ks = kmap(f);
 361         kd = kmap(t);
 362         memmove((void*)VA(kd), (void*)VA(ks), BY2PG);
 363         kunmap(ks);
 364         kunmap(kd);
 365 }
 366
 367 void
 368 uncachepage(Page *p)                    /* Always called with a locked page */
 369 {
 370         Page **l, *f;
 371
 372         if(p->image == 0)
 373                 return;
 374
 375         lock(&palloc.hashlock);
 376         l = &pghash(p->daddr);
 377         for(f = *l; f; f = f->hash) {
 378                 if(f == p) {
 379                         *l = p->hash;
 380                         break;
 381                 }
 382                 l = &f->hash;
 383         }
 384         unlock(&palloc.hashlock);
 385         putimage(p->image);
 386         p->image = 0;
 387         p->daddr = 0;
 388 }
 389
 390 void
 391 cachepage(Page *p, Image *i)
 392 {
 393         Page **l;
 394
 395         /* If this ever happens it should be fixed by calling
 396          * uncachepage instead of panic. I think there is a race
 397          * with pio in which this can happen. Calling uncachepage is
 398          * correct - I just wanted to see if we got here.
 399          */
 400         if(p->image)
 401                 panic("cachepage");
 402
 403         incref(i);
 404         lock(&palloc.hashlock);
 405         p->image = i;
 406         l = &pghash(p->daddr);
 407         p->hash = *l;
 408         *l = p;
 409         unlock(&palloc.hashlock);
 410 }
 411
 412 void
 413 cachedel(Image *i, ulong daddr)
 414 {
 415         Page *f, **l;
 416
 417         lock(&palloc.hashlock);
 418         l = &pghash(daddr);
 419         for(f = *l; f; f = f->hash) {
 420                 if(f->image == i && f->daddr == daddr) {
 421                         lock(f);
 422                         if(f->image == i && f->daddr == daddr){
 423                                 *l = f->hash;
 424                                 putimage(f->image);
 425                                 f->image = 0;
 426                                 f->daddr = 0;
 427                         }
 428                         unlock(f);
 429                         break;
 430                 }
 431                 l = &f->hash;
 432         }
 433         unlock(&palloc.hashlock);
 434 }
 435
 436 Page *
 437 lookpage(Image *i, ulong daddr)
 438 {
 439         Page *f;
 440
 441         lock(&palloc.hashlock);
 442         for(f = pghash(daddr); f; f = f->hash) {
 443                 if(f->image == i && f->daddr == daddr) {
 444                         unlock(&palloc.hashlock);
 445
 446                         lock(&palloc);
 447                         lock(f);
 448                         if(f->image != i || f->daddr != daddr) {
 449                                 unlock(f);
 450                                 unlock(&palloc);
 451                                 return 0;
 452                         }
 453                         if(++f->ref == 1)
 454                                 pageunchain(f);
 455                         unlock(&palloc);
 456                         unlock(f);
 457
 458                         return f;
 459                 }
 460         }
 461         unlock(&palloc.hashlock);
 462
 463         return 0;
 464 }
 465
 466 Pte*
 467 ptecpy(Pte *old)
 468 {
 469         Pte *new;
 470         Page **src, **dst;
 471
 472         new = ptealloc();
 473         dst = &new->pages[old->first-old->pages];
 474         new->first = dst;
 475         for(src = old->first; src <= old->last; src++, dst++)
 476                 if(*src) {
 477                         if(onswap(*src))
 478                                 dupswap(*src);
 479                         else {
 480                                 lock(*src);
 481                                 (*src)->ref++;
 482                                 unlock(*src);
 483                         }
 484                         new->last = dst;
 485                         *dst = *src;
 486                 }
 487
 488         return new;
 489 }
 490
 491 Pte*
 492 ptealloc(void)
 493 {
 494         Pte *new;
 495
 496         new = smalloc(sizeof(Pte));
 497         new->first = &new->pages[PTEPERTAB];
 498         new->last = new->pages;
 499         return new;
 500 }
 501
 502 void
 503 freepte(Segment *s, Pte *p)
 504 {
 505         int ref;
 506         void (*fn)(Page*);
 507         Page *pt, **pg, **ptop;
 508
 509         switch(s->type&SG_TYPE) {
 510         case SG_PHYSICAL:
 511                 fn = s->pseg->pgfree;
 512                 ptop = &p->pages[PTEPERTAB];
 513                 if(fn) {
 514                         for(pg = p->pages; pg < ptop; pg++) {
 515                                 if(*pg == 0)
 516                                         continue;
 517                                 (*fn)(*pg);
 518                                 *pg = 0;
 519                         }
 520                         break;
 521                 }
 522                 for(pg = p->pages; pg < ptop; pg++) {
 523                         pt = *pg;
 524                         if(pt == 0)
 525                                 continue;
 526                         lock(pt);
 527                         ref = --pt->ref;
 528                         unlock(pt);
 529                         if(ref == 0)
 530                                 free(pt);
 531                 }
 532                 break;
 533         default:
 534                 for(pg = p->first; pg <= p->last; pg++)
 535                         if(*pg) {
 536                                 putpage(*pg);
 537                                 *pg = 0;
 538                         }
 539         }
 540         free(p);
 541 }
 542
 543 ulong
 544 pagenumber(Page *p)
 545 {
 546         return p-palloc.pages;
 547 }
 548
 549 void
 550 checkpagerefs(void)
 551 {
 552         int s;
 553         ulong i, np, nwrong;
 554         ulong *ref;
 555
 556         np = palloc.user;
 557         ref = malloc(np*sizeof ref[0]);
 558         if(ref == nil){
 559                 print("checkpagerefs: out of memory\n");
 560                 return;
 561         }
 562
 563         /*
 564          * This may not be exact if there are other processes
 565          * holding refs to pages on their stacks.  The hope is
 566          * that if you run it on a quiescent system it will still
 567          * be useful.
 568          */
 569         s = splhi();
 570         lock(&palloc);
 571         countpagerefs(ref, 0);
 572         portcountpagerefs(ref, 0);
 573         nwrong = 0;
 574         for(i=0; i<np; i++){
 575                 if(palloc.pages[i].ref != ref[i]){
 576                         iprint("page %#.8lux ref %d actual %lud\n",
 577                                 palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
 578                         ref[i] = 1;
 579                         nwrong++;
 580                 }else
 581                         ref[i] = 0;
 582         }
 583         countpagerefs(ref, 1);
 584         portcountpagerefs(ref, 1);
 585         iprint("%lud mistakes found\n", nwrong);
 586         unlock(&palloc);
 587         splx(s);
 588 }
 589
 590 void
 591 portcountpagerefs(ulong *ref, int print)
 592 {
 593         ulong i, j, k, ns, n;
 594         Page **pg, *entry;
 595         Proc *p;
 596         Pte *pte;
 597         Segment *s;
 598
 599         /*
 600          * Pages in segments.  s->mark avoids double-counting.
 601          */
 602         n = 0;
 603         ns = 0;
 604         for(i=0; i<conf.nproc; i++){
 605                 p = proctab(i);
 606                 for(j=0; j<NSEG; j++){
 607                         s = p->seg[j];
 608                         if(s)
 609                                 s->mark = 0;
 610                 }
 611         }
 612         for(i=0; i<conf.nproc; i++){
 613                 p = proctab(i);
 614                 for(j=0; j<NSEG; j++){
 615                         s = p->seg[j];
 616                         if(s == nil || s->mark++)
 617                                 continue;
 618                         ns++;
 619                         for(k=0; k<s->mapsize; k++){
 620                                 pte = s->map[k];
 621                                 if(pte == nil)
 622                                         continue;
 623                                 for(pg = pte->first; pg <= pte->last; pg++){
 624                                         entry = *pg;
 625                                         if(pagedout(entry))
 626                                                 continue;
 627                                         if(print){
 628                                                 if(ref[pagenumber(entry)])
 629                                                         iprint("page %#.8lux in segment %#p\n", entry->pa, s);
 630                                                 continue;
 631                                         }
 632                                         if(ref[pagenumber(entry)]++ == 0)
 633                                                 n++;
 634                                 }
 635                         }
 636                 }
 637         }
 638         if(!print){
 639                 iprint("%lud pages in %lud segments\n", n, ns);
 640                 for(i=0; i<conf.nproc; i++){
 641                         p = proctab(i);
 642                         for(j=0; j<NSEG; j++){
 643                                 s = p->seg[j];
 644                                 if(s == nil)
 645                                         continue;
 646                                 if(s->ref != s->mark){
 647                                         iprint("segment %#p (used by proc %lud pid %lud) has bad ref count %lud actual %lud\n",
 648                                                 s, i, p->pid, s->ref, s->mark);
 649                                 }
 650                         }
 651                 }
 652         }
 653 }
 654