sys/src/9/port/page.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "../port/error.h"
   7
   8 Palloc palloc;
   9
  10 void
  11 pageinit(void)
  12 {
  13         int color, i, j;
  14         Page *p;
  15         Pallocmem *pm;
  16         vlong m, v, u;
  17
  18         if(palloc.pages == nil){
  19                 ulong np;
  20
  21                 np = 0;
  22                 for(i=0; i<nelem(palloc.mem); i++){
  23                         pm = &palloc.mem[i];
  24                         np += pm->npage;
  25                 }
  26                 palloc.pages = xalloc(np*sizeof(Page));
  27                 if(palloc.pages == nil)
  28                         panic("pageinit");
  29         }
  30
  31         color = 0;
  32         palloc.head = nil;
  33         p = palloc.pages;
  34         for(i=0; i<nelem(palloc.mem); i++){
  35                 pm = &palloc.mem[i];
  36                 for(j=0; j<pm->npage; j++){
  37                         memset(p, 0, sizeof *p);
  38                         p->pa = pm->base+j*BY2PG;
  39                         p->color = color;
  40                         color = (color+1)%NCOLOR;
  41                         pagechainhead(p);
  42                         p++;
  43                 }
  44         }
  45
  46         palloc.user = p - palloc.pages;
  47         u = palloc.user*BY2PG;
  48         v = u + conf.nswap*BY2PG;
  49
  50         /* Paging numbers */
  51         swapalloc.highwater = (palloc.user*5)/100;
  52         swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4);
  53
  54         m = 0;
  55         for(i=0; i<nelem(conf.mem); i++)
  56                 if(conf.mem[i].npage)
  57                         m += conf.mem[i].npage*BY2PG;
  58         m += PGROUND(end - (char*)KTZERO);
  59
  60         print("%lldM memory: ", (m+1024*1024-1)/(1024*1024));
  61         print("%lldM kernel data, ", (m-u+1024*1024-1)/(1024*1024));
  62         print("%lldM user, ", u/(1024*1024));
  63         print("%lldM swap\n", v/(1024*1024));
  64 }
  65
  66 void
  67 pagechainhead(Page *p)
  68 {
  69         p->next = palloc.head;
  70         palloc.head = p;
  71         palloc.freecount++;
  72 }
  73
  74 static void
  75 freepages(Page *head, Page *tail, int n)
  76 {
  77         lock(&palloc);
  78         tail->next = palloc.head;
  79         palloc.head = head;
  80         palloc.freecount += n;
  81         unlock(&palloc);
  82
  83         if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
  84                 return;
  85         if(palloc.pwait[1].p != nil)
  86                 wakeup(&palloc.pwait[1]);
  87 }
  88
  89 int
  90 pagereclaim(Image *i, int min)
  91 {
  92         Page **h, **l, **x, *p;
  93         Page *fh, *ft;
  94         int n;
  95
  96         lock(i);
  97         if(i->pgref == 0){
  98                 unlock(i);
  99                 return 0;
 100         }
 101         incref(i);
 102
 103         n = 0;
 104         fh = ft = nil;
 105         for(h = i->pghash; h < &i->pghash[PGHSIZE]; h++){
 106                 l = h;
 107                 x = nil;
 108                 for(p = *l; p != nil; p = p->next){
 109                         if(p->ref == 0)
 110                                 x = l;
 111                         l = &p->next;
 112                 }
 113                 if(x == nil)
 114                         continue;
 115
 116                 p = *x;
 117                 *x = p->next;
 118                 p->next = nil;
 119                 p->image = nil;
 120                 p->daddr = ~0;
 121                 i->pgref--;
 122                 decref(i);
 123
 124                 if(fh == nil)
 125                         fh = p;
 126                 else
 127                         ft->next = p;
 128                 ft = p;
 129                 if(++n >= min)
 130                         break;
 131         }
 132         unlock(i);
 133         putimage(i);
 134
 135         if(n > 0)
 136                 freepages(fh, ft, n);
 137
 138         return n;
 139 }
 140
 141 static int
 142 ispages(void*)
 143 {
 144         return palloc.freecount > swapalloc.highwater || up->noswap && palloc.freecount > 0;
 145 }
 146
 147 Page*
 148 newpage(int clear, Segment **s, uintptr va)
 149 {
 150         Page *p, **l;
 151         KMap *k;
 152         int color;
 153
 154         lock(&palloc);
 155         while(!ispages(nil)){
 156                 unlock(&palloc);
 157                 if(s != nil)
 158                         qunlock(*s);
 159
 160                 if(!waserror()){
 161                         Rendezq *q;
 162
 163                         q = &palloc.pwait[!up->noswap];
 164                         eqlock(q);
 165                         if(!waserror()){
 166                                 kickpager();
 167                                 sleep(q, ispages, nil);
 168                                 poperror();
 169                         }
 170                         qunlock(q);
 171                         poperror();
 172                 }
 173
 174                 /*
 175                  * If called from fault and we lost the segment from
 176                  * underneath don't waste time allocating and freeing
 177                  * a page. Fault will call newpage again when it has
 178                  * reacquired the segment locks
 179                  */
 180                 if(s != nil){
 181                         *s = nil;
 182                         return nil;
 183                 }
 184                 lock(&palloc);
 185         }
 186
 187         /* First try for our colour */
 188         color = getpgcolor(va);
 189         l = &palloc.head;
 190         for(p = *l; p != nil; p = p->next){
 191                 if(p->color == color)
 192                         break;
 193                 l = &p->next;
 194         }
 195
 196         if(p == nil) {
 197                 l = &palloc.head;
 198                 p = *l;
 199         }
 200
 201         *l = p->next;
 202         p->next = nil;
 203         palloc.freecount--;
 204         unlock(&palloc);
 205
 206         p->ref = 1;
 207         p->va = va;
 208         p->modref = 0;
 209         p->txtflush = 0;
 210
 211         if(clear) {
 212                 k = kmap(p);
 213                 memset((void*)VA(k), 0, BY2PG);
 214                 kunmap(k);
 215         }
 216
 217         return p;
 218 }
 219
 220 void
 221 putpage(Page *p)
 222 {
 223         if(onswap(p)) {
 224                 putswap(p);
 225                 return;
 226         }
 227         if(p->image != nil) {
 228                 decref(p);
 229                 return;
 230         }
 231         if(decref(p) == 0)
 232                 freepages(p, p, 1);
 233 }
 234
 235 void
 236 copypage(Page *f, Page *t)
 237 {
 238         KMap *ks, *kd;
 239
 240         ks = kmap(f);
 241         kd = kmap(t);
 242         memmove((void*)VA(kd), (void*)VA(ks), BY2PG);
 243         kunmap(ks);
 244         kunmap(kd);
 245 }
 246
 247 void
 248 cachepage(Page *p, Image *i)
 249 {
 250         Page **h;
 251
 252         lock(i);
 253         p->image = i;
 254         h = &PGHASH(i, p->daddr);
 255         p->next = *h;
 256         *h = p;
 257         incref(i);
 258         i->pgref++;
 259         unlock(i);
 260 }
 261
 262 void
 263 uncachepage(Page *p)
 264 {
 265         Page **l, *x;
 266         Image *i;
 267
 268         i = p->image;
 269         if(i == nil)
 270                 return;
 271
 272         lock(i);
 273         if(p->image != i){
 274                 unlock(i);
 275                 return;
 276         }
 277         l = &PGHASH(i, p->daddr);
 278         for(x = *l; x != nil; x = x->next) {
 279                 if(x == p){
 280                         *l = p->next;
 281                         p->next = nil;
 282                         p->image = nil;
 283                         p->daddr = ~0;
 284                         i->pgref--;
 285                         unlock(i);
 286                         putimage(i);
 287                         return;
 288                 }
 289                 l = &x->next;
 290         }
 291         unlock(i);
 292 }
 293
 294 Page*
 295 lookpage(Image *i, uintptr daddr)
 296 {
 297         Page *p, **h, **l;
 298
 299         lock(i);
 300         l = h = &PGHASH(i, daddr);
 301         for(p = *l; p != nil; p = p->next){
 302                 if(p->daddr == daddr){
 303                         *l = p->next;
 304                         p->next = *h;
 305                         *h = p;
 306                         incref(p);
 307                         unlock(i);
 308                         return p;
 309                 }
 310                 l = &p->next;
 311         }
 312         unlock(i);
 313
 314         return nil;
 315 }
 316
 317 void
 318 cachedel(Image *i, uintptr daddr)
 319 {
 320         Page *p;
 321
 322         while((p = lookpage(i, daddr)) != nil){
 323                 uncachepage(p);
 324                 putpage(p);
 325         }
 326 }
 327
 328
 329 Pte*
 330 ptecpy(Pte *old)
 331 {
 332         Pte *new;
 333         Page **src, **dst;
 334
 335         new = ptealloc();
 336         dst = &new->pages[old->first-old->pages];
 337         new->first = dst;
 338         for(src = old->first; src <= old->last; src++, dst++)
 339                 if(*src != nil) {
 340                         if(onswap(*src))
 341                                 dupswap(*src);
 342                         else
 343                                 incref(*src);
 344                         new->last = dst;
 345                         *dst = *src;
 346                 }
 347
 348         return new;
 349 }
 350
 351 Pte*
 352 ptealloc(void)
 353 {
 354         Pte *new;
 355
 356         new = smalloc(sizeof(Pte));
 357         new->first = &new->pages[PTEPERTAB];
 358         new->last = new->pages;
 359         return new;
 360 }
 361
 362 void
 363 freepte(Segment *s, Pte *p)
 364 {
 365         Page **pg, **pe;
 366
 367         pg = p->first;
 368         pe = p->last;
 369
 370         switch(s->type&SG_TYPE) {
 371         case SG_PHYSICAL:
 372                 while(pg <= pe){
 373                         if(*pg != nil && decref(*pg) == 0)
 374                                 free(*pg);
 375                         pg++;
 376                 }
 377                 break;
 378         default:
 379                 while(pg <= pe){
 380                         if(*pg != nil)
 381                                 putpage(*pg);
 382                         pg++;
 383                 }
 384         }
 385         free(p);
 386 }
 387
 388 ulong
 389 pagenumber(Page *p)
 390 {
 391         return p-palloc.pages;
 392 }
 393
 394 void
 395 checkpagerefs(void)
 396 {
 397         int s;
 398         ulong i, np, nwrong;
 399         ulong *ref;
 400
 401         np = palloc.user;
 402         ref = malloc(np*sizeof ref[0]);
 403         if(ref == nil){
 404                 print("checkpagerefs: out of memory\n");
 405                 return;
 406         }
 407
 408         /*
 409          * This may not be exact if there are other processes
 410          * holding refs to pages on their stacks.  The hope is
 411          * that if you run it on a quiescent system it will still
 412          * be useful.
 413          */
 414         s = splhi();
 415         lock(&palloc);
 416         countpagerefs(ref, 0);
 417         portcountpagerefs(ref, 0);
 418         nwrong = 0;
 419         for(i=0; i<np; i++){
 420                 if(palloc.pages[i].ref != ref[i]){
 421                         iprint("page %#p ref %ld actual %lud\n",
 422                                 palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
 423                         ref[i] = 1;
 424                         nwrong++;
 425                 }else
 426                         ref[i] = 0;
 427         }
 428         countpagerefs(ref, 1);
 429         portcountpagerefs(ref, 1);
 430         iprint("%lud mistakes found\n", nwrong);
 431         unlock(&palloc);
 432         splx(s);
 433 }
 434
 435 void
 436 portcountpagerefs(ulong *ref, int print)
 437 {
 438         ulong i, j, k, ns, n;
 439         Page **pg, *entry;
 440         Proc *p;
 441         Pte *pte;
 442         Segment *s;
 443
 444         /*
 445          * Pages in segments.  s->mark avoids double-counting.
 446          */
 447         n = 0;
 448         ns = 0;
 449         for(i=0; i<conf.nproc; i++){
 450                 p = proctab(i);
 451                 for(j=0; j<NSEG; j++){
 452                         s = p->seg[j];
 453                         if(s != nil)
 454                                 s->mark = 0;
 455                 }
 456         }
 457         for(i=0; i<conf.nproc; i++){
 458                 p = proctab(i);
 459                 for(j=0; j<NSEG; j++){
 460                         s = p->seg[j];
 461                         if(s == nil || s->mark++)
 462                                 continue;
 463                         if((s->type&SG_TYPE) == SG_PHYSICAL)
 464                                 continue;
 465                         ns++;
 466                         for(k=0; k<s->mapsize; k++){
 467                                 pte = s->map[k];
 468                                 if(pte == nil)
 469                                         continue;
 470                                 for(pg = pte->first; pg <= pte->last; pg++){
 471                                         entry = *pg;
 472                                         if(pagedout(entry))
 473                                                 continue;
 474                                         if(print){
 475                                                 if(ref[pagenumber(entry)])
 476                                                         iprint("page %#p in segment %#p\n", entry->pa, s);
 477                                                 continue;
 478                                         }
 479                                         if(ref[pagenumber(entry)]++ == 0)
 480                                                 n++;
 481                                 }
 482                         }
 483                 }
 484         }
 485         if(!print){
 486                 iprint("%lud pages in %lud segments\n", n, ns);
 487                 for(i=0; i<conf.nproc; i++){
 488                         p = proctab(i);
 489                         for(j=0; j<NSEG; j++){
 490                                 s = p->seg[j];
 491                                 if(s == nil)
 492                                         continue;
 493                                 if(s->ref != s->mark){
 494                                         iprint("segment %#p (used by proc %lud pid %lud) has bad ref count %lud actual %lud\n",
 495                                                 s, i, p->pid, s->ref, s->mark);
 496                                 }
 497                         }
 498                 }
 499         }
 500 }
 501