sys/src/9/port/page.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "../port/error.h"
   7
   8 Palloc palloc;
   9
  10 void
  11 pageinit(void)
  12 {
  13         int color, i, j;
  14         Page *p;
  15         Pallocmem *pm;
  16         vlong m, v, u;
  17
  18         if(palloc.pages == nil){
  19                 ulong np;
  20
  21                 np = 0;
  22                 for(i=0; i<nelem(palloc.mem); i++){
  23                         pm = &palloc.mem[i];
  24                         np += pm->npage;
  25                 }
  26                 palloc.pages = xalloc(np*sizeof(Page));
  27                 if(palloc.pages == nil)
  28                         panic("pageinit");
  29         }
  30
  31         color = 0;
  32         palloc.head = nil;
  33         p = palloc.pages;
  34         for(i=0; i<nelem(palloc.mem); i++){
  35                 pm = &palloc.mem[i];
  36                 for(j=0; j<pm->npage; j++){
  37                         memset(p, 0, sizeof *p);
  38                         p->pa = pm->base+j*BY2PG;
  39                         if(cankaddr(p->pa) && (KADDR(p->pa) == nil || KADDR(p->pa) == (void*)-BY2PG))
  40                                 continue;
  41                         p->color = color;
  42                         color = (color+1)%NCOLOR;
  43                         pagechainhead(p);
  44                         p++;
  45                 }
  46         }
  47
  48         palloc.user = p - palloc.pages;
  49         u = palloc.user*BY2PG;
  50         v = u + conf.nswap*BY2PG;
  51
  52         /* Paging numbers */
  53         swapalloc.highwater = (palloc.user*5)/100;
  54         swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4);
  55
  56         m = 0;
  57         for(i=0; i<nelem(conf.mem); i++)
  58                 if(conf.mem[i].npage)
  59                         m += conf.mem[i].npage*BY2PG;
  60         m += PGROUND(end - (char*)KTZERO);
  61
  62         print("%lldM memory: ", (m+1024*1024-1)/(1024*1024));
  63         print("%lldM kernel data, ", (m-u+1024*1024-1)/(1024*1024));
  64         print("%lldM user, ", u/(1024*1024));
  65         print("%lldM swap\n", v/(1024*1024));
  66 }
  67
  68 void
  69 pagechainhead(Page *p)
  70 {
  71         p->next = palloc.head;
  72         palloc.head = p;
  73         palloc.freecount++;
  74 }
  75
  76 void
  77 pagechaindone(void)
  78 {
  79         if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
  80                 return;
  81         if(palloc.pwait[1].p != nil)
  82                 wakeup(&palloc.pwait[1]);
  83 }
  84
  85 static void
  86 freepages(Page *head, Page *tail, ulong np)
  87 {
  88         lock(&palloc);
  89         tail->next = palloc.head;
  90         palloc.head = head;
  91         palloc.freecount += np;
  92         pagechaindone();
  93         unlock(&palloc);
  94 }
  95
  96 ulong
  97 pagereclaim(Image *i, ulong pages)
  98 {
  99         Page **h, **l, **x, *p;
 100         Page *fh, *ft;
 101         ulong np;
 102
 103         if(pages == 0)
 104                 return 0;
 105
 106         lock(i);
 107         if(i->pgref == 0){
 108                 unlock(i);
 109                 return 0;
 110         }
 111         incref(i);
 112
 113         np = 0;
 114         fh = ft = nil;
 115         for(h = i->pghash; h < &i->pghash[PGHSIZE]; h++){
 116                 l = h;
 117                 x = nil;
 118                 for(p = *l; p != nil; p = p->next){
 119                         if(p->ref == 0)
 120                                 x = l;
 121                         l = &p->next;
 122                 }
 123                 if(x == nil)
 124                         continue;
 125
 126                 p = *x;
 127                 *x = p->next;
 128                 p->next = nil;
 129                 p->image = nil;
 130                 p->daddr = ~0;
 131                 i->pgref--;
 132                 decref(i);
 133
 134                 if(fh == nil)
 135                         fh = p;
 136                 else
 137                         ft->next = p;
 138                 ft = p;
 139                 if(++np >= pages)
 140                         break;
 141         }
 142         unlock(i);
 143         putimage(i);
 144
 145         if(np > 0)
 146                 freepages(fh, ft, np);
 147
 148         return np;
 149 }
 150
 151 static int
 152 ispages(void*)
 153 {
 154         return palloc.freecount > swapalloc.highwater || up->noswap && palloc.freecount > 0;
 155 }
 156
 157 Page*
 158 newpage(int clear, Segment **s, uintptr va)
 159 {
 160         Page *p, **l;
 161         KMap *k;
 162         int color;
 163
 164         lock(&palloc);
 165         while(!ispages(nil)){
 166                 unlock(&palloc);
 167                 if(s != nil)
 168                         qunlock(*s);
 169
 170                 if(!waserror()){
 171                         Rendezq *q;
 172
 173                         q = &palloc.pwait[!up->noswap];
 174                         eqlock(q);
 175                         if(!waserror()){
 176                                 kickpager();
 177                                 sleep(q, ispages, nil);
 178                                 poperror();
 179                         }
 180                         qunlock(q);
 181                         poperror();
 182                 }
 183
 184                 /*
 185                  * If called from fault and we lost the segment from
 186                  * underneath don't waste time allocating and freeing
 187                  * a page. Fault will call newpage again when it has
 188                  * reacquired the segment locks
 189                  */
 190                 if(s != nil){
 191                         *s = nil;
 192                         return nil;
 193                 }
 194                 lock(&palloc);
 195         }
 196
 197         /* First try for our colour */
 198         color = getpgcolor(va);
 199         l = &palloc.head;
 200         for(p = *l; p != nil; p = p->next){
 201                 if(p->color == color)
 202                         break;
 203                 l = &p->next;
 204         }
 205
 206         if(p == nil) {
 207                 l = &palloc.head;
 208                 p = *l;
 209         }
 210
 211         *l = p->next;
 212         p->next = nil;
 213         palloc.freecount--;
 214         unlock(&palloc);
 215
 216         p->ref = 1;
 217         p->va = va;
 218         p->modref = 0;
 219         p->txtflush = 0;
 220
 221         if(clear) {
 222                 k = kmap(p);
 223                 memset((void*)VA(k), 0, BY2PG);
 224                 kunmap(k);
 225         }
 226
 227         return p;
 228 }
 229
 230 void
 231 putpage(Page *p)
 232 {
 233         if(onswap(p)) {
 234                 putswap(p);
 235                 return;
 236         }
 237         if(p->image != nil) {
 238                 decref(p);
 239                 return;
 240         }
 241         if(decref(p) == 0)
 242                 freepages(p, p, 1);
 243 }
 244
 245 void
 246 copypage(Page *f, Page *t)
 247 {
 248         KMap *ks, *kd;
 249
 250         ks = kmap(f);
 251         kd = kmap(t);
 252         memmove((void*)VA(kd), (void*)VA(ks), BY2PG);
 253         kunmap(ks);
 254         kunmap(kd);
 255 }
 256
 257 void
 258 cachepage(Page *p, Image *i)
 259 {
 260         Page **h;
 261
 262         lock(i);
 263         p->image = i;
 264         h = &PGHASH(i, p->daddr);
 265         p->next = *h;
 266         *h = p;
 267         incref(i);
 268         i->pgref++;
 269         unlock(i);
 270 }
 271
 272 void
 273 uncachepage(Page *p)
 274 {
 275         Page **l, *x;
 276         Image *i;
 277
 278         i = p->image;
 279         if(i == nil)
 280                 return;
 281
 282         lock(i);
 283         if(p->image != i){
 284                 unlock(i);
 285                 return;
 286         }
 287         l = &PGHASH(i, p->daddr);
 288         for(x = *l; x != nil; x = x->next) {
 289                 if(x == p){
 290                         *l = p->next;
 291                         p->next = nil;
 292                         p->image = nil;
 293                         p->daddr = ~0;
 294                         i->pgref--;
 295                         unlock(i);
 296                         putimage(i);
 297                         return;
 298                 }
 299                 l = &x->next;
 300         }
 301         unlock(i);
 302 }
 303
 304 Page*
 305 lookpage(Image *i, uintptr daddr)
 306 {
 307         Page *p, **h, **l;
 308
 309         lock(i);
 310         l = h = &PGHASH(i, daddr);
 311         for(p = *l; p != nil; p = p->next){
 312                 if(p->daddr == daddr){
 313                         *l = p->next;
 314                         p->next = *h;
 315                         *h = p;
 316                         incref(p);
 317                         unlock(i);
 318                         return p;
 319                 }
 320                 l = &p->next;
 321         }
 322         unlock(i);
 323
 324         return nil;
 325 }
 326
 327 void
 328 cachedel(Image *i, uintptr daddr)
 329 {
 330         Page *p;
 331
 332         while((p = lookpage(i, daddr)) != nil){
 333                 uncachepage(p);
 334                 putpage(p);
 335         }
 336 }
 337
 338
 339 Pte*
 340 ptecpy(Pte *old)
 341 {
 342         Pte *new;
 343         Page **src, **dst;
 344
 345         new = ptealloc();
 346         dst = &new->pages[old->first-old->pages];
 347         new->first = dst;
 348         for(src = old->first; src <= old->last; src++, dst++)
 349                 if(*src != nil) {
 350                         if(onswap(*src))
 351                                 dupswap(*src);
 352                         else
 353                                 incref(*src);
 354                         new->last = dst;
 355                         *dst = *src;
 356                 }
 357
 358         return new;
 359 }
 360
 361 Pte*
 362 ptealloc(void)
 363 {
 364         Pte *new;
 365
 366         new = smalloc(sizeof(Pte));
 367         new->first = &new->pages[PTEPERTAB];
 368         new->last = new->pages;
 369         return new;
 370 }
 371
 372 void
 373 freepte(Segment *s, Pte *p)
 374 {
 375         Page **pg, **pe;
 376
 377         pg = p->first;
 378         pe = p->last;
 379
 380         switch(s->type&SG_TYPE) {
 381         case SG_PHYSICAL:
 382                 while(pg <= pe){
 383                         if(*pg != nil && decref(*pg) == 0)
 384                                 free(*pg);
 385                         pg++;
 386                 }
 387                 break;
 388         default:
 389                 while(pg <= pe){
 390                         if(*pg != nil)
 391                                 putpage(*pg);
 392                         pg++;
 393                 }
 394         }
 395         free(p);
 396 }
 397
 398 ulong
 399 pagenumber(Page *p)
 400 {
 401         return p-palloc.pages;
 402 }
 403
 404 void
 405 checkpagerefs(void)
 406 {
 407         int s;
 408         ulong i, np, nwrong;
 409         ulong *ref;
 410
 411         np = palloc.user;
 412         ref = malloc(np*sizeof ref[0]);
 413         if(ref == nil){
 414                 print("checkpagerefs: out of memory\n");
 415                 return;
 416         }
 417
 418         /*
 419          * This may not be exact if there are other processes
 420          * holding refs to pages on their stacks.  The hope is
 421          * that if you run it on a quiescent system it will still
 422          * be useful.
 423          */
 424         s = splhi();
 425         lock(&palloc);
 426         countpagerefs(ref, 0);
 427         portcountpagerefs(ref, 0);
 428         nwrong = 0;
 429         for(i=0; i<np; i++){
 430                 if(palloc.pages[i].ref != ref[i]){
 431                         iprint("page %#p ref %ld actual %lud\n",
 432                                 palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
 433                         ref[i] = 1;
 434                         nwrong++;
 435                 }else
 436                         ref[i] = 0;
 437         }
 438         countpagerefs(ref, 1);
 439         portcountpagerefs(ref, 1);
 440         iprint("%lud mistakes found\n", nwrong);
 441         unlock(&palloc);
 442         splx(s);
 443 }
 444
 445 void
 446 portcountpagerefs(ulong *ref, int print)
 447 {
 448         ulong i, j, k, ns, n;
 449         Page **pg, *entry;
 450         Proc *p;
 451         Pte *pte;
 452         Segment *s;
 453
 454         /*
 455          * Pages in segments.  s->mark avoids double-counting.
 456          */
 457         n = 0;
 458         ns = 0;
 459         for(i=0; i<conf.nproc; i++){
 460                 p = proctab(i);
 461                 for(j=0; j<NSEG; j++){
 462                         s = p->seg[j];
 463                         if(s != nil)
 464                                 s->mark = 0;
 465                 }
 466         }
 467         for(i=0; i<conf.nproc; i++){
 468                 p = proctab(i);
 469                 for(j=0; j<NSEG; j++){
 470                         s = p->seg[j];
 471                         if(s == nil || s->mark++)
 472                                 continue;
 473                         if((s->type&SG_TYPE) == SG_PHYSICAL)
 474                                 continue;
 475                         ns++;
 476                         for(k=0; k<s->mapsize; k++){
 477                                 pte = s->map[k];
 478                                 if(pte == nil)
 479                                         continue;
 480                                 for(pg = pte->first; pg <= pte->last; pg++){
 481                                         entry = *pg;
 482                                         if(pagedout(entry))
 483                                                 continue;
 484                                         if(print){
 485                                                 if(ref[pagenumber(entry)])
 486                                                         iprint("page %#p in segment %#p\n", entry->pa, s);
 487                                                 continue;
 488                                         }
 489                                         if(ref[pagenumber(entry)]++ == 0)
 490                                                 n++;
 491                                 }
 492                         }
 493                 }
 494         }
 495         if(!print){
 496                 iprint("%lud pages in %lud segments\n", n, ns);
 497                 for(i=0; i<conf.nproc; i++){
 498                         p = proctab(i);
 499                         for(j=0; j<NSEG; j++){
 500                                 s = p->seg[j];
 501                                 if(s == nil)
 502                                         continue;
 503                                 if(s->ref != s->mark){
 504                                         iprint("segment %#p (used by proc %lud pid %lud) has bad ref count %lud actual %lud\n",
 505                                                 s, i, p->pid, s->ref, s->mark);
 506                                 }
 507                         }
 508                 }
 509         }
 510 }
 511