sys/src/9/port/swap.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "../port/error.h"
   7
   8 static int      canflush(Proc*, Segment*);
   9 static void     executeio(void);
  10 static int      needpages(void*);
  11 static void     pageout(Proc*, Segment*);
  12 static void     pagepte(int, Page**);
  13 static void     pager(void*);
  14
  15 Image   swapimage;
  16
  17 static  int     swopen;
  18 static  Page    **iolist;
  19 static  int     ioptr;
  20
  21 static  ulong   genage, genclock, gencount;
  22 static  uvlong  gensum;
  23
  24 static void
  25 gentick(void)
  26 {
  27         genclock++;
  28         if(gencount)
  29                 genage = gensum / gencount;
  30         else
  31                 genage = 0;
  32         gensum = gencount = 0;
  33 }
  34
  35 void
  36 swapinit(void)
  37 {
  38         swapalloc.swmap = xalloc(conf.nswap);
  39         swapalloc.top = &swapalloc.swmap[conf.nswap];
  40         swapalloc.alloc = swapalloc.swmap;
  41         swapalloc.last = swapalloc.swmap;
  42         swapalloc.free = conf.nswap;
  43         swapalloc.xref = 0;
  44
  45         iolist = xalloc(conf.nswppo*sizeof(Page*));
  46         if(swapalloc.swmap == 0 || iolist == 0)
  47                 panic("swapinit: not enough memory");
  48
  49         swapimage.notext = 1;
  50 }
  51
  52 ulong
  53 newswap(void)
  54 {
  55         uchar *look;
  56
  57         lock(&swapalloc);
  58         if(swapalloc.free == 0) {
  59                 unlock(&swapalloc);
  60                 return ~0;
  61         }
  62
  63         look = memchr(swapalloc.last, 0, swapalloc.top-swapalloc.last);
  64         if(look == 0)
  65                 panic("inconsistent swap");
  66
  67         *look = 1;
  68         swapalloc.last = look;
  69         swapalloc.free--;
  70         unlock(&swapalloc);
  71         return (look-swapalloc.swmap) * BY2PG;
  72 }
  73
  74 void
  75 putswap(Page *p)
  76 {
  77         uchar *idx;
  78
  79         lock(&swapalloc);
  80         idx = &swapalloc.swmap[((ulong)p)/BY2PG];
  81         if(*idx == 0)
  82                 panic("putswap %#p ref == 0", p);
  83
  84         if(*idx == 255) {
  85                 if(swapalloc.xref == 0)
  86                         panic("putswap %#p xref == 0", p);
  87
  88                 if(--swapalloc.xref == 0) {
  89                         for(idx = swapalloc.swmap; idx < swapalloc.top; idx++) {
  90                                 if(*idx == 255) {
  91                                         *idx = 0;
  92                                         swapalloc.free++;
  93                                         if(idx < swapalloc.last)
  94                                                 swapalloc.last = idx;
  95                                 }
  96                         }
  97                 }
  98         } else {
  99                 if(--(*idx) == 0) {
 100                         swapalloc.free++;
 101                         if(idx < swapalloc.last)
 102                                 swapalloc.last = idx;
 103                 }
 104         }
 105         unlock(&swapalloc);
 106 }
 107
 108 void
 109 dupswap(Page *p)
 110 {
 111         uchar *idx;
 112
 113         lock(&swapalloc);
 114         idx = &swapalloc.swmap[((ulong)p)/BY2PG];
 115         if(*idx == 255)
 116                 swapalloc.xref++;
 117         else {
 118                 if(++(*idx) == 255)
 119                         swapalloc.xref += 255;
 120         }
 121         unlock(&swapalloc);
 122 }
 123
 124 int
 125 swapcount(ulong daddr)
 126 {
 127         return swapalloc.swmap[daddr/BY2PG];
 128 }
 129
 130 void
 131 kickpager(void)
 132 {
 133         static int started;
 134
 135         if(started)
 136                 wakeup(&swapalloc.r);
 137         else {
 138                 kproc("pager", pager, 0);
 139                 started = 1;
 140         }
 141 }
 142
 143 static void
 144 pager(void *junk)
 145 {
 146         int i;
 147         Segment *s;
 148         Proc *p, *ep;
 149
 150         if(waserror())
 151                 panic("pager: os error");
 152
 153         p = proctab(0);
 154         ep = &p[conf.nproc];
 155
 156 loop:
 157         up->psstate = "Idle";
 158         wakeup(&palloc.r);
 159         sleep(&swapalloc.r, needpages, 0);
 160
 161         while(needpages(junk)) {
 162                 if(swapimage.c && swapalloc.free) {
 163                         p++;
 164                         if(p >= ep){
 165                                 p = proctab(0);
 166                                 gentick();
 167                         }
 168
 169                         if(p->state == Dead || p->noswap)
 170                                 continue;
 171
 172                         if(!canqlock(&p->seglock))
 173                                 continue;               /* process changing its segments */
 174
 175                         for(i = 0; i < NSEG; i++) {
 176                                 if(!needpages(junk)){
 177                                         qunlock(&p->seglock);
 178                                         goto loop;
 179                                 }
 180
 181                                 if(s = p->seg[i]) {
 182                                         switch(s->type&SG_TYPE) {
 183                                         default:
 184                                                 break;
 185                                         case SG_TEXT:
 186                                                 pageout(p, s);
 187                                                 break;
 188                                         case SG_DATA:
 189                                         case SG_BSS:
 190                                         case SG_STACK:
 191                                         case SG_SHARED:
 192                                                 up->psstate = "Pageout";
 193                                                 pageout(p, s);
 194                                                 if(ioptr != 0) {
 195                                                         up->psstate = "I/O";
 196                                                         executeio();
 197                                                 }
 198                                                 break;
 199                                         }
 200                                 }
 201                         }
 202                         qunlock(&p->seglock);
 203                 } else {
 204                         killbig("out of memory");
 205                         freebroken();           /* can use the memory */
 206                         sched();
 207                 }
 208         }
 209         goto loop;
 210 }
 211
 212 static void
 213 pageout(Proc *p, Segment *s)
 214 {
 215         int type, i, size;
 216         ulong age;
 217         Pte *l;
 218         Page **pg, *entry;
 219
 220         if(!canqlock(&s->lk))   /* We cannot afford to wait, we will surely deadlock */
 221                 return;
 222
 223         if(s->steal) {          /* Protected by /dev/proc */
 224                 qunlock(&s->lk);
 225                 return;
 226         }
 227
 228         if(!canflush(p, s)) {   /* Able to invalidate all tlbs with references */
 229                 qunlock(&s->lk);
 230                 putseg(s);
 231                 return;
 232         }
 233
 234         if(waserror()) {
 235                 qunlock(&s->lk);
 236                 putseg(s);
 237                 return;
 238         }
 239
 240         /* Pass through the pte tables looking for memory pages to swap out */
 241         type = s->type&SG_TYPE;
 242         size = s->mapsize;
 243         for(i = 0; i < size; i++) {
 244                 l = s->map[i];
 245                 if(l == 0)
 246                         continue;
 247                 for(pg = l->first; pg < l->last; pg++) {
 248                         entry = *pg;
 249                         if(pagedout(entry))
 250                                 continue;
 251
 252                         if(entry->modref & PG_REF) {
 253                                 entry->modref &= ~PG_REF;
 254                                 entry->gen = genclock;
 255                         }
 256
 257                         if(genclock < entry->gen)
 258                                 age = ~(entry->gen - genclock);
 259                         else
 260                                 age = genclock - entry->gen;
 261                         gensum += age;
 262                         gencount++;
 263                         if(age <= genage)
 264                                 continue;
 265
 266                         pagepte(type, pg);
 267
 268                         if(ioptr >= conf.nswppo)
 269                                 goto out;
 270                 }
 271         }
 272 out:
 273         poperror();
 274         qunlock(&s->lk);
 275         putseg(s);
 276 }
 277
 278 static int
 279 canflush(Proc *p, Segment *s)
 280 {
 281         int i;
 282         Proc *ep;
 283
 284         lock(s);
 285         if(s->ref == 1) {               /* Easy if we are the only user */
 286                 s->ref++;
 287                 unlock(s);
 288                 return canpage(p);
 289         }
 290         s->ref++;
 291         unlock(s);
 292
 293         /* Now we must do hardwork to ensure all processes which have tlb
 294          * entries for this segment will be flushed if we succeed in paging it out
 295          */
 296         p = proctab(0);
 297         ep = &p[conf.nproc];
 298         while(p < ep) {
 299                 if(p->state != Dead) {
 300                         for(i = 0; i < NSEG; i++)
 301                                 if(p->seg[i] == s)
 302                                         if(!canpage(p))
 303                                                 return 0;
 304                 }
 305                 p++;
 306         }
 307         return 1;
 308 }
 309
 310 static void
 311 pagepte(int type, Page **pg)
 312 {
 313         ulong daddr;
 314         Page *outp;
 315
 316         outp = *pg;
 317         switch(type) {
 318         case SG_TEXT:                           /* Revert to demand load */
 319                 putpage(outp);
 320                 *pg = 0;
 321                 break;
 322
 323         case SG_DATA:
 324         case SG_BSS:
 325         case SG_STACK:
 326         case SG_SHARED:
 327                 /*
 328                  *  get a new swap address and clear any pages
 329                  *  referring to it from the cache
 330                  */
 331                 daddr = newswap();
 332                 if(daddr == ~0)
 333                         break;
 334                 cachedel(&swapimage, daddr);
 335
 336                 lock(outp);
 337
 338                 /* forget anything that it used to cache */
 339                 uncachepage(outp);
 340
 341                 /*
 342                  *  incr the reference count to make sure it sticks around while
 343                  *  being written
 344                  */
 345                 outp->ref++;
 346
 347                 /*
 348                  *  enter it into the cache so that a fault happening
 349                  *  during the write will grab the page from the cache
 350                  *  rather than one partially written to the disk
 351                  */
 352                 outp->daddr = daddr;
 353                 cachepage(outp, &swapimage);
 354                 *pg = (Page*)(daddr|PG_ONSWAP);
 355                 unlock(outp);
 356
 357                 /* Add page to IO transaction list */
 358                 iolist[ioptr++] = outp;
 359                 break;
 360         }
 361 }
 362
 363 void
 364 pagersummary(void)
 365 {
 366         print("%lud/%lud memory %lud/%lud swap %d iolist\n",
 367                 palloc.user-palloc.freecount,
 368                 palloc.user, conf.nswap-swapalloc.free, conf.nswap,
 369                 ioptr);
 370 }
 371
 372 static int
 373 pageiocomp(void *a, void *b)
 374 {
 375         Page *p1, *p2;
 376
 377         p1 = *(Page **)a;
 378         p2 = *(Page **)b;
 379         if(p1->daddr > p2->daddr)
 380                 return 1;
 381         else
 382                 return -1;
 383 }
 384
 385 static void
 386 executeio(void)
 387 {
 388         Page *out;
 389         int i, n;
 390         Chan *c;
 391         char *kaddr;
 392         KMap *k;
 393
 394         c = swapimage.c;
 395         qsort(iolist, ioptr, sizeof iolist[0], pageiocomp);
 396         for(i = 0; i < ioptr; i++) {
 397                 if(ioptr > conf.nswppo)
 398                         panic("executeio: ioptr %d > %d", ioptr, conf.nswppo);
 399                 out = iolist[i];
 400                 k = kmap(out);
 401                 kaddr = (char*)VA(k);
 402
 403                 if(waserror())
 404                         panic("executeio: page out I/O error");
 405
 406                 n = devtab[c->type]->write(c, kaddr, BY2PG, out->daddr);
 407                 if(n != BY2PG)
 408                         nexterror();
 409
 410                 kunmap(k);
 411                 poperror();
 412
 413                 /* Free up the page after I/O */
 414                 lock(out);
 415                 out->ref--;
 416                 unlock(out);
 417                 putpage(out);
 418         }
 419         ioptr = 0;
 420 }
 421
 422 static int
 423 needpages(void*)
 424 {
 425         return palloc.freecount < swapalloc.headroom;
 426 }
 427
 428 void
 429 setswapchan(Chan *c)
 430 {
 431         uchar dirbuf[sizeof(Dir)+100];
 432         Dir d;
 433         int n;
 434
 435         if(swapimage.c) {
 436                 if(swapalloc.free != conf.nswap){
 437                         cclose(c);
 438                         error(Einuse);
 439                 }
 440                 cclose(swapimage.c);
 441                 swapimage.c = nil;
 442         }
 443
 444         /*
 445          *  if this isn't a file, set the swap space
 446          *  to be at most the size of the partition
 447          */
 448         if(devtab[c->type]->dc != L'M'){
 449                 n = devtab[c->type]->stat(c, dirbuf, sizeof dirbuf);
 450                 if(n <= 0){
 451                         cclose(c);
 452                         error("stat failed in setswapchan");
 453                 }
 454                 convM2D(dirbuf, n, &d, nil);
 455                 if(d.length < conf.nswap*BY2PG){
 456                         conf.nswap = d.length/BY2PG;
 457                         swapalloc.top = &swapalloc.swmap[conf.nswap];
 458                         swapalloc.free = conf.nswap;
 459                 }
 460         }
 461         c->flag &= ~CCACHE;
 462         swapimage.c = c;
 463 }
 464
 465 int
 466 swapfull(void)
 467 {
 468         return swapalloc.free < conf.nswap/10;
 469 }