#include "fns.h"
#include "../port/error.h"
-#define pghash(daddr) palloc.hash[(daddr>>PGSHIFT)&(PGHSIZE-1)]
-
-struct Palloc palloc;
+Palloc palloc;
void
pageinit(void)
int color, i, j;
Page *p;
Pallocmem *pm;
- ulong m, np, k, vkb, pkb;
+ vlong m, v, u;
- np = 0;
- for(i=0; i<nelem(palloc.mem); i++){
- pm = &palloc.mem[i];
- np += pm->npage;
+ if(palloc.pages == nil){
+ ulong np;
+
+ np = 0;
+ for(i=0; i<nelem(palloc.mem); i++){
+ pm = &palloc.mem[i];
+ np += pm->npage;
+ }
+ palloc.pages = xalloc(np*sizeof(Page));
+ if(palloc.pages == nil)
+ panic("pageinit");
}
- palloc.pages = xalloc(np*sizeof(Page));
- if(palloc.pages == 0)
- panic("pageinit");
color = 0;
- palloc.head = palloc.pages;
- p = palloc.head;
+ palloc.head = nil;
+ p = palloc.pages;
for(i=0; i<nelem(palloc.mem); i++){
pm = &palloc.mem[i];
for(j=0; j<pm->npage; j++){
- p->prev = p-1;
- p->next = p+1;
+ memset(p, 0, sizeof *p);
p->pa = pm->base+j*BY2PG;
+ if(cankaddr(p->pa) && (KADDR(p->pa) == nil || KADDR(p->pa) == (void*)-BY2PG))
+ continue;
p->color = color;
- palloc.freecount++;
color = (color+1)%NCOLOR;
+ pagechainhead(p);
p++;
}
}
- palloc.tail = p - 1;
- palloc.head->prev = 0;
- palloc.tail->next = 0;
palloc.user = p - palloc.pages;
- pkb = palloc.user*BY2PG/1024;
- vkb = pkb + (conf.nswap*BY2PG)/1024;
+ u = palloc.user*BY2PG;
+ v = u + conf.nswap*BY2PG;
/* Paging numbers */
swapalloc.highwater = (palloc.user*5)/100;
for(i=0; i<nelem(conf.mem); i++)
if(conf.mem[i].npage)
m += conf.mem[i].npage*BY2PG;
- k = PGROUND(end - (char*)KTZERO);
- print("%ldM memory: ", (m+k+1024*1024-1)/(1024*1024));
- print("%ldM kernel data, ", (m+k-pkb*1024+1024*1024-1)/(1024*1024));
- print("%ldM user, ", pkb/1024);
- print("%ldM swap\n", vkb/1024);
-}
+ m += PGROUND(end - (char*)KTZERO);
-static void
-pageunchain(Page *p)
-{
- if(canlock(&palloc))
- panic("pageunchain (palloc %p)", &palloc);
- if(p->prev)
- p->prev->next = p->next;
- else
- palloc.head = p->next;
- if(p->next)
- p->next->prev = p->prev;
- else
- palloc.tail = p->prev;
- p->prev = p->next = nil;
- palloc.freecount--;
+ print("%lldM memory: ", (m+1024*1024-1)/(1024*1024));
+ print("%lldM kernel data, ", (m-u+1024*1024-1)/(1024*1024));
+ print("%lldM user, ", u/(1024*1024));
+ print("%lldM swap\n", v/(1024*1024));
}
void
-pagechaintail(Page *p)
+pagechainhead(Page *p)
{
- if(canlock(&palloc))
- panic("pagechaintail");
- if(palloc.tail) {
- p->prev = palloc.tail;
- palloc.tail->next = p;
- }
- else {
- palloc.head = p;
- p->prev = 0;
- }
- palloc.tail = p;
- p->next = 0;
+ p->next = palloc.head;
+ palloc.head = p;
palloc.freecount++;
}
void
-pagechainhead(Page *p)
+pagechaindone(void)
{
- if(canlock(&palloc))
- panic("pagechainhead");
- if(palloc.head) {
- p->next = palloc.head;
- palloc.head->prev = p;
+ if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
+ return;
+ if(palloc.pwait[1].p != nil)
+ wakeup(&palloc.pwait[1]);
+}
+
+static void
+freepages(Page *head, Page *tail, ulong np)
+{
+ lock(&palloc);
+ tail->next = palloc.head;
+ palloc.head = head;
+ palloc.freecount += np;
+ pagechaindone();
+ unlock(&palloc);
+}
+
+ulong
+pagereclaim(Image *i, ulong pages)
+{
+ Page **h, **l, **x, *p;
+ Page *fh, *ft;
+ ulong np;
+
+ if(pages == 0)
+ return 0;
+
+ lock(i);
+ if(i->pgref == 0){
+ unlock(i);
+ return 0;
}
- else {
- palloc.tail = p;
- p->next = 0;
+ incref(i);
+
+ np = 0;
+ fh = ft = nil;
+ for(h = i->pghash; h < &i->pghash[PGHSIZE]; h++){
+ l = h;
+ x = nil;
+ for(p = *l; p != nil; p = p->next){
+ if(p->ref == 0)
+ x = l;
+ l = &p->next;
+ }
+ if(x == nil)
+ continue;
+
+ p = *x;
+ *x = p->next;
+ p->next = nil;
+ p->image = nil;
+ p->daddr = ~0;
+ i->pgref--;
+ decref(i);
+
+ if(fh == nil)
+ fh = p;
+ else
+ ft->next = p;
+ ft = p;
+ if(++np >= pages)
+ break;
}
- palloc.head = p;
- p->prev = 0;
- palloc.freecount++;
+ unlock(i);
+ putimage(i);
+
+ if(np > 0)
+ freepages(fh, ft, np);
+
+ return np;
+}
+
+static int
+ispages(void*)
+{
+ return palloc.freecount > swapalloc.highwater || up->noswap && palloc.freecount > 0;
}
Page*
-newpage(int clear, Segment **s, ulong va)
+newpage(int clear, Segment **s, uintptr va)
{
- Page *p;
+ Page *p, **l;
KMap *k;
- uchar ct;
- int i, hw, color;
+ int color;
lock(&palloc);
- color = getpgcolor(va);
- hw = swapalloc.highwater;
- for(;;) {
- if(palloc.freecount > hw)
- break;
- if(up->kp && palloc.freecount > 0)
- break;
-
+ while(!ispages(nil)){
unlock(&palloc);
- if(s)
- qunlock(&((*s)->lk));
+ if(s != nil)
+ qunlock(*s);
if(!waserror()){
- eqlock(&palloc.pwait); /* Hold memory requesters here */
+ Rendezq *q;
+ q = &palloc.pwait[!up->noswap];
+ eqlock(q);
if(!waserror()){
kickpager();
- tsleep(&palloc.r, ispages, 0, 1000);
+ sleep(q, ispages, nil);
poperror();
}
-
- qunlock(&palloc.pwait);
-
+ qunlock(q);
poperror();
}
* a page. Fault will call newpage again when it has
* reacquired the segment locks
*/
- if(s){
- *s = 0;
- return 0;
+ if(s != nil){
+ *s = nil;
+ return nil;
}
-
lock(&palloc);
}
/* First try for our colour */
- for(p = palloc.head; p; p = p->next)
+ color = getpgcolor(va);
+ l = &palloc.head;
+ for(p = *l; p != nil; p = p->next){
if(p->color == color)
break;
-
- ct = PG_NOFLUSH;
- if(p == 0) {
- p = palloc.head;
- p->color = color;
- ct = PG_NEWCOL;
+ l = &p->next;
}
- pageunchain(p);
+ if(p == nil) {
+ l = &palloc.head;
+ p = *l;
+ }
- lock(p);
- if(p->ref != 0)
- panic("newpage: p->ref %d != 0", p->ref);
+ *l = p->next;
+ p->next = nil;
+ palloc.freecount--;
+ unlock(&palloc);
- uncachepage(p);
- p->ref++;
+ p->ref = 1;
p->va = va;
p->modref = 0;
- for(i = 0; i < MAXMACH; i++)
- p->cachectl[i] = ct;
- unlock(p);
- unlock(&palloc);
+ p->txtflush = 0;
if(clear) {
k = kmap(p);
return p;
}
-int
-ispages(void*)
-{
- return palloc.freecount >= swapalloc.highwater;
-}
-
void
putpage(Page *p)
{
putswap(p);
return;
}
-
- lock(&palloc);
- lock(p);
-
- if(p->ref == 0)
- panic("putpage");
-
- if(--p->ref > 0) {
- unlock(p);
- unlock(&palloc);
+ if(p->image != nil) {
+ decref(p);
return;
}
-
- if(p->image && p->image != &swapimage)
- pagechaintail(p);
- else
- pagechainhead(p);
-
- if(palloc.r.p != 0)
- wakeup(&palloc.r);
-
- unlock(p);
- unlock(&palloc);
-}
-
-Page*
-auxpage(void)
-{
- Page *p;
-
- lock(&palloc);
- p = palloc.head;
- if(palloc.freecount < swapalloc.highwater) {
- unlock(&palloc);
- return 0;
- }
- pageunchain(p);
-
- lock(p);
- if(p->ref != 0)
- panic("auxpage");
- p->ref++;
- uncachepage(p);
- unlock(p);
- unlock(&palloc);
-
- return p;
-}
-
-static int dupretries = 15000;
-
-void
-duppage(Page *p) /* Always call with p locked */
-{
- Page *np;
- int color;
- int retries;
-
- retries = 0;
-retry:
- /* don't dup pages that are shared or have no image */
- if(p->ref != 1 || p->image == nil || p->image->notext)
- return;
-
- if(retries++ > dupretries){
- print("duppage %d, up %p\n", retries, up);
- dupretries += 100;
- if(dupretries > 100000)
- panic("duppage");
- uncachepage(p);
- return;
- }
-
- /*
- * normal lock ordering is to call
- * lock(&palloc) before lock(p).
- * To avoid deadlock, we have to drop
- * our locks and try again. as the page
- * is from the image cache, this might
- * let someone else come in and grab it
- * so we check page ref above.
- */
- if(!canlock(&palloc)){
- unlock(p);
- if(up)
- sched();
- lock(p);
- goto retry;
- }
-
- /* No freelist cache when memory is very low */
- if(palloc.freecount < swapalloc.highwater) {
- unlock(&palloc);
- uncachepage(p);
- return;
- }
-
- color = getpgcolor(p->va);
- for(np = palloc.head; np; np = np->next)
- if(np->color == color)
- break;
-
- /* No page of the correct color */
- if(np == 0) {
- unlock(&palloc);
- uncachepage(p);
- return;
- }
-
- pageunchain(np);
- pagechaintail(np);
-/*
-* XXX - here's a bug? - np is on the freelist but it's not really free.
-* when we unlock palloc someone else can come in, decide to
-* use np, and then try to lock it. they succeed after we've
-* run copypage and cachepage and unlock(np). then what?
-* they call pageunchain before locking(np), so it's removed
-* from the freelist, but still in the cache because of
-* cachepage below. if someone else looks in the cache
-* before they remove it, the page will have a nonzero ref
-* once they finally lock(np).
-*/
- lock(np);
- if(np->ref != 0) /* should never happen */
- panic("duppage: np->ref %d != 0", np->ref);
- unlock(&palloc);
-
- /* Cache the new version */
- uncachepage(np);
- np->va = p->va;
- np->daddr = p->daddr;
- copypage(p, np);
- cachepage(np, p->image);
- unlock(np);
- uncachepage(p);
+ if(decref(p) == 0)
+ freepages(p, p, 1);
}
void
}
void
-uncachepage(Page *p) /* Always called with a locked page */
+cachepage(Page *p, Image *i)
{
- Page **l, *f;
+ Page **h;
- if(p->image == 0)
- return;
-
- lock(&palloc.hashlock);
- l = &pghash(p->daddr);
- for(f = *l; f; f = f->hash) {
- if(f == p) {
- *l = p->hash;
- break;
- }
- l = &f->hash;
- }
- unlock(&palloc.hashlock);
- putimage(p->image);
- p->image = 0;
- p->daddr = 0;
+ lock(i);
+ p->image = i;
+ h = &PGHASH(i, p->daddr);
+ p->next = *h;
+ *h = p;
+ incref(i);
+ i->pgref++;
+ unlock(i);
}
void
-cachepage(Page *p, Image *i)
+uncachepage(Page *p)
{
- Page **l;
+ Page **l, *x;
+ Image *i;
- /* If this ever happens it should be fixed by calling
- * uncachepage instead of panic. I think there is a race
- * with pio in which this can happen. Calling uncachepage is
- * correct - I just wanted to see if we got here.
- */
- if(p->image)
- panic("cachepage");
+ i = p->image;
+ if(i == nil)
+ return;
- incref(i);
- lock(&palloc.hashlock);
- p->image = i;
- l = &pghash(p->daddr);
- p->hash = *l;
- *l = p;
- unlock(&palloc.hashlock);
+ lock(i);
+ if(p->image != i){
+ unlock(i);
+ return;
+ }
+ l = &PGHASH(i, p->daddr);
+ for(x = *l; x != nil; x = x->next) {
+ if(x == p){
+ *l = p->next;
+ p->next = nil;
+ p->image = nil;
+ p->daddr = ~0;
+ i->pgref--;
+ unlock(i);
+ putimage(i);
+ return;
+ }
+ l = &x->next;
+ }
+ unlock(i);
}
-void
-cachedel(Image *i, ulong daddr)
+Page*
+lookpage(Image *i, uintptr daddr)
{
- Page *f, **l;
-
- lock(&palloc.hashlock);
- l = &pghash(daddr);
- for(f = *l; f; f = f->hash) {
- if(f->image == i && f->daddr == daddr) {
- lock(f);
- if(f->image == i && f->daddr == daddr){
- *l = f->hash;
- putimage(f->image);
- f->image = 0;
- f->daddr = 0;
- }
- unlock(f);
- break;
+ Page *p, **h, **l;
+
+ lock(i);
+ l = h = &PGHASH(i, daddr);
+ for(p = *l; p != nil; p = p->next){
+ if(p->daddr == daddr){
+ *l = p->next;
+ p->next = *h;
+ *h = p;
+ incref(p);
+ unlock(i);
+ return p;
}
- l = &f->hash;
+ l = &p->next;
}
- unlock(&palloc.hashlock);
+ unlock(i);
+
+ return nil;
}
-Page *
-lookpage(Image *i, ulong daddr)
+void
+cachedel(Image *i, uintptr daddr)
{
- Page *f;
-
- lock(&palloc.hashlock);
- for(f = pghash(daddr); f; f = f->hash) {
- if(f->image == i && f->daddr == daddr) {
- unlock(&palloc.hashlock);
-
- lock(&palloc);
- lock(f);
- if(f->image != i || f->daddr != daddr) {
- unlock(f);
- unlock(&palloc);
- return 0;
- }
- if(++f->ref == 1)
- pageunchain(f);
- unlock(&palloc);
- unlock(f);
+ Page *p;
- return f;
- }
+ while((p = lookpage(i, daddr)) != nil){
+ uncachepage(p);
+ putpage(p);
}
- unlock(&palloc.hashlock);
-
- return 0;
}
+
Pte*
ptecpy(Pte *old)
{
dst = &new->pages[old->first-old->pages];
new->first = dst;
for(src = old->first; src <= old->last; src++, dst++)
- if(*src) {
+ if(*src != nil) {
if(onswap(*src))
dupswap(*src);
- else {
- lock(*src);
- (*src)->ref++;
- unlock(*src);
- }
+ else
+ incref(*src);
new->last = dst;
*dst = *src;
}
void
freepte(Segment *s, Pte *p)
{
- int ref;
- void (*fn)(Page*);
- Page *pt, **pg, **ptop;
+ Page **pg, **pe;
+
+ pg = p->first;
+ pe = p->last;
switch(s->type&SG_TYPE) {
case SG_PHYSICAL:
- fn = s->pseg->pgfree;
- ptop = &p->pages[PTEPERTAB];
- if(fn) {
- for(pg = p->pages; pg < ptop; pg++) {
- if(*pg == 0)
- continue;
- (*fn)(*pg);
- *pg = 0;
- }
- break;
- }
- for(pg = p->pages; pg < ptop; pg++) {
- pt = *pg;
- if(pt == 0)
- continue;
- lock(pt);
- ref = --pt->ref;
- unlock(pt);
- if(ref == 0)
- free(pt);
+ while(pg <= pe){
+ if(*pg != nil && decref(*pg) == 0)
+ free(*pg);
+ pg++;
}
break;
default:
- for(pg = p->first; pg <= p->last; pg++)
- if(*pg) {
+ while(pg <= pe){
+ if(*pg != nil)
putpage(*pg);
- *pg = 0;
- }
+ pg++;
+ }
}
free(p);
}
nwrong = 0;
for(i=0; i<np; i++){
if(palloc.pages[i].ref != ref[i]){
- iprint("page %#.8lux ref %d actual %lud\n",
+ iprint("page %#p ref %ld actual %lud\n",
palloc.pages[i].pa, palloc.pages[i].ref, ref[i]);
ref[i] = 1;
nwrong++;
p = proctab(i);
for(j=0; j<NSEG; j++){
s = p->seg[j];
- if(s)
+ if(s != nil)
s->mark = 0;
}
}
s = p->seg[j];
if(s == nil || s->mark++)
continue;
+ if((s->type&SG_TYPE) == SG_PHYSICAL)
+ continue;
ns++;
for(k=0; k<s->mapsize; k++){
pte = s->map[k];
continue;
if(print){
if(ref[pagenumber(entry)])
- iprint("page %#.8lux in segment %#p\n", entry->pa, s);
+ iprint("page %#p in segment %#p\n", entry->pa, s);
continue;
}
if(ref[pagenumber(entry)]++ == 0)
projects / plan9front.git / blobdiff