X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=sys%2Fsrc%2F9%2Fpc64%2Fmmu.c;h=78725b3e4791511c0c8042582582f4e1bb7663dd;hb=5f1b70f437ed4f598f9439e201f1f7644f5ce7b5;hp=a475d53e94f255d2924a2df795dd775b878e4a86;hpb=bfbc5ab1970bbf9307e03a42e69b0d55eb92f2ef;p=plan9front.git

diff --git a/sys/src/9/pc64/mmu.c b/sys/src/9/pc64/mmu.c
index a475d53e9..78725b3e4 100644
--- a/sys/src/9/pc64/mmu.c
+++ b/sys/src/9/pc64/mmu.c
@@ -23,24 +23,24 @@ Segdesc gdt[NGDT] =
 [UESEG]		EXECSEGM(3),		/* user code */
 };
 
-static int didmmuinit = 0;
-
 static struct {
 	Lock;
 	MMU	*free;
 
-	int	nshare;
-	int	nalloc;
-	int	nfree;
+	ulong	nalloc;
+	ulong	nfree;
 } mmupool;
 
-/* level */
 enum {
+	/* level */
 	PML4E	= 2,
 	PDPE	= 1,
 	PDE	= 0,
 
 	MAPBITS	= 8*sizeof(m->mmumap[0]),
+
+	/* PAT entry used for write combining */
+	PATWC	= 7,
 };
 
 static void
@@ -73,6 +73,8 @@ taskswitch(uintptr stack)
 	mmuflushtlb();
 }
 
+static void kernelro(void);
+
 void
 mmuinit(void)
 {
@@ -80,11 +82,12 @@ mmuinit(void)
 	vlong v;
 	int i;
 
-	didmmuinit = 1;
-
 	/* zap double map done by l.s */ 
-	m->pml4[0] = 0;
 	m->pml4[512] = 0;
+	m->pml4[0] = 0;
+
+	if(m->machno == 0)
+		kernelro();
 
 	m->tss = mallocz(sizeof(Tss), 1);
 	if(m->tss == nil)
@@ -118,23 +121,26 @@ mmuinit(void)
 	taskswitch((uintptr)m + MACHSIZE);
 	ltr(TSSSEL);
 
-	wrmsr(0xc0000100, 0ull);	/* 64 bit fsbase */
-	wrmsr(0xc0000101, (uvlong)&machp[m->machno]);	/* 64 bit gsbase */
-	wrmsr(0xc0000102, 0ull);	/* kernel gs base */
+	wrmsr(FSbase, 0ull);
+	wrmsr(GSbase, (uvlong)&machp[m->machno]);
+	wrmsr(KernelGSbase, 0ull);
 
 	/* enable syscall extension */
-	rdmsr(0xc0000080, &v);
+	rdmsr(Efer, &v);
 	v |= 1ull;
-	wrmsr(0xc0000080, v);
-
-	/* IA32_STAR */
-	wrmsr(0xc0000081, ((uvlong)UE32SEL << 48) | ((uvlong)KESEL << 32));
-
-	/* IA32_LSTAR */
-	wrmsr(0xc0000082, (uvlong)syscallentry);
-
-	/* SYSCALL flags mask */
-	wrmsr(0xc0000084, 0x200);
+	wrmsr(Efer, v);
+
+	wrmsr(Star, ((uvlong)UE32SEL << 48) | ((uvlong)KESEL << 32));
+	wrmsr(Lstar, (uvlong)syscallentry);
+	wrmsr(Sfmask, 0x200);
+
+	/* IA32_PAT write combining */
+	if((MACHP(0)->cpuiddx & Pat) != 0
+	&& rdmsr(0x277, &v) != -1){
+		v &= ~(255LL<<(PATWC*8));
+		v |= 1LL<<(PATWC*8);	/* WC */
+		wrmsr(0x277, v);
+	}
 }
 
 /*
@@ -144,7 +150,7 @@ mmuinit(void)
 void*
 kaddr(uintptr pa)
 {
-	if(pa > (uintptr)-KZERO)
+	if(pa >= (uintptr)-KZERO)
 		panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
 	return (void*)(pa+KZERO);
 }
@@ -158,7 +164,7 @@ paddr(void *v)
 	if(va >= KZERO)
 		return va-KZERO;
 	if(va >= VMAP)
-		return va-(VMAP-(-KZERO));
+		return va-VMAP;
 	panic("paddr: va=%#p pc=%#p", va, getcallerpc(&v));
 	return 0;
 }
@@ -199,8 +205,6 @@ mmualloc(void)
 			mmupool.free = p->next;
 			mmupool.nalloc += n;
 			mmupool.nfree += n-1;
-
-			mmupool.nshare = mmupool.nalloc / conf.nmach;
 		}
 		unlock(&mmupool);
 	}
@@ -208,12 +212,56 @@ mmualloc(void)
 	return p;
 }
 
+static uintptr*
+mmucreate(uintptr *table, uintptr va, int level, int index)
+{
+	uintptr *page, flags;
+	MMU *p;
+	
+	flags = PTEWRITE|PTEVALID;
+	if(va < VMAP){
+		assert(up != nil);
+		assert((va < USTKTOP) || (va >= KMAP && va < KMAP+KMAPSIZE));
+
+		p = mmualloc();
+		p->index = index;
+		p->level = level;
+		if(va < USTKTOP){
+			flags |= PTEUSER;
+			if(level == PML4E){
+				if((p->next = up->mmuhead) == nil)
+					up->mmutail = p;
+				up->mmuhead = p;
+				m->mmumap[index/MAPBITS] |= 1ull<<(index%MAPBITS);
+			} else {
+				up->mmutail->next = p;
+				up->mmutail = p;
+			}
+			up->mmucount++;
+		} else {
+			if(level == PML4E){
+				up->kmaptail = p;
+				up->kmaphead = p;
+			} else {
+				up->kmaptail->next = p;
+				up->kmaptail = p;
+			}
+			up->kmapcount++;
+		}
+		page = p->page;
+	} else {
+		page = rampage();
+	}
+	memset(page, 0, PTSZ);
+	table[index] = PADDR(page) | flags;
+	return page;
+}
+
 uintptr*
 mmuwalk(uintptr* table, uintptr va, int level, int create)
 {
-	uintptr pte, *page;
+	uintptr pte;
 	int i, x;
-	MMU *p;
 
 	x = PTLX(va, 3);
 	for(i = 2; i >= level; i--){
@@ -221,50 +269,15 @@ mmuwalk(uintptr* table, uintptr va, int level, int create)
 		if(pte & PTEVALID){
 			if(pte & PTESIZE)
 				return 0;
-			table = KADDR(PPN(pte));
-		} else {			
+			pte = PPN(pte);
+			if(pte >= (uintptr)-KZERO)
+				table = (void*)(pte + VMAP);
+			else
+				table = (void*)(pte + KZERO);
+		} else {
 			if(!create)
 				return 0;
-			pte = PTEWRITE|PTEVALID;
-			if(va < VMAP){
-				if(va < TSTKTOP){
-					pte |= PTEUSER;
-
-					p = mmualloc();
-					p->index = x;
-					p->level = i;
-					if(i == PML4E){
-						if((p->next = up->mmuhead) == nil)
-							up->mmutail = p;
-						up->mmuhead = p;
-						m->mmumap[p->index/MAPBITS] |= 1ull<<(p->index%MAPBITS);
-					} else {
-						up->mmutail->next = p;
-						up->mmutail = p;
-					}
-					up->mmucount++;
-				} else if(va >= KMAP && va < (KMAP+KMAPSIZE)) {
-					p = mmualloc();
-					p->index = x;
-					p->level = i;
-					if(i == PML4E){
-						up->kmaptail = p;
-						up->kmaphead = p;
-					} else {
-						up->kmaptail->next = p;
-						up->kmaptail = p;
-					}
-					up->kmapcount++;
-				} else
-					return 0;
-				page = p->page;
-			} else if(didmmuinit) {
-				page = mallocalign(PTSZ, BY2PG, 0, 0);
-			} else
-				page = rampage();
-			memset(page, 0, PTSZ);
-			table[x] = PADDR(page) | pte;
-			table = page;
+			table = mmucreate(table, va, i, x);
 		}
 		x = PTLX(va, i);
 	}
@@ -277,14 +290,67 @@ ptecount(uintptr va, int level)
 	return (1<<PTSHIFT) - (va & PGLSZ(level+1)-1) / PGLSZ(level);
 }
 
+static void
+ptesplit(uintptr* table, uintptr va)
+{
+	uintptr *pte, pa, off;
+
+	pte = mmuwalk(table, va, 1, 0);
+	if(pte == nil || (*pte & PTESIZE) == 0 || (va & PGLSZ(1)-1) == 0)
+		return;
+	table = rampage();
+	if(table == nil)
+		panic("ptesplit: out of memory\n");
+	va &= -PGLSZ(1);
+	pa = *pte & ~PTESIZE;
+	for(off = 0; off < PGLSZ(1); off += PGLSZ(0))
+		table[PTLX(va + off, 0)] = pa + off;
+	*pte = PADDR(table) | PTEVALID|PTEWRITE;
+	invlpg(va);
+}
+
+/*
+ * map kernel text segment readonly
+ * and everything else no-execute.
+ */
+static void
+kernelro(void)
+{
+	uintptr *pte, psz, va;
+
+	ptesplit(m->pml4, APBOOTSTRAP);
+	ptesplit(m->pml4, KTZERO);
+	ptesplit(m->pml4, (uintptr)etext-1);
+
+	for(va = KZERO; va != 0; va += psz){
+		psz = PGLSZ(0);
+		pte = mmuwalk(m->pml4, va, 0, 0);
+		if(pte == nil){
+			if(va & PGLSZ(1)-1)
+				continue;
+			pte = mmuwalk(m->pml4, va, 1, 0);
+			if(pte == nil)
+				continue;
+			psz = PGLSZ(1);
+		}
+		if((*pte & PTEVALID) == 0)
+			continue;
+		if(va >= KTZERO && va < (uintptr)etext)
+			*pte &= ~PTEWRITE;
+		else if(va != (APBOOTSTRAP & -BY2PG))
+			*pte |= PTENOEXEC;
+		invlpg(va);
+	}
+}
+
 void
-pmap(uintptr *pml4, uintptr pa, uintptr va, int size)
+pmap(uintptr pa, uintptr va, vlong size)
 {
 	uintptr *pte, *ptee, flags;
 	int z, l;
 
-	if((size <= 0) || va < VMAP)
-		panic("pmap: pa=%#p va=%#p size=%d", pa, va, size);
+	if(size <= 0 || va < VMAP)
+		panic("pmap: pa=%#p va=%#p size=%lld", pa, va, size);
 	flags = pa;
 	pa = PPN(pa);
 	flags -= pa;
@@ -295,9 +361,9 @@ pmap(uintptr *pml4, uintptr pa, uintptr va, int size)
 			flags |= PTESIZE;
 		l = (flags & PTESIZE) != 0;
 		z = PGLSZ(l);
-		pte = mmuwalk(pml4, va, l, 1);
-		if(pte == 0){
-			pte = mmuwalk(pml4, va, ++l, 0);
+		pte = mmuwalk(m->pml4, va, l, 1);
+		if(pte == nil){
+			pte = mmuwalk(m->pml4, va, ++l, 0);
 			if(pte && (*pte & PTESIZE)){
 				flags |= PTESIZE;
 				z = va & (PGLSZ(l)-1);
@@ -306,7 +372,7 @@ pmap(uintptr *pml4, uintptr pa, uintptr va, int size)
 				size += z;
 				continue;
 			}
-			panic("pmap: pa=%#p va=%#p size=%d", pa, va, size);
+			panic("pmap: pa=%#p va=%#p size=%lld", pa, va, size);
 		}
 		ptee = pte + ptecount(va, l);
 		while(size > 0 && pte < ptee){
@@ -318,6 +384,29 @@ pmap(uintptr *pml4, uintptr pa, uintptr va, int size)
 	}
 }
 
+void
+punmap(uintptr va, vlong size)
+{
+	uintptr *pte;
+	int l;
+
+	va = PPN(va);
+	while(size > 0){
+		if((va % PGLSZ(1)) != 0 || size < PGLSZ(1))
+			ptesplit(m->pml4, va);
+		l = 0;
+		pte = mmuwalk(m->pml4, va, l, 0);
+		if(pte == nil && (va % PGLSZ(1)) == 0 && size >= PGLSZ(1))
+			pte = mmuwalk(m->pml4, va, ++l, 0);
+		if(pte){
+			*pte = 0;
+			invlpg(va);
+		}
+		va += PGLSZ(l);
+		size -= PGLSZ(l);
+	}
+}
+
 static void
 mmuzap(void)
 {
@@ -330,22 +419,18 @@ mmuzap(void)
 
 	/* common case */
 	pte[PTLX(UTZERO, 3)] = 0;
-	pte[PTLX(TSTKTOP, 3)] = 0;
+	pte[PTLX(USTKTOP-1, 3)] = 0;
 	m->mmumap[PTLX(UTZERO, 3)/MAPBITS] &= ~(1ull<<(PTLX(UTZERO, 3)%MAPBITS));
-	m->mmumap[PTLX(TSTKTOP, 3)/MAPBITS] &= ~(1ull<<(PTLX(TSTKTOP, 3)%MAPBITS));
+	m->mmumap[PTLX(USTKTOP-1, 3)/MAPBITS] &= ~(1ull<<(PTLX(USTKTOP-1, 3)%MAPBITS));
 
 	for(i = 0; i < nelem(m->mmumap); pte += MAPBITS, i++){
-		w = m->mmumap[i];
-		if(w == 0)
+		if((w = m->mmumap[i]) == 0)
 			continue;
-		x = 0;
-		do {
+		m->mmumap[i] = 0;
+		for(x = 0; w != 0; w >>= 1, x++){
 			if(w & 1)
 				pte[x] = 0;
-			x++;
-			x >>= 1;
-		} while(w);
-		m->mmumap[i] = 0;
+		}
 	}
 }
 
@@ -357,7 +442,7 @@ mmufree(Proc *proc)
 	p = proc->mmutail;
 	if(p == nil)
 		return;
-	if(m->mmucount < mmupool.nshare){
+	if(m->mmucount+proc->mmucount < 256){
 		p->next = m->mmufree;
 		m->mmufree = proc->mmuhead;
 		m->mmucount += proc->mmucount;
@@ -411,7 +496,7 @@ mmurelease(Proc *proc)
 	if((p = proc->kmaptail) != nil){
 		if((p->next = proc->mmuhead) == nil)
 			proc->mmutail = p;
-		proc->mmuhead = p;
+		proc->mmuhead = proc->kmaphead;
 		proc->mmucount += proc->kmapcount;
 
 		proc->kmaphead = proc->kmaptail = nil;
@@ -432,16 +517,25 @@ putmmu(uintptr va, uintptr pa, Page *)
 	if(pte == 0)
 		panic("putmmu: bug: va=%#p pa=%#p", va, pa);
 	old = *pte;
-	*pte = pa | PTEVALID|PTEUSER;
+	*pte = pa | PTEUSER;
 	splx(x);
 	if(old & PTEVALID)
 		invlpg(va);
 }
 
+/*
+ * Double-check the user MMU.
+ * Error checking only.
+ */
 void
 checkmmu(uintptr va, uintptr pa)
 {
-	USED(va, pa);
+	uintptr *pte;
+
+	pte = mmuwalk(m->pml4, va, 0, 0);
+	if(pte != 0 && (*pte & PTEVALID) != 0 && PPN(*pte) != pa)
+		print("%ld %s: va=%#p pa=%#p pte=%#p\n",
+			up->pid, up->text, va, pa, *pte);
 }
 
 uintptr
@@ -452,12 +546,6 @@ cankaddr(uintptr pa)
 	return -KZERO - pa;
 }
 
-void
-countpagerefs(ulong *ref, int print)
-{
-	USED(ref, print);
-}
-
 KMap*
 kmap(Page *page)
 {
@@ -469,15 +557,13 @@ kmap(Page *page)
 		return (KMap*)KADDR(pa);
 
 	x = splhi();
-	va = KMAP + ((uintptr)up->kmapindex << PGSHIFT);
+	va = KMAP + (((uintptr)up->kmapindex++ << PGSHIFT) & (KMAPSIZE-1));
 	pte = mmuwalk(m->pml4, va, 0, 1);
-	if(pte == 0 || *pte & PTEVALID)
+	if(pte == 0 || (*pte & PTEVALID) != 0)
 		panic("kmap: pa=%#p va=%#p", pa, va);
-	*pte = pa | PTEWRITE|PTEVALID;
- 	up->kmapindex = (up->kmapindex + 1) % (1<<PTSHIFT);
-	if(up->kmapindex == 0)
-		mmuflushtlb();
+	*pte = pa | PTEWRITE|PTENOEXEC|PTEVALID;
 	splx(x);
+	invlpg(va);
 	return (KMap*)va;
 }
 
@@ -501,6 +587,9 @@ kunmap(KMap *k)
 
 /*
  * Add a device mapping to the vmap range.
+ * note that the VMAP and KZERO PDPs are shared
+ * between processors (see mpstartap) so no
+ * synchronization is being done.
  */
 void*
 vmap(uintptr pa, int size)
@@ -508,12 +597,9 @@ vmap(uintptr pa, int size)
 	uintptr va;
 	int o;
 
-	if(size <= 0 || pa >= -VMAP)
-		panic("vmap: pa=%#p size=%d pc=%#p", pa, size, getcallerpc(&pa));
-	if(cankaddr(pa) >= size)
-		va = pa+KZERO;
-	else
-		va = pa+(VMAP-(-KZERO));
+	if(pa+size > VMAPSIZE)
+		return 0;
+	va = pa+VMAP;
 	/*
 	 * might be asking for less than a page.
 	 */
@@ -521,7 +607,7 @@ vmap(uintptr pa, int size)
 	pa -= o;
 	va -= o;
 	size += o;
-	pmap(m->pml4, pa | PTEUNCACHED|PTEWRITE|PTEVALID, va, size);
+	pmap(pa | PTEUNCACHED|PTEWRITE|PTENOEXEC|PTEVALID, va, size);
 	return (void*)(va+o);
 }
 
@@ -532,28 +618,92 @@ vunmap(void *v, int)
 }
 
 /*
- * vmapsync() is currently unused as the VMAP and KZERO PDPs
- * are shared between processors. (see mpstartap)
+ * mark pages as write combining (used for framebuffer)
  */
-int
-vmapsync(uintptr va)
+void
+patwc(void *a, int n)
 {
-	uintptr *pte1, *pte2;
-	int level;
+	uintptr *pte, mask, attr, va;
+	int z, l;
+	vlong v;
 
-	if(va < VMAP || m->machno == 0)
-		return 0;
+	/* check if pat is usable */
+	if((MACHP(0)->cpuiddx & Pat) == 0
+	|| rdmsr(0x277, &v) == -1
+	|| ((v >> PATWC*8) & 7) != 1)
+		return;
+
+	/* set the bits for all pages in range */
+	for(va = (uintptr)a; n > 0; n -= z, va += z){
+		l = 0;
+		pte = mmuwalk(m->pml4, va, l, 0);
+		if(pte == 0)
+			pte = mmuwalk(m->pml4, va, ++l, 0);
+		if(pte == 0 || (*pte & PTEVALID) == 0)
+			panic("patwc: va=%#p", va);
+		z = PGLSZ(l);
+		z -= va & (z-1);
+		mask = l == 0 ? 3<<3 | 1<<7 : 3<<3 | 1<<12;
+		attr = (((PATWC&3)<<3) | ((PATWC&4)<<5) | ((PATWC&4)<<10));
+		*pte = (*pte & ~mask) | (attr & mask);
+	}
+}
 
-	for(level=0; level<2; level++){
-		pte1 = mmuwalk(MACHP(0)->pml4, va, level, 0);
-		if(pte1 && *pte1 & PTEVALID){
-			pte2 = mmuwalk(m->pml4, va, level, 1);
-			if(pte2 == 0)
-				break;
-			if(pte1 != pte2)
-				*pte2 = *pte1;
-			return 1;
+/*
+ * The palloc.pages array and mmupool can be a large chunk
+ * out of the 2GB window above KZERO, so we allocate from
+ * upages and map in the VMAP window before pageinit()
+ */
+void
+preallocpages(void)
+{
+	Pallocmem *pm;
+	uintptr va, base, top;
+	vlong tsize, psize;
+	ulong np, nt;
+	int i;
+
+	np = 0;
+	for(i=0; i<nelem(palloc.mem); i++){
+		pm = &palloc.mem[i];
+		np += pm->npage;
+	}
+	nt = np / 50;	/* 2% for mmupool */
+	np -= nt;
+
+	nt = (uvlong)nt*BY2PG / (sizeof(MMU)+PTSZ);
+	tsize = (uvlong)nt * (sizeof(MMU)+PTSZ);
+
+	psize = (uvlong)np * BY2PG;
+	psize += sizeof(Page) + BY2PG;
+	psize = (psize / (sizeof(Page)+BY2PG)) * sizeof(Page);
+
+	psize += tsize;
+	psize = ROUND(psize, PGLSZ(1));
+
+	for(i=0; i<nelem(palloc.mem); i++){
+		pm = &palloc.mem[i];
+		base = ROUND(pm->base, PGLSZ(1));
+		top = pm->base + (uvlong)pm->npage * BY2PG;
+		if((base + psize) <= VMAPSIZE && (vlong)(top - base) >= psize){
+			pm->base = base + psize;
+			pm->npage = (top - pm->base)/BY2PG;
+
+			va = base + VMAP;
+			pmap(base | PTEGLOBAL|PTEWRITE|PTENOEXEC|PTEVALID, va, psize);
+
+			palloc.pages = (void*)(va + tsize);
+
+			mmupool.nfree = mmupool.nalloc = nt;
+			mmupool.free = (void*)(va + (uvlong)nt*PTSZ);
+			for(i=0; i<nt; i++){
+				mmupool.free[i].page = (uintptr*)va;
+				mmupool.free[i].next = &mmupool.free[i+1];
+				va += PTSZ;
+			}
+			mmupool.free[i-1].next = nil;
+
+			break;
 		}
 	}
-	return 0;
 }