4 * we initially thought that we needn't flush the l2 cache since external
5 * devices needn't see page tables. sadly, reality does not agree with
8 * we use l1 and l2 cache ops here because they are empirically needed.
11 #include "../port/lib.h"
18 #define L1X(va) FEXT((va), 20, 12)
19 #define L2X(va) FEXT((va), 12, 8)
24 L1lo = UZERO/MiB, /* L1X(UZERO)? */
25 #ifdef SMALL_ARM /* well under 1GB of RAM? */
26 L1hi = (USTKTOP+MiB-1)/MiB, /* L1X(USTKTOP+MiB-1)? */
29 * on trimslice, top of 1GB ram can't be addressible, as high
30 * virtual memory (0xfff.....) contains high vectors. We
31 * moved USTKTOP down another MB to utterly avoid KADDR(stack_base)
32 * mapping to high exception vectors. USTKTOP is thus
33 * (0x40000000 - 64*KiB - MiB), which in kernel virtual space is
34 * (0x100000000ull - 64*KiB - MiB), but we need the whole user
35 * virtual address space to be unmapped in a new process.
41 #define ISHOLE(type) ((type) == 0)
43 typedef struct Range Range;
50 int type; /* L1 Section or Coarse? */
53 static void mmul1empty(void);
62 return "4KB-page table(s)";
64 return "1MB section(s)";
66 snprint(numb, sizeof numb, "type %d", type);
76 iprint("l1 maps va (%#8.8lux-%#llux) -> ", rp->startva, rp->endva-1);
77 if (rp->startva == rp->startpa)
78 iprint("identity-mapped");
80 iprint("pa %#8.8lux", rp->startpa);
88 if (attrs & L1sharable)
90 if (attrs & L1wralloc)
94 iprint(" %s\n", typename(rp->type));
100 l2dump(Range *rp, PTE pte)
105 /* dump level 1 page table at virtual addr l1 */
115 /* dump first level of ptes */
116 iprint("cpu%d l1 pt @ %#p:\n", m->machno, PADDR(l1));
117 memset(&rng, 0, sizeof rng);
118 for (va = i = 0; i < 4096; i++, va += MB) {
120 type = pte & (Section|Coarse);
122 pa = pte & ~(MB - 1);
124 pa = pte & ~(KiB - 1);
126 if (!ISHOLE(type) && type == Section)
127 attrs = pte & L1ptedramattrs;
129 /* if a range is open but this pte isn't part, close & open */
131 (pa != rng.endpa || type != rng.type || attrs != rng.attrs))
132 if (rng.endva != 0) { /* range is open? close it */
138 if (ISHOLE(type)) { /* end of any open range? */
139 if (rng.endva != 0) /* range is open? close it */
141 } else { /* continuation or new range */
142 if (rng.endva == 0) { /* no open range? start one */
148 rng.endva = va + MB; /* continue the open range */
154 if (rng.endva != 0) /* close any open range */
160 * map `mbs' megabytes from virt to phys, uncached.
161 * device registers are sharable, except the private memory region:
162 * 2 4K pages, at 0x50040000 on the tegra2.
165 mmumap(uintptr virt, uintptr phys, int mbs)
172 l1 = KADDR(ttbget());
173 for (off = 0; mbs-- > 0; off += MB)
174 l1[L1X(virt + off)] = (phys + off) | Dom0 | L1AP(Krw) |
175 Section | L1sharable;
176 allcache->wbse(l1, L1SIZE);
180 /* identity map `mbs' megabytes from phys */
182 mmuidmap(uintptr phys, int mbs)
184 mmumap(phys, phys, mbs);
192 if ((uintptr)l2pages >= HVECTORS - BY2PG)
200 * replace an L1 section pte with an L2 page table and an L1 coarse pte,
201 * with the same attributes as the original pte and covering the same
216 if (oldpte == Fault || (oldpte & (Coarse|Section)) != Section)
217 return; /* make idempotent */
219 /* wasteful - l2 pages only have 256 entries - fix */
221 * it may be very early, before any memory allocators are
222 * configured, so do a crude allocation from the top of memory.
225 memset(l2, 0, BY2PG);
227 /* write new L1 l2 entry back into L1 descriptors */
228 *l1 = PPN(PADDR(l2))|Dom0|Coarse;
230 /* fill l2 page with l2 ptes with equiv attrs; copy AP bits */
231 x = Small | oldpte & (Cached|Buffered) | (oldpte & (1<<15 | 3<<10)) >> 6;
232 if (oldpte & L1sharable)
234 if (oldpte & L1wralloc)
236 pa = oldpte & ~(MiB - 1);
237 for(tva = va; tva < va + MiB; tva += BY2PG, pa += BY2PG)
238 l2[L2X(tva)] = PPN(pa) | x;
240 /* force l2 page to memory */
241 allcache->wbse(l2, BY2PG);
243 /* clear out the current entry */
244 mmuinvalidateaddr(PPN(va));
246 allcache->wbinvse(l1, sizeof *l1);
247 if ((*l1 & (Coarse|Section)) != Coarse)
248 panic("explode %#p", va);
252 * cpu0's l1 page table has likely changed since we copied it in
253 * launchinit, notably to allocate uncached sections for ucalloc.
254 * so copy it again from cpu0's.
264 newl1 = mallocalign(L1SIZE, L1SIZE, 0, 0);
267 allcache->wbinvse((PTE *)L1, L1SIZE); /* get cpu0's up-to-date copy */
268 memmove(newl1, (PTE *)L1, L1SIZE);
269 allcache->wbse(newl1, L1SIZE);
274 ttbput(PADDR(newl1)); /* switch */
286 // mmudump(m->mmul1); /* DEBUG */
291 /* l1 is base of my l1 descriptor table */
293 l2pteaddr(PTE *l1, uintptr va)
301 if ((pte & (Coarse|Section)) != Coarse)
302 panic("l2pteaddr l1 pte %#8.8ux @ %#p not Coarse",
304 l2pa = pte & ~(KiB - 1);
305 l2 = (PTE *)KADDR(l2pa);
316 if (m->machno != 0) {
324 /* identity map most of the io space */
325 mmuidmap(PHYSIO, (PHYSIOEND - PHYSIO + MB - 1) / MB);
326 /* move the rest to more convenient addresses */
327 mmumap(VIRTNOR, PHYSNOR, 256); /* 0x40000000 v -> 0xd0000000 p */
328 mmumap(VIRTAHB, PHYSAHB, 256); /* 0xb0000000 v -> 0xc0000000 p */
330 /* map high vectors to start of dram, but only 4K, not 1MB */
331 pa -= MACHSIZE+BY2PG; /* page tables must be page aligned */
335 m->mmul1 = l1; /* used by explode in l2pteaddr */
337 /* map private mem region (8K at soc.scu) without sharable bits */
339 *l2pteaddr(l1, va) &= ~L2sharable;
341 *l2pteaddr(l1, va) &= ~L2sharable;
344 * below (and above!) the vectors in virtual space may be dram.
345 * populate the rest of l2 for the last MB.
347 for (va = -MiB; va != 0; va += BY2PG)
348 l2[L2X(va)] = PADDR(va) | L2AP(Krw) | Small | L2ptedramattrs;
349 /* map high vectors page to 0; must match attributes of KZERO->0 map */
350 l2[L2X(HVECTORS)] = PHYSDRAM | L2AP(Krw) | Small | L2ptedramattrs;
352 l1[L1X(HVECTORS)] = pa | Dom0 | Coarse; /* l1 -> ttb-machsize-4k */
354 /* make kernel text unwritable */
355 for(va = KTZERO; va < (ulong)etext; va += BY2PG)
356 *l2pteaddr(l1, va) |= L2apro;
365 // mmudump(l1); /* DEBUG */
369 mmul2empty(Proc* proc, int clear)
376 for(page = *l2; page != nil; page = page->next){
378 memset(UINT2PTR(page->va), 0, BY2PG);
379 l1[page->daddr] = Fault;
380 allcache->wbse(l1, sizeof *l1);
383 *l2 = proc->mmul2cache;
384 proc->mmul2cache = proc->mmul2;
392 /* there's a bug in here */
395 /* clean out any user mappings still in l1 */
396 if(m->mmul1lo > L1lo){
398 m->mmul1[L1lo] = Fault;
400 memset(&m->mmul1[L1lo], 0, m->mmul1lo*sizeof(PTE));
403 if(m->mmul1hi < L1hi){
404 l1 = &m->mmul1[m->mmul1hi];
405 if((L1hi - m->mmul1hi) == 1)
408 memset(l1, 0, (L1hi - m->mmul1hi)*sizeof(PTE));
412 memset(&m->mmul1[L1lo], 0, (L1hi - L1lo)*sizeof(PTE));
414 allcache->wbse(&m->mmul1[L1lo], (L1hi - L1lo)*sizeof(PTE));
418 mmuswitch(Proc* proc)
424 /* do kprocs get here and if so, do they need to? */
425 if(m->mmupid == proc->pid && !proc->newtlb)
427 m->mmupid = proc->pid;
429 /* write back dirty and invalidate caches */
439 /* move in new map */
441 for(page = proc->mmul2; page != nil; page = page->next){
443 l1[x] = PPN(page->pa)|Dom0|Coarse;
444 /* know here that L1lo < x < L1hi */
445 if(x+1 - m->mmul1lo < m->mmul1hi - x)
451 /* make sure map is in memory */
452 /* could be smarter about how much? */
453 allcache->wbse(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
455 /* lose any possible stale tlb entries */
458 //print("mmuswitch l1lo %d l1hi %d %d\n",
459 // m->mmul1lo, m->mmul1hi, proc->kp);
461 wakewfi(); /* in case there's another runnable proc */
476 mmurelease(Proc* proc)
480 /* write back dirty and invalidate caches */
484 for(page = proc->mmul2cache; page != nil; page = next){
487 panic("mmurelease: page->ref %d", page->ref);
490 if(proc->mmul2cache != nil)
492 proc->mmul2cache = nil;
496 /* make sure map is in memory */
497 /* could be smarter about how much? */
498 allcache->wbse(&m->mmul1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE));
500 /* lose any possible stale tlb entries */
505 putmmu(uintptr va, uintptr pa, Page* page)
514 iprint("putmmu(%#p, %#p, %#p) ", va, pa, page->pa);
515 iprint("mmul1 %#p l1 %#p *l1 %#ux x %d pid %ld\n",
516 m->mmul1, l1, *l1, x, up->pid);
518 panic("putmmu: old l1 pte non-zero; stuck?");
521 /* wasteful - l2 pages only have 256 entries - fix */
522 if(up->mmul2cache == nil){
523 /* auxpg since we don't need much? memset if so */
524 pg = newpage(1, 0, 0);
525 pg->va = VA(kmap(pg));
529 up->mmul2cache = pg->next;
530 memset(UINT2PTR(pg->va), 0, BY2PG);
533 pg->next = up->mmul2;
536 /* force l2 page to memory */
537 allcache->wbse((void *)pg->va, BY2PG);
539 *l1 = PPN(pg->pa)|Dom0|Coarse;
540 allcache->wbse(l1, sizeof *l1);
543 iprint("l1 %#p *l1 %#ux x %d pid %ld\n", l1, *l1, x, up->pid);
545 if(x >= m->mmul1lo && x < m->mmul1hi){
546 if(x+1 - m->mmul1lo < m->mmul1hi - x)
552 pte = UINT2PTR(KADDR(PPN(*l1)));
554 iprint("pte %#p index %ld was %#ux\n", pte, L2X(va), *(pte+L2X(va)));
556 panic("putmmu: old l2 pte non-zero; stuck?");
559 /* protection bits are
562 * PTEWRITE|PTEUNCACHED|PTEVALID;
565 if(!(pa & PTEUNCACHED))
571 pte[L2X(va)] = PPN(pa)|x;
572 allcache->wbse(&pte[L2X(va)], sizeof pte[0]);
574 /* clear out the current entry */
575 mmuinvalidateaddr(PPN(va));
577 /* write back dirty entries - we need this because the pio() in
578 * fault.c is writing via a different virt addr and won't clean
579 * its changes out of the dcache. Page coloring doesn't work
580 * on this mmu because the virtual cache is set associative
581 * rather than direct mapped.
585 if(page->txtflush & (1<<m->machno)){
587 page->txtflush &= ~(1<<m->machno);
590 iprint("putmmu %#p %#p %#p\n", va, pa, PPN(pa)|x);
594 mmuuncache(void* v, usize size)
601 * Simple helper for ucalloc().
602 * Uncache a Section, must already be
606 assert(!(va & (1*MiB-1)) && size == 1*MiB);
610 if((*pte & (Section|Coarse)) != Section)
612 *pte &= ~L1ptedramattrs;
614 mmuinvalidateaddr(va);
615 allcache->wbse(pte, 4);
621 mmukmap(uintptr va, uintptr pa, usize size)
629 assert(!(va & (1*MiB-1)) && !(pa & (1*MiB-1)) && size == 1*MiB);
635 *pte = pa|Dom0|L1AP(Krw)|Section;
636 mmuinvalidateaddr(va);
637 allcache->wbse(pte, 4);
643 mmukunmap(uintptr va, uintptr pa, usize size)
651 assert(!(va & (1*MiB-1)) && !(pa & (1*MiB-1)) && size == 1*MiB);
655 if(*pte != (pa|Dom0|L1AP(Krw)|Section))
658 mmuinvalidateaddr(va);
659 allcache->wbse(pte, 4);
665 * Return the number of bytes that can be accessed via KADDR(pa).
666 * If pa is not a valid argument to KADDR, return 0.
671 if((PHYSDRAM == 0 || pa >= PHYSDRAM) && pa < PHYSDRAM+memsize)
672 return PHYSDRAM+memsize - pa;
678 vmap(uintptr pa, usize size)
684 * XXX - replace with new vm stuff.
685 * Crock after crock - the first 4MB is mapped with 2MB pages
686 * so catch that and return good values because the current mmukmap
690 return UINT2PTR(kseg0|pa);
696 size = ROUNDUP(size, BY2PG);
699 pae = mmukmap(va, pa, size);
700 if(pae == 0 || pae-size != pa)
701 panic("vmap(%#p, %ld) called from %#p: mmukmap fails %#p",
702 pa+o, osize, getcallerpc(&pa), pae);
704 return UINT2PTR(va+o);
709 vunmap(void* v, usize size)
712 * XXX - replace with new vm stuff.
713 * Can't do this until do real vmap for all space that
714 * might be used, e.g. stuff below 1MB which is currently
715 * mapped automagically at boot but that isn't used (or
716 * at least shouldn't be used) by the kernel.
717 upafree(PADDR(v), size);
724 * Everything is in domain 0;
725 * domain 0 access bits in the DAC register are set
726 * to Client, which means access is controlled by the
727 * permission values set in the PTE.
729 * L1 access control for the kernel is set to 1 (RW,
730 * no user mode access);
731 * L2 access control for the kernel is set to 1 (ditto)
733 * L1 user mode access is never set;
734 * L2 access control for user mode is set to either
735 * 2 (RO) or 3 (RW) depending on whether text or data,
737 * (To get kernel RO set AP to 0 and S bit in control
739 * Coarse L1 page-tables are used. They have 256 entries
740 * and so consume 1024 bytes per table.
741 * Small L2 page-tables are used. They have 1024 entries
742 * and so consume 4096 bytes per table.
744 * 4KiB. That's the size of 1) a page, 2) the
745 * size allocated for an L2 page-table page (note only 1KiB
746 * is needed per L2 page - to be dealt with later) and
747 * 3) the size of the area in L1 needed to hold the PTEs
748 * to map 1GiB of user space (0 -> 0x3fffffff, 1024 entries).