Skip to content

Commit

Permalink
hal/armv7a: fix asid implementation
Browse files Browse the repository at this point in the history 
 - TLB is not validated after each pmap_swith
 - asids alloc and dealloc functions ivalidate dirty asids

PD-316
  • Loading branch information
HBuczynski committed Dec 2, 2022
1 parent 81bf68d commit b23bc41
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 63 deletions.
14 changes: 13 additions & 1 deletion hal/armv7a/_armv7a.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,23 @@ hal_cpuInvalASID:
.ltorg


.globl hal_cpuInvalVAASID
.type hal_cpuInvalVAASID, %function
hal_cpuInvalVAASID:
dsb
mcr p15, 0, r0, c8, c7, 1 /* TLBIMVA */
dsb
isb
bx lr
.size hal_cpuInvalVAASID, .-hal_cpuInvalVAASID
.ltorg


.globl hal_cpuInvalVA
.type hal_cpuInvalVA, %function
hal_cpuInvalVA:
dsb
mcr p15, 0, r0, c8, c7, 1 /* UTLBIMVA */
mcr p15, 0, r0, c8, c7, 3 /* TLBIMVAA */
dsb
isb
bx lr
Expand Down
2 changes: 1 addition & 1 deletion hal/armv7a/arch/pmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ typedef struct _page_t {


typedef struct _pmap_t {
u8 asid_ix;
u8 asid;
u32 *pdir;
addr_t addr; /* physical address of pdir */
void *start;
Expand Down
3 changes: 3 additions & 0 deletions hal/armv7a/armv7a.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ extern void hal_cpuInvalASID(u8 asid);
extern void hal_cpuInvalVA(ptr_t vaddr);


extern void hal_cpuInvalVAASID(ptr_t vaddr);


/* Invalidate entire Unified TLB*/
extern void hal_cpuInvalTLB(void);

Expand Down
146 changes: 85 additions & 61 deletions hal/armv7a/pmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,21 +56,25 @@ extern unsigned int _etext;
#define PDIR_TYPE_L2TABLE 0x00001
#define PDIR_TYPE_INVALID 0x00000

#define ASID_CNT 256
#define ASID_INVALID 0


struct {
u32 kpdir[0x1000]; /* Has to be first in the structure */
u32 kptab[0x400];
u32 excptab[0x400];
u32 sptab[0x400];
u8 heap[SIZE_PAGE];
pmap_t *asid_map[256];
u8 asids[256];

pmap_t *asidMaps[ASID_CNT];
u8 asidCnt;

addr_t minAddr;
addr_t maxAddr;
u32 start;
u32 end;
spinlock_t lock;
u8 asidptr;
} __attribute__((aligned(0x4000))) pmap_common;


Expand Down Expand Up @@ -113,50 +117,46 @@ static const u16 attrMap[] = {
};


static void _pmap_asidAlloc(pmap_t *pmap)
static void _pmap_asidDealloc(pmap_t *pmap)
{
pmap_t *evicted = NULL;

while (!++pmap_common.asidptr || (evicted = pmap_common.asid_map[pmap_common.asidptr]) != NULL) {
if (evicted != NULL) {
if ((hal_cpuGetContextId() & 0xff) == pmap_common.asids[evicted->asid_ix])
continue;
if (pmap->asid != ASID_INVALID) {
/* Remove all tags in TLB associated with current ASID */
hal_cpuInvalASID(pmap->asid);
hal_cpuBranchInval();
hal_cpuICacheInval();
hal_cpuDataSyncBarrier();
hal_cpuInstrBarrier();

evicted->asid_ix = 0;
break;
}
pmap_common.asidMaps[pmap->asid] = NULL;
pmap->asid = ASID_INVALID;
}

pmap_common.asid_map[pmap_common.asidptr] = pmap;
pmap->asid_ix = pmap_common.asidptr;

hal_cpuInvalASID(pmap_common.asids[pmap->asid_ix]);
hal_cpuDataSyncBarrier();
hal_cpuInstrBarrier();
}


static void _pmap_asidDealloc(pmap_t *pmap)
static void _pmap_asidAlloc(pmap_t *pmap)
{
pmap_t *last;
addr_t tmp;

if (pmap->asid_ix != 0) {
if (pmap->asid_ix != pmap_common.asidptr) {
pmap_common.asid_map[pmap->asid_ix] = last = pmap_common.asid_map[pmap_common.asidptr];
last->asid_ix = pmap->asid_ix;
tmp = pmap_common.asids[last->asid_ix];
pmap_common.asids[last->asid_ix] = pmap_common.asids[pmap_common.asidptr];
pmap_common.asids[pmap_common.asidptr] = tmp;
}
int i;
u8 newAsid = ++pmap_common.asidCnt;

pmap_common.asid_map[pmap_common.asidptr] = NULL;
/* When ASID's counter wraps around, the new set of ASID is prepared */
if (newAsid == ASID_INVALID) {
hal_cpuInvalTLB();
hal_cpuBranchInval();
hal_cpuICacheInval();

if (!--pmap_common.asidptr)
pmap_common.asidptr--;
newAsid = 1;
pmap_common.asidCnt = 1;

pmap->asid_ix = 0;
for (i = 0; i < ASID_CNT; i++) {
if (pmap_common.asidMaps[i] != NULL) {
pmap_common.asidMaps[i]->asid = ASID_INVALID;
pmap_common.asidMaps[i] = NULL;
}
}
}

pmap_common.asidMaps[newAsid] = pmap;
pmap->asid = newAsid;
}


Expand All @@ -165,13 +165,14 @@ int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr)
{
pmap->pdir = vaddr;
pmap->addr = p->addr;
pmap->asid_ix = 0;
pmap->asid = ASID_INVALID;

hal_memset(pmap->pdir, 0, (VADDR_KERNEL) >> 18);
hal_memcpy(&pmap->pdir[ID_PDIR(VADDR_KERNEL)], &kpmap->pdir[ID_PDIR(VADDR_KERNEL)], (VADDR_MAX - VADDR_KERNEL + 1) >> 18);

hal_cpuDataMemoryBarrier();
hal_cpuDataSyncBarrier();

return EOK;
}

Expand All @@ -183,8 +184,9 @@ addr_t pmap_destroy(pmap_t *pmap, int *i)
int max = ((VADDR_USR_MAX + SIZE_PAGE - 1) & ~(SIZE_PAGE - 1)) >> 20;

hal_spinlockSet(&pmap_common.lock, &sc);
if (pmap->asid_ix != 0)
if (pmap->asid != ASID_INVALID) {
_pmap_asidDealloc(pmap);
}
hal_spinlockClear(&pmap_common.lock, &sc);

while (*i < max) {
Expand All @@ -201,22 +203,20 @@ addr_t pmap_destroy(pmap_t *pmap, int *i)

void _pmap_switch(pmap_t *pmap)
{
if (pmap->asid_ix == 0)
if (pmap->asid == ASID_INVALID) {
_pmap_asidAlloc(pmap);
else if (hal_cpuGetTTBR0() == (pmap->addr | TTBR_CACHE_CONF))
}
else if (hal_cpuGetTTBR0() == ((u32)pmap->addr | TTBR_CACHE_CONF)) {
return;
}

/* Assign new user's page dir to TTBR0 register */
hal_cpuDataSyncBarrier();
hal_cpuSetContextId(0);
hal_cpuInstrBarrier();
hal_cpuSetTTBR0(pmap->addr | TTBR_CACHE_CONF);
hal_cpuInstrBarrier();
hal_cpuSetContextId((u32)pmap->pdir | pmap_common.asids[pmap->asid_ix]);

/* TODO: invalidate TLB only if asids pool is run out.
This code should be moved to _pmap_asidAlloc and _pmap_asidDealloc */
hal_cpuInvalTLB();
hal_cpuSetContextId((u32)pmap->pdir | pmap->asid);
hal_cpuInstrBarrier();

hal_cpuBranchInval();
hal_cpuICacheInval();
Expand All @@ -237,14 +237,21 @@ static void _pmap_writeEntry(ptr_t *ptable, void *va, addr_t pa, int attributes,
{
int pti = ID_PTABLE((ptr_t)va);

hal_cpuCleanDataCache((ptr_t)&ptable[pti], (ptr_t)&ptable[pti] + sizeof(ptr_t));
if (attributes & PGHD_PRESENT)
ptable[pti] = (pa & ~0xfff) | attrMap[attributes & 0x1f];
else
ptable[pti] = 0;

hal_cpuFlushDataCache((ptr_t)&ptable[pti], (ptr_t)&ptable[pti] + sizeof(ptr_t));
hal_cpuDataSyncBarrier();
hal_cpuInvalVA(((ptr_t)va & ~0xfff) | asid);

/* If entry is not a global, invalidate VA associated with specific address space */
if (asid != ASID_INVALID) {
hal_cpuInvalVAASID(((ptr_t)va & ~0xfff) | asid);
}
else {
hal_cpuInvalVA((ptr_t)va & ~0xfff);
}

hal_cpuBranchInval();
hal_cpuICacheInval();
Expand All @@ -253,26 +260,33 @@ static void _pmap_writeEntry(ptr_t *ptable, void *va, addr_t pa, int attributes,

static void _pmap_addTable(pmap_t *pmap, int pdi, addr_t pa)
{
pa = (pa & ~0xfff) | PDIR_TYPE_L2TABLE;
int i;

pa = (pa & ~0xfff) | PDIR_TYPE_L2TABLE;
pdi &= ~3;
hal_cpuFlushDataCache((ptr_t)&pmap->pdir[pdi], (ptr_t)&pmap->pdir[pdi] + 4 * sizeof(ptr_t));

/* L2 table contains 256 entries (0x400). PAGE_SIZE is equal 0x1000 so that four L2 tables are added. */
pmap->pdir[pdi] = pa;
pmap->pdir[pdi + 1] = pa + 0x400;
pmap->pdir[pdi + 2] = pa + 0x800;
pmap->pdir[pdi + 3] = pa + 0xc00;

hal_cpuInvalASID(pmap_common.asids[pmap->asid_ix]);
hal_cpuFlushDataCache((ptr_t)&pmap->pdir[pdi], (ptr_t)&pmap->pdir[pdi] + 4 * sizeof(ptr_t));

hal_cpuDataSyncBarrier();
hal_cpuInstrBarrier();
/* TODO: check if it is needed */
for (i = 0; i < 4; ++i) {
hal_cpuInvalVA((ptr_t)&pmap->pdir[pdi + i] & ~0xfff);
}

hal_cpuBranchInval();
hal_cpuICacheInval();
}


static void _pmap_mapScratch(addr_t pa, unsigned char asid)
{
hal_cpuFlushDataCache((ptr_t)pmap_common.sptab, (ptr_t)pmap_common.sptab + SIZE_PAGE);

_pmap_writeEntry(pmap_common.kptab, pmap_common.sptab, pa, PGHD_PRESENT | PGHD_READ | PGHD_WRITE, asid);
}

Expand All @@ -287,7 +301,7 @@ int pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *alloc)
pdi = ID_PDIR((ptr_t)va);

hal_spinlockSet(&pmap_common.lock, &sc);
asid = pmap_common.asids[pmap->asid_ix];
asid = pmap->asid;

/* If no page table is allocated add new one */
if (pmap->pdir[pdi] == PDIR_TYPE_INVALID) {
Expand Down Expand Up @@ -316,6 +330,13 @@ int pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *alloc)
return EOK;
}

if (attr & PGHD_USER) {
hal_cpuInvalVAASID(((ptr_t)va & ~0xfff) | asid);
}
else {
hal_cpuInvalVA((ptr_t)va & ~0xfff);
}

if (attr & PGHD_EXEC || attr & PGHD_NOT_CACHED || attr & PGHD_DEV) {
/* Invalidate cache for this pa to prevent corrupting it later when cache lines get evicted.
* First map it into our address space if necessary. */
Expand Down Expand Up @@ -358,14 +379,16 @@ int pmap_remove(pmap_t *pmap, void *vaddr)
}

/* Map page table corresponding to vaddr */
asid = pmap_common.asids[pmap->asid_ix];
asid = pmap->asid;
_pmap_mapScratch(addr, asid);

if (pmap_common.sptab[pti] == 0) {
hal_spinlockClear(&pmap_common.lock, &sc);
return EOK;
}

hal_cpuFlushDataCache((ptr_t)vaddr, (ptr_t)vaddr + SIZE_PAGE);

_pmap_writeEntry(pmap_common.sptab, vaddr, 0, 0, asid);

hal_spinlockClear(&pmap_common.lock, &sc);
Expand All @@ -392,7 +415,7 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr)
return 0;
}

asid = pmap_common.asids[pmap->asid_ix];
asid = pmap->asid;
_pmap_mapScratch(addr, asid);
addr = pmap_common.sptab[pti];
hal_spinlockClear(&pmap_common.lock, &sc);
Expand Down Expand Up @@ -540,14 +563,14 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend)
int i;
void *v;

pmap_common.asidptr = 0;
pmap->asid_ix = 0;
pmap->asid = 0;

for (i = 0; i < sizeof(pmap_common.asid_map) / sizeof(pmap_common.asid_map[0]); ++i) {
pmap_common.asid_map[i] = NULL;
pmap_common.asids[i] = i;
for (i = 0; i < ASID_CNT; ++i) {
pmap_common.asidMaps[i] = NULL;
}

pmap_common.asidCnt = 0;

hal_spinlockCreate(&pmap_common.lock, "pmap_common.lock");

pmap_common.minAddr = ADDR_DDR;
Expand Down Expand Up @@ -583,3 +606,4 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend)
for (v = *vend; v < (void *)VADDR_KERNEL + (4 * 1024 * 1024); v += SIZE_PAGE)
pmap_remove(pmap, v);
}

0 comments on commit b23bc41

Please sign in to comment.