From 9e41aa7759fc1741280d90cf6b5940ced69213e8 Mon Sep 17 00:00:00 2001 From: CTCaer Date: Thu, 14 Mar 2024 09:21:06 +0200 Subject: [PATCH] bdk: smmu: refactor and update driver - Allow ASID to be configured - Allow 34-bit PAs - Use special type for setting PDE/PTE config - Initialize all pages as non accessible - Add function for mapping 4MB regions directly - Add SMMU heap reset function - Correct address load OP to 32-bit and remove alignment on SMMU enable payload - Refactor all defines --- bdk/mem/smmu.c | 205 +++++++++++++++++++++++++++++++++---------------- bdk/mem/smmu.h | 63 ++++++++++----- bdk/sec/tsec.c | 27 +++---- 3 files changed, 197 insertions(+), 98 deletions(-) diff --git a/bdk/mem/smmu.c b/bdk/mem/smmu.c index 0128103..65b85aa 100644 --- a/bdk/mem/smmu.c +++ b/bdk/mem/smmu.c @@ -18,6 +18,7 @@ #include +#include #include #include #include @@ -25,27 +26,39 @@ #include #include -#define SMMU_PAGE_SHIFT 12 -#define SMMU_PAGE_SIZE (1 << SMMU_PAGE_SHIFT) +/*! SMMU register defines */ +#define SMMU_ASID(asid) (((asid) << 24u) | ((asid) << 16u) | ((asid) << 8u) | (asid)) +#define SMMU_ENABLE BIT(31) +#define SMMU_TLB_ACTIVE_LINES(l) ((l) << 0u) +#define SMMU_TLB_RR_ARBITRATION BIT(28) +#define SMMU_TLB_HIT_UNDER_MISS BIT(29) +#define SMMU_TLB_STATS_ENABLE BIT(31) +#define SMUU_PTC_INDEX_MAP(m) ((m) << 0u) +#define SMUU_PTC_LINE_MASK(m) ((m) << 8u) +#define SMUU_PTC_REQ_LIMIT(l) ((l) << 24u) +#define SMUU_PTC_CACHE_ENABLE BIT(29) +#define SMUU_PTC_STATS_ENABLE BIT(31) + +/*! Page table defines */ +#define SMMU_4MB_REGION 0 +#define SMMU_PAGE_TABLE 1 #define SMMU_PDIR_COUNT 1024 -#define SMMU_PDIR_SIZE (sizeof(u32) * SMMU_PDIR_COUNT) #define SMMU_PTBL_COUNT 1024 -#define SMMU_PTBL_SIZE (sizeof(u32) * SMMU_PTBL_COUNT) -#define SMMU_PDIR_SHIFT 12 -#define SMMU_PDE_SHIFT 12 -#define SMMU_PTE_SHIFT 12 -#define SMMU_PFN_MASK 0x000FFFFF -#define SMMU_ADDR_TO_PFN(addr) ((addr) >> 12) -#define SMMU_ADDR_TO_PDN(addr) ((addr) >> 22) -#define SMMU_PDN_TO_ADDR(addr) ((pdn) << 22) -#define SMMU_MK_PDIR(page, attr) (((page) >> SMMU_PDIR_SHIFT) | (attr)) -#define SMMU_MK_PDE(page, attr) (((page) >> SMMU_PDE_SHIFT) | (attr)) +#define SMMU_PAGE_SHIFT 12u +#define SMMU_PTN_SHIFT SMMU_PAGE_SHIFT +#define SMMU_PDN_SHIFT 22u +#define SMMU_ADDR_TO_PFN(addr) ((addr) >> SMMU_PAGE_SHIFT) +#define SMMU_ADDR_TO_PTN(addr) ((addr) >> SMMU_PTN_SHIFT) +#define SMMU_ADDR_TO_PDN(addr) ((addr) >> SMMU_PDN_SHIFT) +#define SMMU_PTN_TO_ADDR(ptn) ((ptn) << SMMU_PTN_SHIFT) +#define SMMU_PDN_TO_ADDR(pdn) ((pdn) << SMMU_PDN_SHIFT) +#define SMMU_PTB(page, attr) (((attr) << 29u) | ((page) >> SMMU_PAGE_SHIFT)) -u8 *_pageheap = (u8 *)SMMU_HEAP_ADDR; +static void *smmu_heap = (void *)SMMU_HEAP_ADDR; -// Enabling SMMU requires a TZ secure write: MC(MC_SMMU_CONFIG) = 1; -u8 smmu_payload[] __attribute__((aligned(16))) = { - 0xC1, 0x00, 0x00, 0x58, // 0x00: LDR X1, =0x70019010 +// Enabling SMMU requires a TZ (EL3) secure write. MC(MC_SMMU_CONFIG) = 1; +static const u8 smmu_enable_payload[] = { + 0xC1, 0x00, 0x00, 0x18, // 0x00: LDR W1, =0x70019010 0x20, 0x00, 0x80, 0xD2, // 0x04: MOV X0, #0x1 0x20, 0x00, 0x00, 0xB9, // 0x08: STR W0, [X1] 0x1F, 0x71, 0x08, 0xD5, // 0x0C: IC IALLUIS @@ -56,17 +69,22 @@ u8 smmu_payload[] __attribute__((aligned(16))) = { void *smmu_page_zalloc(u32 num) { - u8 *res = _pageheap; - _pageheap += SZ_PAGE * num; - memset(res, 0, SZ_PAGE * num); - return res; + void *page = smmu_heap; + memset(page, 0, SZ_PAGE * num); + + smmu_heap += SZ_PAGE * num; + + return page; } -static u32 *_smmu_pdir_alloc() +static pde_t *_smmu_pdir_alloc() { - u32 *pdir = (u32 *)smmu_page_zalloc(1); - for (int pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++) - pdir[pdn] = _PDE_VACANT(pdn); + pde_t *pdir = (pde_t *)smmu_page_zalloc(1); + + // Initialize pdes with no permissions. + for (u32 pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++) + pdir[pdn].huge.page = pdn; + return pdir; } @@ -77,9 +95,12 @@ static void _smmu_flush_regs() void smmu_flush_all() { + + // Flush the entire page table cache. MC(MC_SMMU_PTC_FLUSH) = 0; _smmu_flush_regs(); + // Flush the entire table. MC(MC_SMMU_TLB_FLUSH) = 0; _smmu_flush_regs(); } @@ -88,8 +109,8 @@ void smmu_init() { MC(MC_SMMU_PTB_ASID) = 0; MC(MC_SMMU_PTB_DATA) = 0; - MC(MC_SMMU_TLB_CONFIG) = 0x30000030; - MC(MC_SMMU_PTC_CONFIG) = 0x28000F3F; + MC(MC_SMMU_TLB_CONFIG) = SMMU_TLB_HIT_UNDER_MISS | SMMU_TLB_RR_ARBITRATION | SMMU_TLB_ACTIVE_LINES(48); + MC(MC_SMMU_PTC_CONFIG) = SMUU_PTC_CACHE_ENABLE | SMUU_PTC_REQ_LIMIT(8) | SMUU_PTC_LINE_MASK(0xF) | SMUU_PTC_INDEX_MAP(0x3F); MC(MC_SMMU_PTC_FLUSH) = 0; MC(MC_SMMU_TLB_FLUSH) = 0; } @@ -101,7 +122,8 @@ void smmu_enable() if (enabled) return; - ccplex_boot_cpu0((u32)smmu_payload, false); + // Launch payload on CCPLEX in order to set SMMU enable bit. + ccplex_boot_cpu0((u32)smmu_enable_payload, false); msleep(100); ccplex_powergate_cpu0(); @@ -110,63 +132,114 @@ void smmu_enable() enabled = true; } -u32 *smmu_init_domain4(u32 dev_base, u32 asid) +void smmu_reset_heap() { - u32 *pdir = _smmu_pdir_alloc(); - - MC(MC_SMMU_PTB_ASID) = asid; - MC(MC_SMMU_PTB_DATA) = SMMU_MK_PDIR((u32)pdir, _PDIR_ATTR); - _smmu_flush_regs(); - - MC(dev_base) = 0x80000000 | (asid << 24) | (asid << 16) | (asid << 8) | (asid); - _smmu_flush_regs(); - - return pdir; + smmu_heap = (void *)SMMU_HEAP_ADDR; } -u32 *smmu_get_pte(u32 *pdir, u32 iova) +void *smmu_init_domain(u32 dev_base, u32 asid) { - u32 ptn = SMMU_ADDR_TO_PFN(iova); - u32 pdn = SMMU_ADDR_TO_PDN(iova); - u32 *ptbl; + void *ptb = _smmu_pdir_alloc(); - if (pdir[pdn] != _PDE_VACANT(pdn)) - ptbl = (u32 *)((pdir[pdn] & SMMU_PFN_MASK) << SMMU_PDIR_SHIFT); + MC(MC_SMMU_PTB_ASID) = asid; + MC(MC_SMMU_PTB_DATA) = SMMU_PTB((u32)ptb, SMMU_ATTR_ALL); + _smmu_flush_regs(); + + // Use the same macro for both quad and single domains. Reserved bits are not set anyway. + MC(dev_base) = SMMU_ENABLE | SMMU_ASID(asid); + _smmu_flush_regs(); + + return ptb; +} + +void smmu_deinit_domain(u32 dev_base, u32 asid) +{ + MC(MC_SMMU_PTB_ASID) = asid; + MC(MC_SMMU_PTB_DATA) = 0; + MC(dev_base) = 0; + _smmu_flush_regs(); +} + +void smmu_domain_bypass(u32 dev_base, bool bypass) +{ + if (bypass) + { + smmu_flush_all(); + bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false); + MC(dev_base) &= ~SMMU_ENABLE; + } else { - ptbl = (u32 *)smmu_page_zalloc(1); + bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false); + MC(dev_base) |= SMMU_ENABLE; + smmu_flush_all(); + } + _smmu_flush_regs(); +} + +static pte_t *_smmu_get_pte(pde_t *pdir, u32 iova) +{ + u32 pdn = SMMU_ADDR_TO_PDN(iova); + pte_t *ptbl; + + // Get 4MB page table or initialize one. + if (pdir[pdn].tbl.attr) + ptbl = (pte_t *)(SMMU_PTN_TO_ADDR(pdir[pdn].tbl.table)); + else + { + // Allocate page table. + ptbl = (pte_t *)smmu_page_zalloc(1); + + // Get address. u32 addr = SMMU_PDN_TO_ADDR(pdn); - for (int pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SMMU_PAGE_SIZE) - ptbl[pn] = _PTE_VACANT(addr); - pdir[pdn] = SMMU_MK_PDE((u32)ptbl, _PDE_ATTR | _PDE_NEXT); + + // Initialize page table with no permissions. + for (u32 pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SZ_PAGE) + ptbl[pn].page = SMMU_ADDR_TO_PFN(addr); + + // Set page table to the page directory. + pdir[pdn].tbl.table = SMMU_ADDR_TO_PTN((u32)ptbl); + pdir[pdn].tbl.next = SMMU_PAGE_TABLE; + pdir[pdn].tbl.attr = SMMU_ATTR_ALL; + smmu_flush_all(); } - return &ptbl[ptn % SMMU_PTBL_COUNT]; + return &ptbl[SMMU_ADDR_TO_PTN(iova) % SMMU_PTBL_COUNT]; } -void smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr) +void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr) { - for (int i = 0; i < cnt; i++) + // Map pages to page table entries. VA/PA should be aligned to 4KB. + for (u32 i = 0; i < pages; i++) { - u32 *pte = smmu_get_pte(pdir, addr); - *pte = SMMU_ADDR_TO_PFN(page) | attr; - addr += SZ_PAGE; - page += SZ_PAGE; + pte_t *pte = _smmu_get_pte((pde_t *)ptb, iova); + + pte->page = SMMU_ADDR_TO_PFN(iopa); + pte->attr = attr; + + iova += SZ_PAGE; + iopa += SZ_PAGE; } + smmu_flush_all(); } -u32 *smmu_init_domain(u32 asid) +void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr) { - return smmu_init_domain4(asid, 1); -} + pde_t *pdir = (pde_t *)ptb; -void smmu_deinit_domain(u32 asid) -{ - MC(MC_SMMU_PTB_ASID) = 1; - MC(MC_SMMU_PTB_DATA) = 0; - MC(asid) = 0; - _smmu_flush_regs(); -} + // Map 4MB regions to page directory entries. VA/PA should be aligned to 4MB. + for (u32 i = 0; i < regions; i++) + { + u32 pdn = SMMU_ADDR_TO_PDN(iova); + pdir[pdn].huge.page = SMMU_ADDR_TO_PDN(iopa); + pdir[pdn].huge.next = SMMU_4MB_REGION; + pdir[pdn].huge.attr = attr; + iova += SZ_4M; + iopa += SZ_4M; + } + + smmu_flush_all(); +} diff --git a/bdk/mem/smmu.h b/bdk/mem/smmu.h index c0c765c..e45a672 100644 --- a/bdk/mem/smmu.h +++ b/bdk/mem/smmu.h @@ -15,32 +15,57 @@ * along with this program. If not, see . */ +#include + #include #define MC_SMMU_AVPC_ASID 0x23C #define MC_SMMU_TSEC_ASID 0x294 -#define SMMU_PDE_NEXT_SHIFT 28 -#define MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT 29 -#define MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT 30 -#define MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT 31 -#define _READABLE (1 << MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT) -#define _WRITABLE (1 << MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT) -#define _NONSECURE (1 << MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT) -#define _PDE_NEXT (1 << SMMU_PDE_NEXT_SHIFT) -#define _MASK_ATTR (_READABLE | _WRITABLE | _NONSECURE) -#define _PDIR_ATTR (_READABLE | _WRITABLE | _NONSECURE) -#define _PDE_ATTR (_READABLE | _WRITABLE | _NONSECURE) -#define _PDE_VACANT(pdn) (((pdn) << 10) | _PDE_ATTR) -#define _PTE_ATTR (_READABLE | _WRITABLE | _NONSECURE) -#define _PTE_VACANT(addr) (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR) +#define SMMU_NS BIT(0) +#define SMMU_WRITE BIT(1) +#define SMMU_READ BIT(2) +#define SMMU_ATTR_ALL (SMMU_READ | SMMU_WRITE | SMMU_NS) + +typedef struct _pde_t { + union { + union { + struct { + u32 table:22; + u32 rsvd:6; + u32 next:1; + u32 attr:3; + } tbl; + + struct { + u32 rsvd_:10; + u32 page:12; + u32 rsvd:6; + u32 next:1; + u32 attr:3; + } huge; + }; + + u32 pde; + }; +} pde_t; + +typedef struct _pte_t { + u32 page:22; + u32 rsvd:7; + u32 attr:3; +} pte_t; + +static_assert(sizeof(pde_t) == sizeof(u32), "pde_t size is wrong!"); +static_assert(sizeof(pte_t) == sizeof(u32), "pte_t size is wrong!"); void *smmu_page_zalloc(u32 num); void smmu_flush_all(); void smmu_init(); void smmu_enable(); -u32 *smmu_init_domain4(u32 dev_base, u32 asid); -u32 *smmu_get_pte(u32 *pdir, u32 iova); -void smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr); -u32 *smmu_init_domain(u32 asid); -void smmu_deinit_domain(u32 asid); +void smmu_reset_heap(); +void *smmu_init_domain(u32 dev_base, u32 asid); +void smmu_deinit_domain(u32 dev_base, u32 asid); +void smmu_domain_bypass(u32 dev_base, bool bypass); +void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr); +void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr); diff --git a/bdk/sec/tsec.c b/bdk/sec/tsec.c index 487076b..687af40 100644 --- a/bdk/sec/tsec.c +++ b/bdk/sec/tsec.c @@ -70,8 +70,9 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) int res = 0; u8 *fwbuf = NULL; u32 type = tsec_ctxt->type; - u32 *pdir, *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec; + u32 *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec; u32 *pkg11_magic_off; + void *ptb; bpmp_mmu_disable(); bpmp_freq_t prev_fid = bpmp_clk_rate_set(BPMP_CLK_NORMAL); @@ -145,7 +146,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) if (type == TSEC_FW_TYPE_EMU) { // Init SMMU translation for TSEC. - pdir = smmu_init_domain(MC_SMMU_TSEC_ASID); + ptb = smmu_init_domain(MC_SMMU_TSEC_ASID, 1); smmu_init(); // Enable SMMU. @@ -155,7 +156,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) car = smmu_page_zalloc(1); memcpy(car, (void *)CLOCK_BASE, SZ_PAGE); car[CLK_RST_CONTROLLER_CLK_SOURCE_TSEC / 4] = CLK_SRC_DIV(2); - smmu_map(pdir, CLOCK_BASE, (u32)car, 1, _WRITABLE | _READABLE | _NONSECURE); + smmu_map(ptb, CLOCK_BASE, (u32)car, 1, SMMU_WRITE | SMMU_READ | SMMU_NS); // Fuse driver. fuse = smmu_page_zalloc(1); @@ -163,38 +164,38 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) fuse[0x82C / 4] = 0; fuse[0x9E0 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1; fuse[0x9E4 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1; - smmu_map(pdir, (FUSE_BASE - 0x800), (u32)fuse, 1, _READABLE | _NONSECURE); + smmu_map(ptb, (FUSE_BASE - 0x800), (u32)fuse, 1, SMMU_READ | SMMU_NS); // Power management controller. pmc = smmu_page_zalloc(1); - smmu_map(pdir, RTC_BASE, (u32)pmc, 1, _READABLE | _NONSECURE); + smmu_map(ptb, RTC_BASE, (u32)pmc, 1, SMMU_READ | SMMU_NS); // Flow control. flowctrl = smmu_page_zalloc(1); - smmu_map(pdir, FLOW_CTLR_BASE, (u32)flowctrl, 1, _WRITABLE | _NONSECURE); + smmu_map(ptb, FLOW_CTLR_BASE, (u32)flowctrl, 1, SMMU_WRITE | SMMU_NS); // Security engine. se = smmu_page_zalloc(1); memcpy(se, (void *)SE_BASE, SZ_PAGE); - smmu_map(pdir, SE_BASE, (u32)se, 1, _READABLE | _WRITABLE | _NONSECURE); + smmu_map(ptb, SE_BASE, (u32)se, 1, SMMU_READ | SMMU_WRITE | SMMU_NS); // Memory controller. mc = smmu_page_zalloc(1); memcpy(mc, (void *)MC_BASE, SZ_PAGE); mc[MC_IRAM_BOM / 4] = 0; mc[MC_IRAM_TOM / 4] = DRAM_START; - smmu_map(pdir, MC_BASE, (u32)mc, 1, _READABLE | _NONSECURE); + smmu_map(ptb, MC_BASE, (u32)mc, 1, SMMU_READ | SMMU_NS); // IRAM iram = smmu_page_zalloc(0x30); memcpy(iram, tsec_ctxt->pkg1, 0x30000); // PKG1.1 magic offset. - pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / 4)); - smmu_map(pdir, 0x40010000, (u32)iram, 0x30, _READABLE | _WRITABLE | _NONSECURE); + pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / sizeof(u32))); + smmu_map(ptb, 0x40010000, (u32)iram, 0x30, SMMU_READ | SMMU_WRITE | SMMU_NS); // Exception vectors evec = smmu_page_zalloc(1); - smmu_map(pdir, EXCP_VEC_BASE, (u32)evec, 1, _READABLE | _WRITABLE | _NONSECURE); + smmu_map(ptb, EXCP_VEC_BASE, (u32)evec, 1, SMMU_READ | SMMU_WRITE | SMMU_NS); } // Execute firmware. @@ -229,7 +230,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) if (kidx != 8) { res = -6; - smmu_deinit_domain(MC_SMMU_TSEC_ASID); + smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1); goto out_free; } @@ -240,7 +241,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt) memcpy(tsec_keys, &key, 0x20); memcpy(tsec_ctxt->pkg1, iram, 0x30000); - smmu_deinit_domain(MC_SMMU_TSEC_ASID); + smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1); // for (int i = 0; i < kidx; i++) // gfx_printf("key %08X\n", key[i]);