// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved. */ #define pr_fmt(fmt) "io-pgtable-fast: " fmt #include #include #include #include #include #include #include #include #include #include #include #define AV8L_FAST_MAX_ADDR_BITS 48 /* Struct accessors */ #define iof_pgtable_to_data(x) \ container_of((x), struct av8l_fast_io_pgtable, iop) #define iof_pgtable_ops_to_pgtable(x) \ container_of((x), struct io_pgtable, ops) #define iof_pgtable_ops_to_data(x) \ iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x)) struct av8l_fast_io_pgtable { struct io_pgtable iop; av8l_fast_iopte *pgd; av8l_fast_iopte *puds[4]; av8l_fast_iopte *pmds; struct page **pages; /* page table memory */ int nr_pages; dma_addr_t base; dma_addr_t start; dma_addr_t end; }; /* Page table bits */ #define AV8L_FAST_PTE_TYPE_SHIFT 0 #define AV8L_FAST_PTE_TYPE_MASK 0x3 #define AV8L_FAST_PTE_TYPE_BLOCK 1 #define AV8L_FAST_PTE_TYPE_TABLE 3 #define AV8L_FAST_PTE_TYPE_PAGE 3 #define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63) #define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53) #define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10) #define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8) #define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8) #define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8) #define AV8L_FAST_PTE_SH_MASK (((av8l_fast_iopte)3) << 8) #define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5) #define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0) #define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2) /* Ignore the contiguous bit for block splitting */ #define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52) #define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \ AV8L_FAST_PTE_ATTR_HI_MASK) #define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000) /* Stage-1 PTE */ #define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6) #define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6) #define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6) #define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6) #define AV8L_FAST_PTE_ATTRINDX_SHIFT 2 #define AV8L_FAST_PTE_ATTRINDX_MASK 0x7 #define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11) /* Stage-2 PTE */ #define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6) #define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6) #define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6) #define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2) #define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2) #define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2) /* Register bits */ #define ARM_32_LPAE_TCR_EAE (1 << 31) #define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) #define AV8L_FAST_TCR_TG0_4K (0 << 14) #define AV8L_FAST_TCR_TG0_64K (1 << 14) #define AV8L_FAST_TCR_TG0_16K (2 << 14) #define AV8L_FAST_TCR_SH0_SHIFT 12 #define AV8L_FAST_TCR_SH0_MASK 0x3 #define AV8L_FAST_TCR_SH_NS 0 #define AV8L_FAST_TCR_SH_OS 2 #define AV8L_FAST_TCR_SH_IS 3 #define AV8L_FAST_TCR_ORGN0_SHIFT 10 #define AV8L_FAST_TCR_IRGN0_SHIFT 8 #define AV8L_FAST_TCR_RGN_MASK 0x3 #define AV8L_FAST_TCR_RGN_NC 0 #define AV8L_FAST_TCR_RGN_WBWA 1 #define AV8L_FAST_TCR_RGN_WT 2 #define AV8L_FAST_TCR_RGN_WB 3 #define AV8L_FAST_TCR_SL0_SHIFT 6 #define AV8L_FAST_TCR_SL0_MASK 0x3 #define AV8L_FAST_TCR_T0SZ_SHIFT 0 #define AV8L_FAST_TCR_SZ_MASK 0xf #define AV8L_FAST_TCR_PS_SHIFT 16 #define AV8L_FAST_TCR_PS_MASK 0x7 #define AV8L_FAST_TCR_IPS_SHIFT 32 #define AV8L_FAST_TCR_IPS_MASK 0x7 #define AV8L_FAST_TCR_PS_32_BIT 0x0ULL #define AV8L_FAST_TCR_PS_36_BIT 0x1ULL #define AV8L_FAST_TCR_PS_40_BIT 0x2ULL #define AV8L_FAST_TCR_PS_42_BIT 0x3ULL #define AV8L_FAST_TCR_PS_44_BIT 0x4ULL #define AV8L_FAST_TCR_PS_48_BIT 0x5ULL #define AV8L_FAST_TCR_EPD1_SHIFT 23 #define AV8L_FAST_TCR_EPD1_FAULT 1 #define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3) #define AV8L_FAST_MAIR_ATTR_MASK 0xff #define AV8L_FAST_MAIR_ATTR_DEVICE 0x04 #define AV8L_FAST_MAIR_ATTR_NC 0x44 #define AV8L_FAST_MAIR_ATTR_WBRWA 0xff #define AV8L_FAST_MAIR_ATTR_UPSTREAM 0xf4 #define AV8L_FAST_MAIR_ATTR_IDX_NC 0 #define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1 #define AV8L_FAST_MAIR_ATTR_IDX_DEV 2 #define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM 3 #define AV8L_FAST_PAGE_SHIFT 12 #define PTE_MAIR_IDX(pte) \ ((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) & \ AV8L_FAST_PTE_ATTRINDX_MASK) #define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK) #define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12)) #ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB #include #include static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list); void av8l_register_notify(struct notifier_block *nb) { atomic_notifier_chain_register(&av8l_notifier_list, nb); } EXPORT_SYMBOL(av8l_register_notify); static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) { if (unlikely(*ptep)) { atomic_notifier_call_chain( &av8l_notifier_list, MAPPED_OVER_STALE_TLB, (void *) ptep); pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n", *ptep, ptep); pr_err("Nearby memory:\n"); print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS, 32, 8, ptep - 16, 32 * sizeof(*ptep), false); } } void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base, u64 start, u64 end, bool skip_sync) { int i; struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); av8l_fast_iopte *pmdp = iopte_pmd_offset(pmds, base, start); for (i = start >> AV8L_FAST_PAGE_SHIFT; i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) { if (!(*pmdp & AV8L_FAST_PTE_VALID)) { *pmdp = 0; if (!skip_sync) dmac_clean_range(pmdp, pmdp + 1); } pmdp++; } } #else static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) { } #endif static void av8l_clean_range(struct io_pgtable_ops *ops, av8l_fast_iopte *start, av8l_fast_iopte *end) { struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops); if (!(iop->cfg.quirks & IO_PGTABLE_QUIRK_NO_DMA)) dmac_clean_range(start, end); } static av8l_fast_iopte av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot) { av8l_fast_iopte pte = AV8L_FAST_PTE_XN | AV8L_FAST_PTE_TYPE_PAGE | AV8L_FAST_PTE_AF | AV8L_FAST_PTE_nG | AV8L_FAST_PTE_SH_OS; if (prot & IOMMU_MMIO) pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV << AV8L_FAST_PTE_ATTRINDX_SHIFT); else if (prot & IOMMU_CACHE) pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE << AV8L_FAST_PTE_ATTRINDX_SHIFT); else if (prot & IOMMU_USE_UPSTREAM_HINT) pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM << AV8L_FAST_PTE_ATTRINDX_SHIFT); if (!(prot & IOMMU_WRITE)) pte |= AV8L_FAST_PTE_AP_RO; else pte |= AV8L_FAST_PTE_AP_RW; return pte; } static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova); unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT; av8l_fast_iopte pte; pte = av8l_fast_prot_to_pte(data, prot); paddr &= AV8L_FAST_PTE_ADDR_MASK; for (i = 0; i < nptes; i++, paddr += SZ_4K) { __av8l_check_for_stale_tlb(ptep + i); *(ptep + i) = pte | paddr; } av8l_clean_range(ops, ptep, ptep + nptes); return 0; } int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { return av8l_fast_map(ops, iova, paddr, size, prot); } static size_t __av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size, bool allow_stale_tlb) { struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); unsigned long nptes; av8l_fast_iopte *ptep; int val = allow_stale_tlb ? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI : 0; ptep = iopte_pmd_offset(data->pmds, data->base, iova); nptes = size >> AV8L_FAST_PAGE_SHIFT; memset(ptep, val, sizeof(*ptep) * nptes); av8l_clean_range(ops, ptep, ptep + nptes); if (!allow_stale_tlb) io_pgtable_tlb_flush_all(&data->iop); return size; } /* caller must take care of tlb cache maintenance */ void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova, size_t size) { __av8l_fast_unmap(ops, iova, size, true); } static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size) { return __av8l_fast_unmap(ops, iova, size, false); } static int av8l_fast_map_sg(struct io_pgtable_ops *ops, unsigned long iova, struct scatterlist *sgl, unsigned int nents, int prot, size_t *size) { struct scatterlist *sg; int i; for_each_sg(sgl, sg, nents, i) { av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot); iova += sg->length; } return nents; } int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, unsigned long iova, struct scatterlist *sgl, unsigned int nents, int prot, size_t *size) { return av8l_fast_map_sg(ops, iova, sgl, nents, prot, size); } #if defined(CONFIG_ARM64) #define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27) #elif defined(CONFIG_ARM) #define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27) #endif static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); av8l_fast_iopte pte, *pgdp, *pudp, *pmdp; unsigned long pgd; phys_addr_t phys; const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT; const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK; const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE; const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE; const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK; /* TODO: clean up some of these magic numbers... */ pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova); pgdp = (av8l_fast_iopte *)pgd; pte = *pgdp; if (((pte >> pts) & ptm) != ptt) return 0; pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18)); pte = *pudp; if (((pte >> pts) & ptm) != ptt) return 0; pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9)); pte = *pmdp; if (((pte >> pts) & ptm) != ptp) return 0; phys = pte & am; return phys | (iova & 0xfff); } phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, unsigned long iova) { return av8l_fast_iova_to_phys(ops, iova); } static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops, unsigned long iova) { struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova); return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) && ((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) || (PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS))); } bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, unsigned long iova) { return av8l_fast_iova_coherent(ops, iova); } static struct av8l_fast_io_pgtable * av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg) { struct av8l_fast_io_pgtable *data; data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; data->iop.ops = (struct io_pgtable_ops) { .map = av8l_fast_map, .map_sg = av8l_fast_map_sg, .unmap = av8l_fast_unmap, .iova_to_phys = av8l_fast_iova_to_phys, .is_iova_coherent = av8l_fast_iova_coherent, }; return data; } /* * We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and * 2048 pages for pmds (each pud page contains 512 table entries, each * pointing to a pmd). */ #define NUM_PGD_PAGES 1 #define NUM_PUD_PAGES 4 #define NUM_PMD_PAGES 2048 #define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES) /* undefine arch specific definitions which depends on page table format */ #undef pud_index #undef pud_mask #undef pud_next #undef pmd_index #undef pmd_mask #undef pmd_next #define pud_index(addr) (((addr) >> 30) & 0x3) #define pud_mask(addr) ((addr) & ~((1UL << 30) - 1)) #define pud_next(addr, end) \ ({ unsigned long __boundary = pud_mask(addr + (1UL << 30));\ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ }) #define pmd_index(addr) (((addr) >> 21) & 0x1ff) #define pmd_mask(addr) ((addr) & ~((1UL << 21) - 1)) #define pmd_next(addr, end) \ ({ unsigned long __boundary = pmd_mask(addr + (1UL << 21));\ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ }) static int av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, struct io_pgtable_cfg *cfg, void *cookie) { int i, j, pg = 0; struct page **pages, *page; dma_addr_t base = cfg->iova_base; dma_addr_t end = cfg->iova_end; dma_addr_t pud, pmd; int pmd_pg_index; pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN | __GFP_NORETRY); if (!pages) pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES); if (!pages) return -ENOMEM; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) goto err_free_pages_arr; pages[pg++] = page; data->pgd = page_address(page); /* * We need max 2048 entries at level 2 to map 4GB of VA space. A page * can hold 512 entries, so we need max 4 pages. */ for (i = pud_index(base), pud = base; pud < end; ++i, pud = pud_next(pud, end)) { av8l_fast_iopte pte, *ptep; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) goto err_free_pages; pages[pg++] = page; data->puds[i] = page_address(page); pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; ptep = ((av8l_fast_iopte *)data->pgd) + i; *ptep = pte; } dmac_clean_range(data->pgd, data->pgd + 4); /* * We have max 4 puds, each of which can point to 512 pmds, so we'll * have max 2048 pmds, each of which can hold 512 ptes, for a grand * total of 2048*512=1048576 PTEs. */ pmd_pg_index = pg; for (i = pud_index(base), pud = base; pud < end; ++i, pud = pud_next(pud, end)) { for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end); ++j, pmd = pmd_next(pmd, end)) { av8l_fast_iopte pte, *pudp; void *addr; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) goto err_free_pages; pages[pg++] = page; addr = page_address(page); dmac_clean_range(addr, addr + SZ_4K); pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; pudp = data->puds[i] + j; *pudp = pte; } dmac_clean_range(data->puds[i], data->puds[i] + 512); } /* * We map the pmds into a virtually contiguous space so that we * don't have to traverse the first two levels of the page tables * to find the appropriate pud. Instead, it will be a simple * offset from the virtual base of the pmds. */ data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index, VM_IOREMAP, PAGE_KERNEL); if (!data->pmds) goto err_free_pages; data->pages = pages; data->nr_pages = pg; data->base = base; data->end = end; return 0; err_free_pages: for (i = 0; i < pg; ++i) __free_page(pages[i]); err_free_pages_arr: kvfree(pages); return -ENOMEM; } static struct io_pgtable * av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg; struct av8l_fast_io_pgtable *data = av8l_fast_alloc_pgtable_data(cfg); if (!data) return NULL; /* restrict according to the fast map requirements */ cfg->ias = 32; cfg->pgsize_bitmap = SZ_4K; /* TCR */ if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) | (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT); else if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_IRGN0_SHIFT) | (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT); else reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) | (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT); reg |= AV8L_FAST_TCR_TG0_4K; switch (cfg->oas) { case 32: reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; case 36: reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; case 40: reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; case 42: reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; case 44: reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; case 48: reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT); break; default: goto out_free_data; } reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT; reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT; #if defined(CONFIG_ARM) reg |= ARM_32_LPAE_TCR_EAE; #endif cfg->av8l_fast_cfg.tcr = reg; /* MAIRs */ reg = (AV8L_FAST_MAIR_ATTR_NC << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) | (AV8L_FAST_MAIR_ATTR_WBRWA << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) | (AV8L_FAST_MAIR_ATTR_DEVICE << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) | (AV8L_FAST_MAIR_ATTR_UPSTREAM << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM)); cfg->av8l_fast_cfg.mair[0] = reg; cfg->av8l_fast_cfg.mair[1] = 0; /* Allocate all page table memory! */ if (av8l_fast_prepopulate_pgtables(data, cfg, cookie)) goto out_free_data; cfg->av8l_fast_cfg.pmds = data->pmds; /* TTBRs */ cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd); cfg->av8l_fast_cfg.ttbr[1] = 0; return &data->iop; out_free_data: kfree(data); return NULL; } static void av8l_fast_free_pgtable(struct io_pgtable *iop) { int i; struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop); vunmap(data->pmds); for (i = 0; i < data->nr_pages; ++i) __free_page(data->pages[i]); kvfree(data->pages); kfree(data); } struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = { .alloc = av8l_fast_alloc_pgtable, .free = av8l_fast_free_pgtable, }; #ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST #include static struct io_pgtable_cfg *cfg_cookie; static void dummy_tlb_flush_all(void *cookie) { WARN_ON(cookie != cfg_cookie); } static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { WARN_ON(cookie != cfg_cookie); WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); } static void dummy_tlb_sync(void *cookie) { WARN_ON(cookie != cfg_cookie); } static struct iommu_gather_ops dummy_tlb_ops __initdata = { .tlb_flush_all = dummy_tlb_flush_all, .tlb_add_flush = dummy_tlb_add_flush, .tlb_sync = dummy_tlb_sync, }; /* * Returns true if the iova range is successfully mapped to the contiguous * phys range in ops. */ static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops, const unsigned long iova_start, const phys_addr_t phys_start, const size_t size) { u64 iova = iova_start; phys_addr_t phys = phys_start; while (iova < (iova_start + size)) { /* + 42 just to make sure offsetting is working */ if (ops->iova_to_phys(ops, iova + 42) != (phys + 42)) return false; iova += SZ_4K; phys += SZ_4K; } return true; } static int __init av8l_fast_positive_testing(void) { int failed = 0; u64 iova; struct io_pgtable_ops *ops; struct io_pgtable_cfg cfg; struct av8l_fast_io_pgtable *data; av8l_fast_iopte *pmds; u64 max = SZ_1G * 4ULL - 1; u64 base = 0; cfg = (struct io_pgtable_cfg) { .quirks = 0, .tlb = &dummy_tlb_ops, .ias = 32, .oas = 32, .pgsize_bitmap = SZ_4K, .iova_base = base, .iova_end = max, }; cfg_cookie = &cfg; ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg); if (WARN_ON(!ops)) return 1; data = iof_pgtable_ops_to_data(ops); pmds = data->pmds; /* map the entire 4GB VA space with 4K map calls */ for (iova = base; iova < max; iova += SZ_4K) { if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) { failed++; continue; } } if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, base, max - base))) failed++; /* unmap it all */ for (iova = base; iova < max; iova += SZ_4K) { if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K)) failed++; } /* sweep up TLB proving PTEs */ av8l_fast_clear_stale_ptes(pmds, base, base, max, false); /* map the entire 4GB VA space with 8K map calls */ for (iova = base; iova < max; iova += SZ_8K) { if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) { failed++; continue; } } if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, base, max - base))) failed++; /* unmap it all with 8K unmap calls */ for (iova = base; iova < max; iova += SZ_8K) { if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K)) failed++; } /* sweep up TLB proving PTEs */ av8l_fast_clear_stale_ptes(pmds, base, base, max, false); /* map the entire 4GB VA space with 16K map calls */ for (iova = base; iova < max; iova += SZ_16K) { if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) { failed++; continue; } } if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, base, max - base))) failed++; /* unmap it all */ for (iova = base; iova < max; iova += SZ_16K) { if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K)) failed++; } /* sweep up TLB proving PTEs */ av8l_fast_clear_stale_ptes(pmds, base, base, max, false); /* map the entire 4GB VA space with 64K map calls */ for (iova = base; iova < max; iova += SZ_64K) { if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) { failed++; continue; } } if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, base, max - base))) failed++; /* unmap it all at once */ if (WARN_ON(ops->unmap(ops, base, max - base) != (max - base))) failed++; free_io_pgtable_ops(ops); return failed; } static int __init av8l_fast_do_selftests(void) { int failed = 0; failed += av8l_fast_positive_testing(); pr_err("selftest: completed with %d failures\n", failed); return 0; } subsys_initcall(av8l_fast_do_selftests); #endif