From 0c2147213bf96043a7f105e59cc76280a9bcb2d0 Mon Sep 17 00:00:00 2001 From: wangkuanhong Date: Wed, 17 Jul 2024 15:44:26 +0800 Subject: [PATCH 104/219] refactor: move es_dev_buf and iommu_rsv into kernel Changelogs: move es_dev_buf and iommu_rsv into kernel Signed-off-by: wangkuanhong --- arch/riscv/configs/win2030_defconfig | 2 + drivers/memory/eswin/Kconfig | 2 + drivers/memory/eswin/Makefile | 4 +- drivers/memory/eswin/es_dev_buf/Kconfig | 8 + drivers/memory/eswin/es_dev_buf/Makefile | 5 + drivers/memory/eswin/es_dev_buf/es_dev_buf.c | 194 +++++++ .../es_dev_buf/include/linux/dsp_dma_buf.h | 44 ++ drivers/memory/eswin/es_iommu_rsv/Kconfig | 7 + drivers/memory/eswin/es_iommu_rsv/Makefile | 5 + .../es_iommu_rsv/include/linux/es_iommu_rsv.h | 10 + drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c | 506 ++++++++++++++++++ 11 files changed, 786 insertions(+), 1 deletion(-) create mode 100644 drivers/memory/eswin/es_dev_buf/Kconfig create mode 100644 drivers/memory/eswin/es_dev_buf/Makefile create mode 100644 drivers/memory/eswin/es_dev_buf/es_dev_buf.c create mode 100644 drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h create mode 100644 drivers/memory/eswin/es_iommu_rsv/Kconfig create mode 100644 drivers/memory/eswin/es_iommu_rsv/Makefile create mode 100644 drivers/memory/eswin/es_iommu_rsv/include/linux/es_iommu_rsv.h create mode 100644 drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c diff --git a/arch/riscv/configs/win2030_defconfig b/arch/riscv/configs/win2030_defconfig index e985f5c3732c..1603e0ec1bf7 100644 --- a/arch/riscv/configs/win2030_defconfig +++ b/arch/riscv/configs/win2030_defconfig @@ -740,6 +740,8 @@ CONFIG_ESWIN_BUDDY=y CONFIG_ESWIN_PROC=y CONFIG_ESWIN_RSVMEM_HEAP=y CONFIG_ESWIN_MMZ_VB=y +CONFIG_ESWIN_DEV_DMA_BUF=y +CONFIG_ESWIN_IOMMU_RSV=y CONFIG_PWM=y CONFIG_PWM_ESWIN=y CONFIG_RESET_ESWIN_WIN2030=y diff --git a/drivers/memory/eswin/Kconfig b/drivers/memory/eswin/Kconfig index 013e2f376dd3..3078936326e4 100644 --- a/drivers/memory/eswin/Kconfig +++ b/drivers/memory/eswin/Kconfig @@ -29,5 +29,7 @@ source "drivers/memory/eswin/es_buddy/Kconfig" source "drivers/memory/eswin/es_proc/Kconfig" source "drivers/memory/eswin/es_rsvmem_heap/Kconfig" source "drivers/memory/eswin/es_mmz_vb/Kconfig" +source "drivers/memory/eswin/es_dev_buf/Kconfig" +source "drivers/memory/eswin/es_iommu_rsv/Kconfig" endif diff --git a/drivers/memory/eswin/Makefile b/drivers/memory/eswin/Makefile index 3e5e09bab5dd..c07fadc64d44 100644 --- a/drivers/memory/eswin/Makefile +++ b/drivers/memory/eswin/Makefile @@ -6,7 +6,9 @@ obj-$(CONFIG_ESWIN_BUDDY) += es_buddy/ obj-$(CONFIG_ESWIN_PROC) += es_proc/ obj-$(CONFIG_ESWIN_RSVMEM_HEAP) += es_rsvmem_heap/ obj-$(CONFIG_ESWIN_RSVMEM_HEAP) += es_mmz_vb/ +obj-$(CONFIG_ESWIN_DEV_DMA_BUF) += es_dev_buf/ +obj-$(CONFIG_ESWIN_IOMMU_RSV) += es_iommu_rsv/ ES_MEM_HEADER := drivers/memory/eswin/ -COPY_HEADERS := $(shell cp $(ES_MEM_HEADER)/*.h include/linux) \ No newline at end of file +COPY_HEADERS := $(shell cp $(ES_MEM_HEADER)/*.h include/linux) diff --git a/drivers/memory/eswin/es_dev_buf/Kconfig b/drivers/memory/eswin/es_dev_buf/Kconfig new file mode 100644 index 000000000000..78e59f570ac9 --- /dev/null +++ b/drivers/memory/eswin/es_dev_buf/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 + +config ESWIN_DEV_DMA_BUF + tristate "ESWIN dev dma buf interface" + depends on ESWIN_RSVMEM_HEAP + help + eswin device dma buf + diff --git a/drivers/memory/eswin/es_dev_buf/Makefile b/drivers/memory/eswin/es_dev_buf/Makefile new file mode 100644 index 000000000000..3ba183ff62bd --- /dev/null +++ b/drivers/memory/eswin/es_dev_buf/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_ESWIN_DEV_DMA_BUF) += es_dev_buf.o + +ES_DEV_DMA_BUF_HEADER := drivers/memory/eswin/es_dev_buf/include/linux +COPY_HEADERS:=$(shell cp $(ES_DEV_DMA_BUF_HEADER)/*.h include/linux) + diff --git a/drivers/memory/eswin/es_dev_buf/es_dev_buf.c b/drivers/memory/eswin/es_dev_buf/es_dev_buf.c new file mode 100644 index 000000000000..02bb8e2de2da --- /dev/null +++ b/drivers/memory/eswin/es_dev_buf/es_dev_buf.c @@ -0,0 +1,194 @@ +// Copyright © 2023 ESWIN. All rights reserved. +// +// Beijing ESWIN Computing Technology Co., Ltd and its affiliated companies ("ESWIN") retain +// all intellectual property and proprietary rights in and to this software. Except as expressly +// authorized by ESWIN, no part of the software may be released, copied, distributed, reproduced, +// modified, adapted, translated, or created derivative work of, in whole or in part. + +#include +#include +#include +#include + +MODULE_IMPORT_NS(DMA_BUF); + +dma_addr_t dev_mem_attach(struct dma_buf *buf, struct device *dev, + enum dma_data_direction direc, + struct dma_buf_attachment **attachment) +{ + struct sg_table *table; + dma_addr_t reg; + unsigned int size; + struct dma_buf_attachment *attach; + struct scatterlist *sg = NULL; + u64 addr; + int len; + int i = 0; + + attach = dma_buf_attach(buf, dev); + if (IS_ERR(attach)) { + pr_err("%s, dma buf attach error.\n", __func__); + return 0; + } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + table = dma_buf_map_attachment_unlocked(attach, direc); +#else + table = dma_buf_map_attachment(attach, direc); +#endif + if (IS_ERR(table)) { + pr_err("%s, dma buf map attachment error.\n", __func__); + dma_buf_detach(buf, attach); + return 0; + } + + /* + * workaround: caching sgt table in attachment. + */ + if (!attach->dmabuf->ops->cache_sgt_mapping) { + attach->sgt = table; + } + + for_each_sgtable_dma_sg(table, sg, i) { + addr = sg_dma_address(sg); + len = sg_dma_len(sg); + pr_debug("%s, i=%d, sg addr=0x%llx, len=0x%x, phyaddr=0x%llx.\n", + __func__, i, addr, len, dma_to_phys(dev, addr)); + } + + reg = sg_dma_address(table->sgl); + size = sg_dma_len(table->sgl); + if (attachment != NULL) { + *attachment = attach; + } + + return reg; +} +EXPORT_SYMBOL(dev_mem_attach); + +int dev_mem_detach(struct dma_buf_attachment *attach, + enum dma_data_direction direction) +{ + struct sg_table *table; + + if (!attach->dmabuf->ops->cache_sgt_mapping) { + table = attach->sgt; + attach->sgt = NULL; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + dma_buf_unmap_attachment_unlocked(attach, table, DMA_BIDIRECTIONAL); +#else + dma_buf_unmap_attachment(attach, table, DMA_BIDIRECTIONAL); +#endif + dma_buf_detach(attach->dmabuf, attach); + return 0; +} +EXPORT_SYMBOL(dev_mem_detach); + +// this interface only for x86 emulation +int dev_mem_alloc(size_t size, enum es_malloc_policy policy, + struct dma_buf **buf) +{ + int fd; + struct dma_buf *dma_buf; + + switch (policy) { + case ES_MEM_ALLOC_RSV: + fd = dma_heap_kalloc("reserved", size, O_RDWR, 0); + break; + case ES_MEM_ALLOC_NORMAL: + fd = dma_heap_kalloc("system", size, O_RDWR, 0); + break; + case ES_MEM_ALLOC_NORMAL_COHERENT: + fd = dma_heap_kalloc("system", size, O_RDWR | O_DSYNC, 0); + break; + case ES_MEM_ALLOC_CMA: + fd = dma_heap_kalloc("linux,cma", size, O_RDWR, 0); + break; + case ES_MEM_ALLOC_CMA_COHERENT: + fd = dma_heap_kalloc("linux,cma", size, O_RDWR | O_DSYNC, 0); + break; + case ES_MEM_ALLOC_CMA_LLC: + fd = dma_heap_kalloc("linux,cma", size, O_RDWR | O_SYNC, 0); + break; + case ES_MEM_ALLOC_SPRAM_DIE0: + fd = dma_heap_kalloc("llc_spram0", size, O_RDWR, 1); + break; + case ES_MEM_ALLOC_SPRAM_DIE1: + fd = dma_heap_kalloc("llc_spram1", size, O_RDWR, 1); + break; + default: + pr_err("policy = %d not supported.\n", policy); + return -EINVAL; + } + if (fd < 0) { + return fd; + } + dma_buf = dma_buf_get(fd); + if (IS_ERR(dma_buf)) { + return -EINVAL; + } + if (buf != NULL) { + *buf = dma_buf; + } + + return fd; +} +EXPORT_SYMBOL(dev_mem_alloc); + +int dev_mem_free(struct dma_buf *buf) +{ + if (buf != NULL) { + dma_buf_put(buf); + } + return 0; +} +EXPORT_SYMBOL(dev_mem_free); + +void *dev_mem_vmap(struct dev_buffer_desc *buffer) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + struct iosys_map map; +#else + struct dma_buf_map map; +#endif + int ret; + + if (!buffer) { + return NULL; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + ret = dma_buf_vmap_unlocked(buffer->buf, &map); +#else + ret = dma_buf_vmap(buffer->buf, &map); +#endif + if (ret) { + return NULL; + } + return map.vaddr; +} +EXPORT_SYMBOL(dev_mem_vmap); + +void dev_mem_vunmap(struct dev_buffer_desc *buffer, void *virt) +{ + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + struct iosys_map map; +#else + struct dma_buf_map map; +#endif + if (!buffer || !virt) { + return; + } + + map.vaddr = virt; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + dma_buf_vunmap_unlocked(buffer->buf, &map); +#else + dma_buf_vunmap(buffer->buf, &map); +#endif +} +EXPORT_SYMBOL(dev_mem_vunmap); + +MODULE_LICENSE("GPL"); diff --git a/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h b/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h new file mode 100644 index 000000000000..ab6962ad1ccf --- /dev/null +++ b/drivers/memory/eswin/es_dev_buf/include/linux/dsp_dma_buf.h @@ -0,0 +1,44 @@ +// Copyright © 2023 ESWIN. All rights reserved. +// +// Beijing ESWIN Computing Technology Co., Ltd and its affiliated companies ("ESWIN") retain +// all intellectual property and proprietary rights in and to this software. Except as expressly +// authorized by ESWIN, no part of the software may be released, copied, distributed, reproduced, +// modified, adapted, translated, or created derivative work of, in whole or in part. + +#ifndef __DSP_DMA_BUF_H_ +#define __DSP_DMA_BUF_H_ +#include + +struct dev_buffer_desc { + struct dma_buf *buf; + u32 offset; + u32 len; +}; + +enum es_malloc_policy { + ES_MEM_ALLOC_RSV, + ES_MEM_ALLOC_NORMAL, + ES_MEM_ALLOC_NORMAL_COHERENT, + ES_MEM_ALLOC_CMA, + ES_MEM_ALLOC_CMA_COHERENT, + ES_MEM_ALLOC_CMA_LLC, + ES_MEM_ALLOC_SPRAM_DIE0, + ES_MEM_ALLOC_SPRAM_DIE1, +}; + +int dev_mem_alloc(size_t size, enum es_malloc_policy policy, + struct dma_buf **buf); +int dev_mem_free(struct dma_buf *buf); + +int dev_mem_alloc_pool(size_t size, enum es_malloc_policy policy, char *mmb, + char *zone, struct dma_buf **buf); + +dma_addr_t dev_mem_attach(struct dma_buf *buf, struct device *dev, + enum dma_data_direction direc, + struct dma_buf_attachment **attachment); +int dev_mem_detach(struct dma_buf_attachment *attach, + enum dma_data_direction direction); +void *dev_mem_vmap(struct dev_buffer_desc *dev_buffer); +void dev_mem_vunmap(struct dev_buffer_desc *dev_buf, void *vaddr); + +#endif diff --git a/drivers/memory/eswin/es_iommu_rsv/Kconfig b/drivers/memory/eswin/es_iommu_rsv/Kconfig new file mode 100644 index 000000000000..f04159dd39df --- /dev/null +++ b/drivers/memory/eswin/es_iommu_rsv/Kconfig @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +config ESWIN_IOMMU_RSV + tristate "ESWIN IOMMU reserved for device interface" + help + eswin iommu rsv intf for device + diff --git a/drivers/memory/eswin/es_iommu_rsv/Makefile b/drivers/memory/eswin/es_iommu_rsv/Makefile new file mode 100644 index 000000000000..a4fa098eb518 --- /dev/null +++ b/drivers/memory/eswin/es_iommu_rsv/Makefile @@ -0,0 +1,5 @@ +obj-$(CONFIG_ESWIN_IOMMU_RSV) += iommu_rsv.o + +ES_IOMMU_RSV_HEADER := drivers/memory/eswin/es_iommu_rsv/include/linux + +COPY_HEADERS := $(shell cp $(ES_IOMMU_RSV_HEADER)/*h include/linux) diff --git a/drivers/memory/eswin/es_iommu_rsv/include/linux/es_iommu_rsv.h b/drivers/memory/eswin/es_iommu_rsv/include/linux/es_iommu_rsv.h new file mode 100644 index 000000000000..3e11a6421502 --- /dev/null +++ b/drivers/memory/eswin/es_iommu_rsv/include/linux/es_iommu_rsv.h @@ -0,0 +1,10 @@ +#ifndef __ESWIN_IOMMU_RSV__ +#define __ESWIN_IOMMU_RSV__ +#include +#include + +int iommu_unmap_rsv_iova(struct device *dev, void *cpu_addr, dma_addr_t iova, unsigned long size); +int iommu_map_rsv_iova_with_phys(struct device *dev, dma_addr_t iova, unsigned long size, phys_addr_t paddr, unsigned long attrs); +void *iommu_map_rsv_iova(struct device *dev, dma_addr_t iova, unsigned long size, gfp_t gfp, unsigned long attrs); + +#endif diff --git a/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c b/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c new file mode 100644 index 000000000000..397601410e6e --- /dev/null +++ b/drivers/memory/eswin/es_iommu_rsv/iommu_rsv.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * A fairly generic DMA-API to IOMMU-API glue layer. + * + * Copyright (C) 2014-2015 ARM Ltd. + * + * based in part on arch/arm/mm/dma-mapping.c: + * Copyright (C) 2000-2004 Russell King + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BUF_SIZE 0x300000 + +#define IOVA_FQ_SIZE 256 + +/* Timeout (in ms) after which entries are flushed from the queue */ +#define IOVA_FQ_TIMEOUT 10 + +/* Flush queue entry for deferred flushing */ +struct iova_fq_entry +{ + unsigned long iova_pfn; + unsigned long pages; + struct list_head freelist; + u64 counter; /* Flush counter when this entry was added */ +}; + +/* Per-CPU flush queue structure */ +struct iova_fq +{ + struct iova_fq_entry entries[IOVA_FQ_SIZE]; + unsigned int head, tail; + spinlock_t lock; +}; + +enum iommu_dma_cookie_type +{ + IOMMU_DMA_IOVA_COOKIE, + IOMMU_DMA_MSI_COOKIE, +}; + +struct iommu_dma_cookie +{ + enum iommu_dma_cookie_type type; + union + { + struct + { + struct iova_domain iovad; + + struct iova_fq __percpu *fq; /* Flush queue */ + /* Number of TLB flushes that have been started */ + atomic64_t fq_flush_start_cnt; + /* Number of TLB flushes that have been finished */ + atomic64_t fq_flush_finish_cnt; + /* Timer to regularily empty the flush queues */ + struct timer_list fq_timer; + /* 1 when timer is active, 0 when not */ + atomic_t fq_timer_on; + }; + /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ + dma_addr_t msi_iova; + }; + struct list_head msi_page_list; + + /* Domain for flush queue callback; NULL if flush queue not in use */ + struct iommu_domain *fq_domain; +}; + +static void *(*es_common_pages_remap)(struct page **pages, size_t size, pgprot_t prot, const void *caller) = NULL; +static void (*__iommu_dma_unmap)(struct device *dev, dma_addr_t dma_addr, size_t size) = NULL; + +static void (*__iommu_dma_free)(struct device *dev, size_t size, void *cpu_add) = NULL; +static void (*dma_prep_coherent)(struct page *page, size_t size) = NULL; +static unsigned long (*klp_name)(const char *name) = NULL; + +static struct iova_domain *es_get_iovad(struct iommu_domain *domain) +{ + struct iova_domain *iovad; + + iovad = (struct iova_domain *)(&(domain->iova_cookie->iovad)); + return iovad; +} + +static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, + unsigned long attrs) +{ + int prot = coherent ? IOMMU_CACHE : 0; + + if (attrs & DMA_ATTR_PRIVILEGED) + prot |= IOMMU_PRIV; + + switch (dir) + { + case DMA_BIDIRECTIONAL: + return prot | IOMMU_READ | IOMMU_WRITE; + case DMA_TO_DEVICE: + return prot | IOMMU_READ; + case DMA_FROM_DEVICE: + return prot | IOMMU_WRITE; + default: + return 0; + } +} + +static void __iommu_dma_free_pages(struct page **pages, int count) +{ + while (count--) + __free_page(pages[count]); + kvfree(pages); +} + +static struct page **__iommu_dma_alloc_pages(struct device *dev, + unsigned int count, unsigned long order_mask, gfp_t gfp) +{ + struct page **pages; + unsigned int i = 0, nid = dev_to_node(dev); + + order_mask &= (2U << MAX_ORDER) - 1; + if (!order_mask) + return NULL; + + pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL); + if (!pages) + return NULL; + + /* IOMMU can map any pages, so himem can also be used here */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + + /* It makes no sense to muck about with huge pages */ + gfp &= ~__GFP_COMP; + + while (count) + { + struct page *page = NULL; + unsigned int order_size; + + /* + * Higher-order allocations are a convenience rather + * than a necessity, hence using __GFP_NORETRY until + * falling back to minimum-order allocations. + */ + // 2U << __fls(count): 得到count的mask， 比如count为0b1000, mask就是0b1111. + // mask和order_mask相与，取出最最高bit，就是针对当前count可以分配内存的最大block size。 + for (order_mask &= (2U << __fls(count)) - 1; + order_mask; order_mask &= ~order_size) + { + unsigned int order = __fls(order_mask); + gfp_t alloc_flags = gfp; + + order_size = 1U << order; + if (order_mask > order_size) + alloc_flags |= __GFP_NORETRY; + page = alloc_pages_node(nid, alloc_flags, order); + if (!page) + continue; + if (order) + split_page(page, order); + break; + } + if (!page) + { + __iommu_dma_free_pages(pages, i); + return NULL; + } + count -= order_size; + while (order_size--) + pages[i++] = page++; + } + return pages; +} + +static struct page **iommu_rsv_iova_alloc_pages(struct device *dev, unsigned long iova, unsigned long size, struct sg_table *sgt, gfp_t gfp, unsigned long attrs) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iova_domain *iovad = es_get_iovad(domain); //&domain->iova_cookie->iovad; + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + struct page **pages = NULL; + int ret; + unsigned int alloc_sizes = domain->pgsize_bitmap; + unsigned int count, min_size; + + min_size = alloc_sizes & -alloc_sizes; + if (min_size < PAGE_SIZE) + { + min_size = PAGE_SIZE; + alloc_sizes |= PAGE_SIZE; + } + else + { + size = ALIGN(size, min_size); + } + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, gfp); + if (pages == NULL) + { + dev_err(dev, "alloc pages failed.\n"); + return NULL; + } + size = iova_align(iovad, size); + + if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL)) + { + goto out_free_iova; + } + + if (!(ioprot & IOMMU_CACHE)) + { + struct scatterlist *sg; + int i; + + for_each_sg(sgt->sgl, sg, sgt->orig_nents, i) + dma_prep_coherent(sg_page(sg), sg->length); + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot, GFP_KERNEL); +#else + ret = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot); +#endif + if (ret < size) + { + dev_err(dev, "%s %d fail %d\n", __func__, __LINE__, ret); + goto out_free_sg; + } + return pages; + +out_free_sg: + sg_free_table(sgt); +out_free_iova: + __iommu_dma_free_pages(pages, count); + return NULL; +} + +static void *iommu_rsv_iova_alloc_remap(struct device *dev, dma_addr_t iova, unsigned long size, gfp_t gfp, unsigned long attrs) +{ + + struct page **pages = NULL; + struct sg_table sgt; + void *cpu_addr; + + pages = iommu_rsv_iova_alloc_pages(dev, iova, size, &sgt, gfp, attrs); + if (pages == NULL) + { + dev_err(dev, "alloc pages failed.\n"); + return NULL; + } + + sg_free_table(&sgt); + cpu_addr = es_common_pages_remap(pages, size, pgprot_dmacoherent(PAGE_KERNEL), __builtin_return_address(0)); + if (!cpu_addr) + goto out_unmap; + return cpu_addr; + +out_unmap: + __iommu_dma_unmap(dev, iova, size); + __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); + return NULL; +} + +void *iommu_map_rsv_iova(struct device *dev, dma_addr_t iova, unsigned long size, gfp_t gfp, unsigned long attrs) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + phys_addr_t phys; + + if (!size) + { + dev_err(dev, "%s, error, size is 0.\n", __func__); + return NULL; + } + + phys = iommu_iova_to_phys(domain, iova); + if (phys != 0) + { + dev_err(dev, "%s, iova=0x%llx, have exits phys=0x%llx\n", __func__, iova, phys); + return NULL; + } + + gfp &= ~(__GFP_DMA | GFP_DMA32 | __GFP_HIGHMEM); + gfp |= __GFP_ZERO; + + if (gfpflags_allow_blocking(gfp)) + { + return iommu_rsv_iova_alloc_remap(dev, iova, size, gfp, attrs); + } + return NULL; +} +EXPORT_SYMBOL(iommu_map_rsv_iova); + +int iommu_map_rsv_iova_with_phys(struct device *dev, dma_addr_t iova, unsigned long size, phys_addr_t paddr, unsigned long attrs) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iova_domain *iovad = es_get_iovad(domain); + bool coherent = dev_is_dma_coherent(dev); + int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + size_t iova_off; + unsigned int min_pagesz; + phys_addr_t phys; + + if (!size) + { + dev_err(dev, "%s, error, size is 0.\n", __func__); + return -EINVAL; + } + phys = iommu_iova_to_phys(domain, iova); + if (phys != 0) + { + dev_err(dev, "%s, iova=0x%llx, have exits phys=0x%llx\n", __func__, iova, paddr); + return -EINVAL; + } + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * * both the virtual address and the physical one, as well as + * * the size of the mapping, must be aligned (at least) to the + * * size of the smallest page supported by the hardware + */ + + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) + { + pr_err("unaligned: iova 0x%llx pa %pa size 0x%zx min_pagesz 0x%x\n", iova, &paddr, size, min_pagesz); + return -EINVAL; + } + + iova_off = iova_offset(iovad, phys); + size = iova_align(iovad, size + iova_off); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + if (iommu_map(domain, iova, paddr - iova_off, size, ioprot, GFP_ATOMIC)) +#else + if (iommu_map_atomic(domain, iova, paddr - iova_off, size, ioprot)) +#endif + { + free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); + return -EIO; + } + return 0; +} +EXPORT_SYMBOL(iommu_map_rsv_iova_with_phys); + +int iommu_unmap_rsv_iova(struct device *dev, void *cpu_addr, dma_addr_t iova, unsigned long size) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct iova_domain *iovad = es_get_iovad(domain); + phys_addr_t phys; + + phys = iommu_iova_to_phys(domain, iova); + if (phys == 0) + { + dev_err(dev, "%s, iova=0x%llx, have not maps\n", __func__, iova); + return -EINVAL; + } + + if (!size) + { + return -EINVAL; + } + size = iova_align(iovad, size); + iommu_unmap(domain, iova, size); + + if (cpu_addr != NULL && __iommu_dma_free) + { + __iommu_dma_free(dev, size, cpu_addr); + } + return 0; +} +EXPORT_SYMBOL(iommu_unmap_rsv_iova); + +static void eswin_iommu_put_func(void) +{ + if (__iommu_dma_free) + { + __iommu_dma_free = NULL; + } + + if (es_common_pages_remap) + { + es_common_pages_remap = NULL; + } + + if (__iommu_dma_unmap) + { + __iommu_dma_unmap = NULL; + } + + if (dma_prep_coherent) + { + dma_prep_coherent = NULL; + } +} + +void *kaddr_lookup_name(const char *fname_raw) +{ + int i; + unsigned long kaddr; + char *fname; + char fname_lookup[KSYM_SYMBOL_LEN]; + + fname = kzalloc(strlen(fname_raw)+5, GFP_KERNEL); + if (!fname) { + return NULL; + } + strcpy(fname, fname_raw); + strcat(fname, "+0x0"); + fname[strlen(fname_raw) + 4] = '\0'; + + kaddr = (unsigned long) &sprint_symbol; + kaddr &= 0xfffffffffffff000; + + for (i = 0x0 ; i < 0x1000000; i++ ) + { + sprint_symbol(fname_lookup, kaddr); + + if (strncmp(fname_lookup, fname, strlen(fname)) == 0) + { + kfree(fname); + return (void *)kaddr; + } + kaddr += 0x2; + } + + kaddr = (unsigned long) &sprint_symbol; + kaddr &= 0xfffffffffffff000; + + for (i = 0x1000000; i >= 0 && kaddr; i--) { + sprint_symbol(fname_lookup, kaddr); + if (strncmp(fname_lookup, fname, strlen(fname)) == 0) + { + kfree(fname); + return (void *)kaddr; + } + kaddr -= 0x2; + } + kfree(fname); + return NULL; +} + +static int __init eswin_iommu_rsv_init(void) +{ + + klp_name = kaddr_lookup_name("kallsyms_lookup_name"); + if (klp_name == NULL) { + printk("cannot lookup name.\n"); + return -EINVAL; + } + __iommu_dma_free = (void *)klp_name("__iommu_dma_free"); + if (!__iommu_dma_free) + { + goto err; + } + + es_common_pages_remap = (void *)klp_name("dma_common_pages_remap"); + if (!es_common_pages_remap) + { + goto err; + } + + __iommu_dma_unmap = (void *)klp_name("__iommu_dma_unmap"); + if (!__iommu_dma_unmap) + { + goto err; + } + + dma_prep_coherent = (void *)klp_name("arch_dma_prep_coherent"); + if (!dma_prep_coherent) { + goto err; + } + + return 0; +err: + eswin_iommu_put_func(); + return -EINVAL; +} + +static void __exit eswin_iommu_rsv_exit(void) +{ + eswin_iommu_put_func(); +} + +module_init(eswin_iommu_rsv_init); +module_exit(eswin_iommu_rsv_exit); + +MODULE_LICENSE("GPL"); -- 2.47.0