From b18abf055d3043206dafe9789e77ed7b19dbd2af Mon Sep 17 00:00:00 2001 From: Sreenad Menon Date: Mon, 4 Oct 2021 20:18:23 +0530 Subject: [PATCH] swiotlb: Handle swiotlb region outside of linear-mapped memory Porting "swiotlb: Handle swiotlb region outside of linear-mapped memory" (Commit ID:cf0e9880897f635a2422a4479f498fcbc5b8468e)from msm-5.10. Change-Id: I9bb5297994ac1a75c45f91b4a45969b9bf9bdb27 Signed-off-by: Sreenad Menon Signed-off-by: Peng Yang --- include/linux/swiotlb.h | 19 ++++++++++++ kernel/dma/Kconfig | 12 ++++++++ kernel/dma/swiotlb.c | 68 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 7ed35dd3de6e..b308829db6de 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -130,6 +130,18 @@ unsigned int swiotlb_max_segment(void); size_t swiotlb_max_mapping_size(struct device *dev); bool is_swiotlb_active(struct device *dev); void __init swiotlb_adjust_size(unsigned long size); + +#ifdef CONFIG_SWIOTLB_NONLINEAR +int swiotlb_late_init_with_tblpaddr(char *tlb, + phys_addr_t tlb_paddr, unsigned long nslabs); +#else +static inline int swiotlb_late_init_with_tblpaddr(char *tlb, + phys_addr_t tlb_paddr, unsigned long nslabs) +{ + return -EINVAL; +} +#endif + #else static inline void swiotlb_init(bool addressing_limited, unsigned int flags) { @@ -162,6 +174,13 @@ static inline bool is_swiotlb_active(struct device *dev) static inline void swiotlb_adjust_size(unsigned long size) { } + +static inline int swiotlb_late_init_with_tblpaddr(char *tlb, + phys_addr_t tlb_paddr, unsigned long nslabs) +{ + return -EINVAL; +} + #endif /* CONFIG_SWIOTLB */ extern void swiotlb_print_info(void); diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 56866aaa2ae1..b4e42f1f0839 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -93,6 +93,18 @@ config DMA_RESTRICTED_POOL and . If unsure, say "n". +config SWIOTLB_NONLINEAR + bool "Allow swiotlb to use non-linear memory as bounce buffers" + depends on SWIOTLB + help + This allows swiotlb driver to work with memory regions where + physical to virtual address translations can't be done using APIs + such as phys_to_virt. These could be reserved memory regions that + are not mapped by default or could be seen as "device" memory + accessed via ioremap(). + + If unsure, say "n". + # # Should be selected if we can mmap non-coherent mappings to userspace. # The only thing that is really required is a way to set an uncached bit diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index cb50f8d38360..c0e877cf4348 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -69,6 +69,16 @@ struct io_tlb_mem io_tlb_default_mem; phys_addr_t swiotlb_unencrypted_base; +#ifdef CONFIG_SWIOTLB_NONLINEAR +phys_addr_t io_tlb_start, io_tlb_end; +static unsigned long io_tlb_nslabs; +static char *io_tlb_vstart; + +static inline unsigned char *swiotlb_phys_to_virt(phys_addr_t tlb_addr); +#else +#define swiotlb_phys_to_virt phys_to_virt +#endif + static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT; static int __init @@ -194,7 +204,7 @@ void __init swiotlb_update_mem_attributes(void) static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start, unsigned long nslabs, unsigned int flags, bool late_alloc) { - void *vaddr = phys_to_virt(start); + void *vaddr = swiotlb_phys_to_virt(start); unsigned long bytes = nslabs << IO_TLB_SHIFT, i; mem->nslabs = nslabs; @@ -352,6 +362,47 @@ retry: return 0; } +#ifdef CONFIG_SWIOTLB_NONLINEAR +static int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) +{ + struct io_tlb_mem *mem = &io_tlb_default_mem; + unsigned long bytes = nslabs << IO_TLB_SHIFT; + + /* protect against double initialization */ + if (WARN_ON_ONCE(mem->nslabs)) + return -ENOMEM; + + mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(array_size(sizeof(*mem->slots), nslabs))); + if (!mem->slots) + return -ENOMEM; + + set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT); + swiotlb_init_io_tlb_mem(mem, io_tlb_start, nslabs, true); + + swiotlb_print_info(); + return 0; +} + +int swiotlb_late_init_with_tblpaddr(char *tlb, + phys_addr_t tlb_paddr, unsigned long nslabs) +{ + unsigned long bytes; + + if (io_tlb_start) + return -EBUSY; + + bytes = nslabs << IO_TLB_SHIFT; + io_tlb_nslabs = nslabs; + io_tlb_start = tlb_paddr; + io_tlb_vstart = tlb; + io_tlb_end = io_tlb_start + bytes; + + return swiotlb_late_init_with_tbl(tlb, nslabs); +} +EXPORT_SYMBOL(swiotlb_late_init_with_tblpaddr); +#endif /* CONFIG_SWIOTLB_NONLINEAR */ + void __init swiotlb_exit(void) { struct io_tlb_mem *mem = &io_tlb_default_mem; @@ -389,6 +440,13 @@ static unsigned int swiotlb_align_offset(struct device *dev, u64 addr) return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1); } +#ifdef CONFIG_SWIOTLB_NONLINEAR +static inline unsigned char *swiotlb_phys_to_virt(phys_addr_t tlb_addr) +{ + return (unsigned char *)(io_tlb_vstart + (tlb_addr - io_tlb_start)); +} +#endif + /* * Bounce: copy the swiotlb buffer from or back to the original dma location */ @@ -400,7 +458,7 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size phys_addr_t orig_addr = mem->slots[index].orig_addr; size_t alloc_size = mem->slots[index].alloc_size; unsigned long pfn = PFN_DOWN(orig_addr); - unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; + unsigned char *vaddr = swiotlb_phys_to_virt(tlb_addr); unsigned int tlb_offset, orig_addr_offset; if (orig_addr == INVALID_PHYS_ADDR) @@ -620,6 +678,9 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE); return tlb_addr; } +#ifdef CONFIG_SWIOTLB_NONLINEAR +EXPORT_SYMBOL(swiotlb_tbl_unmap_single); +#endif static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr) { @@ -731,6 +792,9 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size, arch_sync_dma_for_device(swiotlb_addr, size, dir); return dma_addr; } +#ifdef CONFIG_SWIOTLB_NONLINEAR +EXPORT_SYMBOL(swiotlb_map); +#endif size_t swiotlb_max_mapping_size(struct device *dev) {