/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2019 MediaTek Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef CONFIG_MTK_SMI_EXT #include "smi_public.h" #endif #ifdef CONFIG_MTK_IOMMU_V2 #include "mach/mt_iommu.h" #endif #ifdef PSEUDO_M4U_TEE_SERVICE_ENABLE #include "pseudo_m4u_sec.h" #endif #ifdef M4U_GZ_SERVICE_ENABLE #include "pseudo_m4u_gz_sec.h" #endif #include "pseudo_m4u_log.h" #if defined(CONFIG_TRUSTONIC_TEE_SUPPORT) && \ !defined(CONFIG_MTK_TEE_GP_SUPPORT) #include "mobicore_driver_api.h" static const struct mc_uuid_t m4u_drv_uuid = M4U_DRV_UUID; static struct mc_session_handle m4u_dci_session; static struct m4u_msg *m4u_dci_msg; #endif #ifndef ARM_MAPPING_ERROR #define ARM_MAPPING_ERROR (~(dma_addr_t)0x0) #endif static struct m4u_client_t *ion_m4u_client; int m4u_log_level = 2; int m4u_log_to_uart = 2; static LIST_HEAD(pseudo_sglist); /* this is the mutex lock to protect mva_sglist->list*/ static spinlock_t pseudo_list_lock; static const struct of_device_id mtk_pseudo_of_ids[] = { { .compatible = "mediatek,mt-pseudo_m4u",}, {} }; static const struct of_device_id mtk_pseudo_port_of_ids[] = { { .compatible = "mediatek,mt-pseudo_m4u-port",}, {} }; #define M4U_L2_ENABLE 1 /* garbage collect related */ #define MVA_REGION_FLAG_NONE 0x0 #define MVA_REGION_HAS_TLB_RANGE 0x1 #define MVA_REGION_REGISTER 0x2 static unsigned long pseudo_mmubase[TOTAL_M4U_NUM]; static unsigned long pseudo_larbbase[SMI_LARB_NR]; static struct m4u_device *pseudo_mmu_dev; static inline unsigned int pseudo_readreg32( unsigned long base, unsigned int offset) { unsigned int val; val = ioread32((void *)(base + offset)); return val; } static inline void pseudo_writereg32(unsigned long base, unsigned int offset, unsigned int val) { mt_reg_sync_writel(val, (void *)(base + offset)); } static inline void pseudo_set_reg_by_mask( unsigned long M4UBase, unsigned int reg, unsigned long mask, unsigned int val) { unsigned int regval; regval = pseudo_readreg32(M4UBase, reg); regval = (regval & (~mask)) | val; pseudo_writereg32(M4UBase, reg, regval); } static inline unsigned int pseudo_get_reg_by_mask( unsigned long Base, unsigned int reg, unsigned int mask) { return pseudo_readreg32(Base, reg) & mask; } static inline int m4u_kernel2user_port(int kernelport) { return kernelport; } static inline int m4u_get_larbid(int kernel_port) { return MTK_IOMMU_TO_LARB(kernel_port); } static int m4u_port_2_larb_port(int kernel_port) { return MTK_IOMMU_TO_PORT(kernel_port); } static char *m4u_get_module_name(int portID) { return iommu_get_port_name(portID); } static int get_pseudo_larb(unsigned int port) { int i, j, fake_nr; fake_nr = ARRAY_SIZE(pseudo_dev_larb_fake); for (i = 0; i < fake_nr; i++) { for (j = 0; j < 32; j++) { if (pseudo_dev_larb_fake[i].port[j] == -1) break; if (pseudo_dev_larb_fake[i].port[j] == port) { return i; } } } return -1; } struct device *pseudo_get_larbdev(int portid) { struct pseudo_device *pseudo = NULL; unsigned int larbid, larbport, fake_nr; int index = -1; fake_nr = ARRAY_SIZE(pseudo_dev_larb_fake); larbid = m4u_get_larbid(portid); larbport = m4u_port_2_larb_port(portid); if (larbid >= (SMI_LARB_NR + fake_nr) || larbport >= 32) goto out; if (larbid >= 0 && larbid < SMI_LARB_NR) { pseudo = &pseudo_dev_larb[larbid]; } else if (larbid < (SMI_LARB_NR + fake_nr)) { index = get_pseudo_larb(portid); if (index >= 0 && index < fake_nr) pseudo = &pseudo_dev_larb_fake[index]; } out: if (!pseudo) { #if (CONFIG_MTK_IOMMU_PGTABLE_EXT == 32) /* * for 34bit IOVA space, boundary is mandatory * we cannot use a default device for iova mapping */ #ifndef CONFIG_FPGA_EARLY_PORTING index = get_pseudo_larb(M4U_PORT_OVL_DEBUG); if (index >= 0 && index < fake_nr) pseudo = &pseudo_dev_larb_fake[index]; #else pseudo = &pseudo_dev_larb_fake[2]; #endif #endif } if (pseudo && pseudo->dev) return pseudo->dev; M4U_ERR("err, p:%d(%d-%d) index=%d fake_nr=%d smi_nr=%d\n", portid, larbid, larbport, index, fake_nr, SMI_LARB_NR); return NULL; } int larb_clock_on(int larb, bool config_mtcmos) { int ret = 0; #ifdef CONFIG_MTK_SMI_EXT if (larb >= SMI_LARB_NR || larb < 0) { M4U_MSG("invalid larb:%d, total:%d\n", larb, SMI_LARB_NR); return -1; } if (!pseudo_dev_larb[larb].dev || !strcmp(pseudo_larbname[larb], "m4u_none")) { M4U_MSG("ignore the invalid larb:%d\n", larb); return 0; } if (larb < ARRAY_SIZE(pseudo_larb_clk_name)) ret = smi_bus_prepare_enable(larb, pseudo_larb_clk_name[larb]); if (ret) { M4U_ERR("err larb %d\n", larb); ret = -1; } #endif return ret; } void larb_clock_off(int larb, bool config_mtcmos) { #ifdef CONFIG_MTK_SMI_EXT int ret = 0; if (larb >= SMI_LARB_NR || larb < 0) { M4U_MSG("invalid larb:%d, total:%d\n", larb, SMI_LARB_NR); return; } if (!pseudo_dev_larb[larb].dev || !strcmp(pseudo_larbname[larb], "m4u_none")) { M4U_MSG("ignore the invalid larb:%d\n", larb); return; } if (larb < ARRAY_SIZE(pseudo_larb_clk_name)) ret = smi_bus_disable_unprepare(larb, pseudo_larb_clk_name[larb]); if (ret) M4U_MSG("err: larb %d\n", larb); #endif } #ifdef M4U_MTEE_SERVICE_ENABLE #include "tz_cross/trustzone.h" #include "trustzone/kree/system.h" #include "tz_cross/ta_m4u.h" KREE_SESSION_HANDLE m4u_session; bool m4u_tee_en; static DEFINE_MUTEX(gM4u_port_tee); static int pseudo_session_init(void) { TZ_RESULT ret; ret = KREE_CreateSession(TZ_TA_M4U_UUID, &m4u_session); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u CreateSession error %d\n", ret); return -1; } M4U_MSG("create session : 0x%x\n", (unsigned int)m4u_session); m4u_tee_en = true; return 0; } int m4u_larb_restore_sec(unsigned int larb_idx) { MTEEC_PARAM param[4]; uint32_t paramTypes; TZ_RESULT ret; if (!m4u_tee_en) /*tee may not init*/ return -2; if (larb_idx == 0 || larb_idx == 4) { /*only support disp*/ param[0].value.a = larb_idx; paramTypes = TZ_ParamTypes1(TZPT_VALUE_INPUT); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_LARB_REG_RESTORE, paramTypes, param); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u reg backup ServiceCall error %d\n", ret); return -1; } } return 0; } int m4u_larb_backup_sec(unsigned int larb_idx) { MTEEC_PARAM param[4]; uint32_t paramTypes; TZ_RESULT ret; if (!m4u_tee_en) /*tee may not init */ return -2; if (larb_idx == 0 || larb_idx == 4) { /*only support disp*/ param[0].value.a = larb_idx; paramTypes = TZ_ParamTypes1(TZPT_VALUE_INPUT); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_LARB_REG_BACKUP, paramTypes, param); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u reg backup ServiceCall error %d\n", ret); return -1; } } return 0; } int smi_reg_backup_sec(void) { uint32_t paramTypes; TZ_RESULT ret; paramTypes = TZ_ParamTypes1(TZPT_NONE); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_REG_BACKUP, paramTypes, NULL); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u reg backup ServiceCall error %d\n", ret); return -1; } return 0; } int smi_reg_restore_sec(void) { uint32_t paramTypes; TZ_RESULT ret; paramTypes = TZ_ParamTypes1(TZPT_NONE); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_REG_RESTORE, paramTypes, NULL); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u reg backup ServiceCall error %d\n", ret); return -1; } return 0; } #if 0 int pseudo_do_config_port(struct M4U_PORT_STRUCT *pM4uPort) { MTEEC_PARAM param[4]; uint32_t paramTypes; TZ_RESULT ret; /* do not config port if session has not been inited. */ if (!m4u_session) return 0; param[0].value.a = pM4uPort->ePortID; param[0].value.b = pM4uPort->Virtuality; param[1].value.a = pM4uPort->Distance; param[1].value.b = pM4uPort->Direction; paramTypes = TZ_ParamTypes2(TZPT_VALUE_INPUT, TZPT_VALUE_INPUT); mutex_lock(&gM4u_port_tee); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_CONFIG_PORT, paramTypes, param); mutex_unlock(&gM4u_port_tee); if (ret != TZ_RESULT_SUCCESS) M4U_MSG("ServiceCall error 0x%x\n", ret); return 0; } #endif static int pseudo_sec_init(unsigned int u4NonSecPa, unsigned int L2_enable, unsigned int *security_mem_size) { MTEEC_PARAM param[4]; uint32_t paramTypes; TZ_RESULT ret; param[0].value.a = u4NonSecPa; param[0].value.b = L2_enable; param[1].value.a = 1; paramTypes = TZ_ParamTypes2(TZPT_VALUE_INPUT, TZPT_VALUE_OUTPUT); ret = KREE_TeeServiceCall(m4u_session, M4U_TZCMD_SEC_INIT, paramTypes, param); if (ret != TZ_RESULT_SUCCESS) { M4U_MSG("m4u sec init error 0x%x\n", ret); return -1; } *security_mem_size = param[1].value.a; return 0; } #if 0 /* the caller should enable smi clock, it should be only called by mtk_smi.c*/ int pseudo_config_port_tee(int kernelport) { struct M4U_PORT_STRUCT pM4uPort; pM4uPort.ePortID = m4u_kernel2user_port(kernelport); pM4uPort.Virtuality = 1; pM4uPort.Distance = 1; pM4uPort.Direction = 1; #ifdef M4U_MTEE_SERVICE_ENABLE return pseudo_do_config_port(&pM4uPort); #else return 0; #endif } #endif #endif /* make sure the va size is page aligned to get the continues iova. */ int m4u_va_align(unsigned long *addr, unsigned long *size) { int offset, remain; /* we need to align the bufaddr to make sure the iova is continues */ offset = *addr & (M4U_PAGE_SIZE - 1); if (offset) { *addr &= ~(M4U_PAGE_SIZE - 1); *size += offset; } /* make sure we alloc one page size iova at least */ remain = *size % M4U_PAGE_SIZE; if (remain) *size += M4U_PAGE_SIZE - remain; /* dma32 would skip the last page, we added it here */ /* *size += PAGE_SIZE; */ return offset; } int pseudo_get_reg_of_path(unsigned int port, bool is_va, unsigned int *reg, unsigned int *mask, unsigned int *value) { unsigned long larb_base; unsigned int larb, larb_port; larb = m4u_get_larbid(port); larb_port = m4u_port_2_larb_port(port); larb_base = pseudo_larbbase[larb]; if (!larb_base) { M4U_DBG("larb(%u) not existed, no need of config\n", larb); return -1; } *reg = larb_base + SMI_LARB_NON_SEC_CONx(larb_port); *mask = F_SMI_MMU_EN; if (is_va) *value = 1; else *value = 0; return 0; } int m4u_get_boundary(int port) { #if (CONFIG_MTK_IOMMU_PGTABLE_EXT > 32) struct device *dev = pseudo_get_larbdev(port); if (!dev) return -1; return mtk_iommu_get_boundary_id(dev); #else return 0; #endif } int m4u_get_dma_buf_port(struct device *dev) { return mtk_iommu_get_port_id(dev); } static inline int pseudo_config_port(struct M4U_PORT_STRUCT *pM4uPort, bool is_user) { /* all the port will be attached by dma and configed by iommu driver */ unsigned long larb_base; unsigned int larb, larb_port, bit32 = 0; unsigned int old_value = 0, value; int ret = 0; char *name; larb = m4u_get_larbid(pM4uPort->ePortID); larb_port = m4u_port_2_larb_port(pM4uPort->ePortID); name = iommu_get_port_name(pM4uPort->ePortID); if (is_user && name && strcmp(name, pM4uPort->name)) { M4U_MSG("port:%d name(%s) not matched(%s)\n", pM4uPort->ePortID, pM4uPort->name, name); report_custom_config_port( name, pM4uPort->name, pM4uPort->ePortID); return -1; } if (is_user && !pM4uPort->Virtuality) { M4U_MSG("port:%d name(%s) user(%d) cannot bypass iommu\n", pM4uPort->ePortID, pM4uPort->name, is_user); report_custom_config_port( name, pM4uPort->name, pM4uPort->ePortID); return -5; } if (pM4uPort->Virtuality) { bit32 = m4u_get_boundary(pM4uPort->ePortID); if (bit32 < 0 || bit32 >= (1 << (CONFIG_MTK_IOMMU_PGTABLE_EXT - 32))) { M4U_MSG("enable larb%d fail\n", larb); return -2; } } larb_base = pseudo_larbbase[larb]; if (!larb_base) { M4U_DBG("larb %d not existed, no need of config\n", larb); return -3; } old_value = pseudo_readreg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port)); if (pM4uPort->Virtuality) { value = (old_value & ~F_SMI_ADDR_BIT32) | (bit32 << 8) | (bit32 << 10) | (bit32 << 12) | (bit32 << 14) | F_SMI_MMU_EN; } else { value = old_value & ~F_SMI_ADDR_BIT32 & ~F_SMI_MMU_EN; } if (value == old_value) goto out; pseudo_writereg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port), value); /* debug use */ if (value != pseudo_readreg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port))) { M4U_ERR( "%d(%d-%d),vir=%d, bd=%d, old=0x%x, expect=0x%x, cur=0x%x\n", pM4uPort->ePortID, larb, larb_port, pM4uPort->Virtuality, bit32, old_value, value, pseudo_readreg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port))); ret = -4; } M4U_INFO("%s, l%d-p%d, userspace:%d switch fr %d to %d, bd:%d\n", m4u_get_module_name(pM4uPort->ePortID), larb, larb_port, is_user, old_value, value, bit32); out: return ret; } int pseudo_dump_port(int port, bool ignore_power) { /* all the port will be attached by dma and configed by iommu driver */ unsigned long larb_base; unsigned int larb, larb_port; unsigned int regval = 0; int ret = 0; larb = m4u_get_larbid(port); larb_port = m4u_port_2_larb_port(port); if (larb >= SMI_LARB_NR || larb_port >= 32) { M4U_MSG("port:%d, larb:%d is fake, or port:%d invalid\n", port, larb, larb_port); return 0; } larb_base = pseudo_larbbase[larb]; if (!larb_base) { M4U_DBG("larb:%d not existed, no need of config\n", larb); return 0; } if (!ignore_power) { ret = larb_clock_on(larb, 1); if (ret < 0) { M4U_MSG("enable larb%d fail\n", larb); return ret; } } regval = pseudo_readreg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port)); M4U_MSG( "port %d(%d-%d): %s -- config:0x%x, mmu:0x%x, bit32:0x%x\n", port, larb, larb_port, iommu_get_port_name(MTK_M4U_ID(larb, larb_port)), regval, regval & F_SMI_MMU_EN, F_SMI_ADDR_BIT32_VAL(regval)); if (!ignore_power) larb_clock_off(larb, 1); return ret; } int pseudo_dump_all_port_status(struct seq_file *s) { /* all the port will be attached by dma and configed by iommu driver */ unsigned long larb_base; unsigned int larb, larb_port, count; unsigned int regval = 0, regval_sec = 0; int ret = 0; for (larb = 0; larb < SMI_LARB_NR; larb++) { larb_base = pseudo_larbbase[larb]; if (!larb_base) { M4U_DBG("larb(%u) not existed, no need of config\n", larb); continue; } ret = larb_clock_on(larb, 1); if (ret < 0) { M4U_ERR("err enable larb%d\n", larb); continue; } count = mtk_iommu_get_larb_port_count(larb); M4U_PRINT_SEQ(s, "====== larb:%d, total %d ports ======\n", larb, count); for (larb_port = 0; larb_port < count; larb_port++) { regval = pseudo_readreg32(larb_base, SMI_LARB_NON_SEC_CONx(larb_port)); #ifdef SMI_LARB_SEC_CON_EN regval_sec = pseudo_readreg32(larb_base, SMI_LARB_SEC_CONx(larb_port)); #else regval_sec = mtk_iommu_dump_sec_larb(larb, larb_port); #endif M4U_PRINT_SEQ(s, "port%d: %s -- normal:0x%x, secure:0x%x mmu:0x%x, bit32:0x%x\n", larb_port, iommu_get_port_name( MTK_M4U_ID(larb, larb_port)), regval, regval_sec, regval & F_SMI_MMU_EN, F_SMI_ADDR_BIT32_VAL(regval)); } larb_clock_off(larb, 1); } return ret; } static int m4u_put_unlock_page(struct page *page) { if (!page) return 0; if (!PageReserved(page)) SetPageDirty(page); put_page(page); return 0; } /* to-do: need modification to support 4G DRAM */ static phys_addr_t m4u_user_v2p(unsigned long va) { unsigned long pageOffset = (va & (M4U_PAGE_SIZE - 1)); pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; phys_addr_t pa; if (!current) { M4U_MSG("%s va 0x%lx, err current\n", __func__, va); return 0; } if (!current->mm) { M4U_MSG("warning: tgid=0x%x, name=%s, va 0x%lx\n", current->tgid, current->comm, va); return 0; } pgd = pgd_offset(current->mm, va); /* what is tsk->mm */ if (pgd_none(*pgd) || pgd_bad(*pgd)) { M4U_DBG("%s va=0x%lx, err pgd\n", __func__, va); return 0; } pud = pud_offset(pgd, va); if (pud_none(*pud) || pud_bad(*pud)) { M4U_DBG("%s va=0x%lx, err pud\n", __func__, va); return 0; } pmd = pmd_offset(pud, va); if (pmd_none(*pmd) || pmd_bad(*pmd)) { M4U_MSG("%s va=0x%lx, err pmd\n", __func__, va); return 0; } pte = pte_offset_map(pmd, va); if (pte_present(*pte)) { pa = (pte_val(*pte) & (PHYS_MASK) & (PAGE_MASK)) | pageOffset; pte_unmap(pte); return pa; } pte_unmap(pte); M4U_DBG("%s, va=0x%lx err pte\n", __func__, va); return 0; } int __m4u_get_user_pages(int eModuleID, struct task_struct *tsk, struct mm_struct *mm, unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct *vmas) { int i, ret; unsigned long vm_flags; if (nr_pages <= 0) return 0; /* VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); */ if (!!pages != !!(gup_flags & FOLL_GET)) { M4U_ERR("error: !!pages != !!(gup_flags & FOLL_GET),"); M4U_MSG("gup_flags & FOLL_GET=0x%x\n", gup_flags & FOLL_GET); } /* * Require read or write permissions. * If FOLL_FORCE is set, we only require the "MAY" flags. */ vm_flags = (gup_flags & FOLL_WRITE) ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); vm_flags &= (gup_flags & FOLL_FORCE) ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); i = 0; M4U_DBG("Try to get_user_pages from start vaddr 0x%lx with %d pages\n", start, nr_pages); do { struct vm_area_struct *vma; M4U_DBG("For a new vma area from 0x%lx\n", start); if (vmas) vma = vmas; else vma = find_extend_vma(mm, start); if (!vma) { M4U_ERR("error: vma not find, start=0x%x, module=%d\n", (unsigned int)start, eModuleID); return i ? i : -EFAULT; } if (((~vma->vm_flags) & (VM_IO | VM_PFNMAP | VM_SHARED | VM_WRITE)) == 0) { M4U_ERR("error: m4u_get_pages: bypass garbage pages!"); M4U_MSG("vma->vm_flags=0x%x, start=0x%lx, module=%d\n", (unsigned int)(vma->vm_flags), start, eModuleID); return i ? i : -EFAULT; } if (vma->vm_flags & VM_IO) M4U_DBG("warning: vma is marked as VM_IO\n"); if (vma->vm_flags & VM_PFNMAP) { M4U_MSG ("err vma permission,0x%x,0x%lx,%d\n", (unsigned int)(vma->vm_flags), start, eModuleID); M4U_MSG ("maybe remap of un-permit vm_flags!\n"); /* m4u_dump_maps(start); */ return i ? i : -EFAULT; } if (!(vm_flags & vma->vm_flags)) { M4U_MSG ("%s err flag, 0x%x,0x%x,0x%lx,%d\n", __func__, (unsigned int)vm_flags, (unsigned int)(vma->vm_flags), start, eModuleID); /* m4u_dump_maps(start); */ return i ? : -EFAULT; } do { struct page *page = NULL; unsigned int foll_flags = gup_flags; /* * If we have a pending SIGKILL, don't keep faulting * pages and potentially allocating memory. */ if (unlikely(fatal_signal_pending(current))) return i ? i : -ERESTARTSYS; ret = get_user_pages(start, 1, (vma->vm_flags & VM_WRITE), &page, NULL); if (ret == 1) pages[i] = page; while (!page) { int ret; ret = handle_mm_fault(vma, start, (foll_flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0); if (ret & VM_FAULT_ERROR) { if (ret & VM_FAULT_OOM) { M4U_ERR("error: no memory,"); M4U_MSG("addr:0x%lx (%d %d)\n", start, i, eModuleID); /* m4u_dump_maps(start); */ return i ? i : -ENOMEM; } if (ret & (VM_FAULT_HWPOISON | VM_FAULT_SIGBUS)) { M4U_ERR("error: invalid va,"); M4U_MSG("addr:0x%lx (%d %d)\n", start, i, eModuleID); /* m4u_dump_maps(start); */ return i ? i : -EFAULT; } } if (ret & VM_FAULT_MAJOR) tsk->maj_flt++; else tsk->min_flt++; /* * The VM_FAULT_WRITE bit tells us that * do_wp_page has broken COW when necessary, * even if maybe_mkwrite decided not to set * pte_write. We can thus safely do subsequent * page lookups as if they were reads. But only * do so when looping for pte_write is futile: * in some cases userspace may also be wanting * to write to the gotten user page, which a * read fault here might prevent (a readonly * page might get reCOWed by userspace write). */ if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) foll_flags &= ~FOLL_WRITE; ret = get_user_pages(start, 1, (vma->vm_flags & VM_WRITE), &page, NULL); if (ret == 1) pages[i] = page; } if (IS_ERR(page)) { M4U_ERR("error: faulty page is returned,"); M4U_MSG("addr:0x%lx (%d %d)\n", start, i, eModuleID); /* m4u_dump_maps(start); */ return i ? i : PTR_ERR(page); } i++; start += M4U_PAGE_SIZE; nr_pages--; } while (nr_pages && start < vma->vm_end); } while (nr_pages); return i; } /* refer to mm/memory.c:get_user_pages() */ int m4u_get_user_pages(int eModuleID, struct task_struct *tsk, struct mm_struct *mm, unsigned long start, int nr_pages, int write, int force, struct page **pages, struct vm_area_struct *vmas) { int flags = FOLL_TOUCH; if (pages) flags |= FOLL_GET; if (write) flags |= FOLL_WRITE; if (force) flags |= FOLL_FORCE; return __m4u_get_user_pages(eModuleID, tsk, mm, start, nr_pages, flags, pages, vmas); } /* /> m4u driver internal use function */ /* /> should not be called outside m4u kernel driver */ static int m4u_get_pages(int eModuleID, unsigned long BufAddr, unsigned long BufSize, unsigned long *pPhys) { int ret; unsigned int i; int page_num; unsigned long start_pa; unsigned int write_mode = 0; struct vm_area_struct *vma = NULL; M4U_MSG("%s: module=%s,BufAddr=0x%lx,BufSize=%ld,0x%lx\n", __func__, m4u_get_module_name(eModuleID), BufAddr, BufSize, PAGE_OFFSET); /* caculate page number */ page_num = (BufSize + (BufAddr & 0xfff)) / M4U_PAGE_SIZE; if ((BufAddr + BufSize) & 0xfff) page_num++; if (BufSize > 200*1024*1024) { M4U_MSG("alloc size=0x%lx, bigger than limit=0x%x\n", BufSize, 200*1024*1024); return -EFAULT; } if (BufAddr < PAGE_OFFSET) { /* from user space */ start_pa = m4u_user_v2p(BufAddr); if (!start_pa) { M4U_ERR("err v2p\n"); return -EFAULT; } down_read(¤t->mm->mmap_sem); vma = find_vma(current->mm, BufAddr); if (vma == NULL) { M4U_MSG("cannot find vma:module=%s,va=0x%lx-0x%lx\n", m4u_get_module_name(eModuleID), BufAddr, BufSize); up_read(¤t->mm->mmap_sem); return -1; } write_mode = (vma->vm_flags & VM_WRITE) ? 1 : 0; if ((vma->vm_flags) & VM_PFNMAP) { unsigned long bufEnd = BufAddr + BufSize - 1; if (bufEnd > vma->vm_end + M4U_PAGE_SIZE) { M4U_MSG("%s:n=%d,%s,v=0x%lx,s=0x%lx,f=0x%x\n", __func__, page_num, m4u_get_module_name(eModuleID), BufAddr, BufSize, (unsigned int)vma->vm_flags); M4U_MSG("but vma is: start=0x%lx,end=0x%lx\n", (unsigned long)vma->vm_start, (unsigned long)vma->vm_end); up_read(¤t->mm->mmap_sem); return -1; } up_read(¤t->mm->mmap_sem); for (i = 0; i < page_num; i++) { unsigned long va_align = BufAddr & (~M4U_PAGE_MASK); unsigned long va_next; int err_cnt; unsigned int flags; for (err_cnt = 0; err_cnt < 30; err_cnt++) { va_next = va_align + 0x1000 * i; flags = (vma->vm_flags & VM_WRITE) ? FAULT_FLAG_WRITE : 0; *(pPhys + i) = m4u_user_v2p(va_next); if (!*(pPhys + i) && (va_next >= vma->vm_start) && (va_next <= vma->vm_end)) { handle_mm_fault(vma, va_next, flags); cond_resched(); } else break; } if (err_cnt > 20) { M4U_MSG("fault_cnt %d,0x%lx,%d,0x%x\n", err_cnt, BufAddr, i, page_num); M4U_MSG("%s, va=0x%lx-0x%lx, 0x%x\n", m4u_get_module_name(eModuleID), BufAddr, BufSize, (unsigned int)vma->vm_flags); up_read(¤t->mm->mmap_sem); return -1; } } M4U_MSG("%s, va=0x%lx, size=0x%lx, vm_flag=0x%x\n", m4u_get_module_name(eModuleID), BufAddr, BufSize, (unsigned int)vma->vm_flags); } else { ret = m4u_get_user_pages(eModuleID, current, current->mm, BufAddr, page_num, write_mode, 0, (struct page **)pPhys, vma); up_read(¤t->mm->mmap_sem); if (ret < page_num) { /* release pages first */ for (i = 0; i < ret; i++) m4u_put_unlock_page((struct page *) (*(pPhys + i))); if (unlikely(fatal_signal_pending(current))) { M4U_ERR("error: receive sigkill when"); M4U_MSG("get_user_pages,%d %d,%s,%s\n", page_num, ret, m4u_get_module_name(eModuleID), current->comm); } /* * return value bigger than 0 but smaller * than expected, trigger red screen */ if (ret > 0) { M4U_ERR("error:page_num=%d, return=%d", page_num, ret); M4U_MSG("module=%s, current_proc:%s\n", m4u_get_module_name(eModuleID), current->comm); M4U_MSG("maybe the allocated VA size"); M4U_MSG("is smaller than the size"); M4U_MSG("config to m4u_alloc_mva()!"); } else { M4U_ERR("error: page_num=%d,return=%d", page_num, ret); M4U_MSG("module=%s, current_proc:%s\n", m4u_get_module_name(eModuleID), current->comm); M4U_MSG("maybe the VA is deallocated"); M4U_MSG("before call m4u_alloc_mva(),"); M4U_MSG("or no VA has beallocated!"); } return -EFAULT; } for (i = 0; i < page_num; i++) { *(pPhys + i) = page_to_phys((struct page *) (*(pPhys + i))); } } } else { /* from kernel space */ #ifndef CONFIG_ARM64 if (BufAddr >= VMALLOC_START && BufAddr <= VMALLOC_END) { /* vmalloc */ struct page *ppage; for (i = 0; i < page_num; i++) { ppage = vmalloc_to_page((unsigned int *) (BufAddr + i * M4U_PAGE_SIZE)); *(pPhys + i) = page_to_phys(ppage) & 0xfffff000; } } else { /* kmalloc */ #endif for (i = 0; i < page_num; i++) *(pPhys + i) = virt_to_phys((void *)((BufAddr & 0xfffff000) + i * M4U_PAGE_SIZE)); #ifndef CONFIG_ARM64 } #endif } /*get_page_exit:*/ return page_num; } /* make a sgtable for virtual buffer */ #define M4U_GET_PAGE_NUM(va, size) \ (((((unsigned long)va) & (M4U_PAGE_SIZE-1)) +\ (size) + (M4U_PAGE_SIZE-1)) >> 12) /* * the upper caller should make sure the va is page aligned * get the pa from va, and calc the size of pa, fill the pa into the sgtable. * if the va does not have pages, fill the sg_dma_address with pa. * We need to modify the arm_iommu_map_sg inter face. */ struct sg_table *pseudo_get_sg(int portid, unsigned long va, int size) { int i, page_num, ret, have_page, get_pages; struct sg_table *table; struct scatterlist *sg; struct page *page; struct vm_area_struct *vma = NULL; unsigned long *pPhys; page_num = M4U_GET_PAGE_NUM(va, size); table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) { M4U_MSG("kzalloc failed table is null.\n"); return NULL; } ret = sg_alloc_table(table, page_num, GFP_KERNEL); if (ret) { M4U_MSG("sg alloc table failed %d. page_num is %d\n", ret, page_num); kfree(table); return NULL; } sg = table->sgl; pPhys = vmalloc(page_num * sizeof(unsigned long *)); if (pPhys == NULL) { M4U_MSG("m4u_fill_pagetable : error to vmalloc %d*4 size\n", page_num); goto err_free; } get_pages = m4u_get_pages(portid, va, size, pPhys); if (get_pages <= 0) { M4U_DBG("Error : m4u_get_pages failed\n"); goto err_free; } down_read(¤t->mm->mmap_sem); vma = find_vma(current->mm, va); if (vma && vma->vm_flags & VM_PFNMAP) have_page = 0; else have_page = 1; up_read(¤t->mm->mmap_sem); for (i = 0; i < page_num; i++) { va += i * M4U_PAGE_SIZE; if (((pPhys[i] & (M4U_PAGE_SIZE - 1)) != (va & (M4U_PAGE_SIZE - 1)) || !pPhys[i]) && (i != page_num - 1)) { M4U_MSG("m4u user v2p failed, pa is 0x%lx\n", pPhys[i]); } if (!pPhys[i] && i < page_num - 1) { M4U_MSG("get pa failed, pa is 0. 0x%lx, %d, %d, %s\n", va, page_num, i, iommu_get_port_name(portid)); goto err_free; } if (have_page) { page = phys_to_page(pPhys[i]); sg_set_page(sg, page, M4U_PAGE_SIZE, 0); sg_dma_len(sg) = M4U_PAGE_SIZE; } else { /* * the pa must not be set to zero or DMA would omit * this page and then the mva allocation would be * failed. So just make the last pages's pa as it's * previous page plus page size. It's ok to do so since * the hw would not access this very last page. DMA * would like to ovmit the very last sg if the pa is 0 */ if ((i == page_num - 1) && (pPhys[i] == 0)) { /* i == 0 should be take care of specially. */ if (i) pPhys[i] = pPhys[i - 1] + M4U_PAGE_SIZE; else pPhys[i] = M4U_PAGE_SIZE; } sg_dma_address(sg) = pPhys[i]; sg_dma_len(sg) = M4U_PAGE_SIZE; } sg = sg_next(sg); } vfree(pPhys); return table; err_free: sg_free_table(table); kfree(table); if (pPhys) vfree(pPhys); return NULL; } static struct sg_table *pseudo_find_sgtable(unsigned long mva) { struct mva_sglist *entry; list_for_each_entry(entry, &pseudo_sglist, list) { if (entry->mva == mva) { return entry->table; } } return NULL; } static struct sg_table *pseudo_add_sgtable(struct mva_sglist *mva_sg) { struct sg_table *table; unsigned long flags = 0; spin_lock_irqsave(&pseudo_list_lock, flags); table = pseudo_find_sgtable(mva_sg->mva); if (table) { spin_unlock_irqrestore(&pseudo_list_lock, flags); return table; } table = mva_sg->table; list_add(&mva_sg->list, &pseudo_sglist); spin_unlock_irqrestore(&pseudo_list_lock, flags); M4U_DBG("adding pseudo_sglist, mva = 0x%lx\n", mva_sg->mva); return table; } static struct m4u_buf_info_t *pseudo_alloc_buf_info(void) { struct m4u_buf_info_t *pList = NULL; pList = kzalloc(sizeof(struct m4u_buf_info_t), GFP_KERNEL); if (pList == NULL) { M4U_MSG("pList=0x%p\n", pList); return NULL; } M4U_DBG("pList size %d, ptr %p\n", (int)sizeof(struct m4u_buf_info_t), pList); INIT_LIST_HEAD(&(pList->link)); return pList; } static int pseudo_free_buf_info(struct m4u_buf_info_t *pList) { kfree(pList); return 0; } static int pseudo_client_add_buf(struct m4u_client_t *client, struct m4u_buf_info_t *pList) { mutex_lock(&(client->dataMutex)); list_add(&(pList->link), &(client->mvaList)); mutex_unlock(&(client->dataMutex)); return 0; } /* * find or delete a buffer from client list * @param client -- client to be searched * @param mva -- mva to be searched * @param del -- should we del this buffer from client? * * @return buffer_info if found, NULL on fail * @remark * @see * @to-do we need to add multi domain support here. * @author K Zhang @date 2013/11/14 */ static struct m4u_buf_info_t *pseudo_client_find_buf( struct m4u_client_t *client, unsigned long mva, int del) { struct list_head *pListHead; struct m4u_buf_info_t *pList = NULL; struct m4u_buf_info_t *ret = NULL; if (client == NULL) { M4U_ERR("m4u_delete_from_garbage_list(), client is NULL!\n"); return NULL; } mutex_lock(&(client->dataMutex)); list_for_each(pListHead, &(client->mvaList)) { pList = container_of(pListHead, struct m4u_buf_info_t, link); if (pList->mva == mva) break; } if (pListHead == &(client->mvaList)) { ret = NULL; } else { if (del) list_del(pListHead); ret = pList; } mutex_unlock(&(client->dataMutex)); return ret; } static bool pseudo_is_acp_port(unsigned int port) { unsigned int count = ARRAY_SIZE(pseudo_acp_port_array); unsigned int i; for (i = 0; i < count; i++) { if (pseudo_acp_port_array[i] == port) return true; } return false; } int m4u_switch_acp(unsigned int port, unsigned long iova, size_t size, bool is_acp) { struct device *dev = pseudo_get_larbdev(port); if (!dev) { M4U_MSG("%s dev NULL!\n", __func__); return -EINVAL; } if (!pseudo_is_acp_port(port)) { M4U_MSG("invalid p:%d, va:0x%lx, sz:0x%lx\n", port, iova, size); return -EINVAL; } M4U_MSG("%s %d, switch acp, iova=0x%lx, size=0x%lx, acp=%d\n", __func__, __LINE__, iova, size, is_acp); return mtk_iommu_switch_acp(dev, iova, size, is_acp); } EXPORT_SYMBOL(m4u_switch_acp); /* * dump the current status of pgtable * this is the runtime mapping result of IOVA and PA */ #define MTK_PGTABLE_DUMP_RANGE SZ_16M #define MTK_PGTABLE_DUMP_LEVEL_FULL (1) #define MTK_PGTABLE_DUMP_LEVEL_ION (0) #define PORT_MAX_COUNT (5) #define PORT_LEAKAGE_LINE (1024 * 1024) //unit of KB static unsigned int port_size[MTK_IOMMU_LARB_NR][32]; static unsigned int port_iova[MTK_IOMMU_LARB_NR][32]; static unsigned int unknown_port_size; static void m4u_clear_port_size(void) { int i, j; unknown_port_size = 0; for (i = 0; i < MTK_IOMMU_LARB_NR; i++) { for (j = 0; j < 32; j++) { port_size[i][j] = 0; port_iova[i][j] = 0; } } } static void m4u_add_port_size(unsigned int larb, unsigned int port, unsigned long size, unsigned long iova) { if (larb < MTK_IOMMU_LARB_NR && port < 32) { port_size[larb][port] += (unsigned int)(size / 1024); if (!port_iova[larb][port]) port_iova[larb][port] = (unsigned int)(iova / 1024); } else unknown_port_size += (unsigned int)(size / 1024); } #if BITS_PER_LONG == 32 void m4u_find_max_port_size(unsigned long long base, unsigned long long max, unsigned int *err_port, unsigned int *err_size) #else void m4u_find_max_port_size(unsigned long base, unsigned long max, unsigned int *err_port, unsigned int *err_size) #endif { unsigned int i, j, k, t; int size[PORT_MAX_COUNT] = {0, 0, 0, 0, 0}; int port[PORT_MAX_COUNT] = {-1, -1, -1, -1, -1}; unsigned int start = (unsigned int)(base / 1024); unsigned int end = (unsigned int)(max / 1024); *err_port = M4U_PORT_UNKNOWN; *err_size = 0; for (i = 0; i < MTK_IOMMU_LARB_NR; i++) { for (j = 0; j < 32; j++) { if (port_iova[i][j] < start || port_iova[i][j] > end) continue; for (k = 0; k < PORT_MAX_COUNT; k++) { if (port[k] == MTK_M4U_ID(i, j)) break; if (port_size[i][j] > size[k]) { for (t = PORT_MAX_COUNT - 1; t > k; t--) { size[t] = size[t-1]; port[t] = port[t-1]; } size[k] = port_size[i][j]; port[k] = MTK_M4U_ID(i, j); break; } } #if 0 if (port_size[i][j] > 0) pr_notice("%s, p:%d(%d/%d),s:%d, %d, %d, %d, %d, %d\n", __func__, MTK_M4U_ID(i, j), i, j, port_size[i][j], port[0], port[1], port[2], port[3], port[4]); #endif } } pr_notice(" ******the top %d iova user in domain(0x%lx~0x%lx)******\n", PORT_MAX_COUNT, base, max); for (k = 0; k < PORT_MAX_COUNT; k++) { if (unknown_port_size > size[k]) { if (unknown_port_size > *err_size && unknown_port_size >= PORT_LEAKAGE_LINE) { *err_port = M4U_PORT_UNKNOWN; *err_size = unknown_port_size; } pr_notice(" >>> unknown port: size:%uKB\n", unknown_port_size); unknown_port_size = 0;//only dump one time } else { if (port[k] == -1) continue; if (size[k] > *err_size) { *err_port = port[k]; *err_size = size[k]; } pr_notice(" >>> %s(%d): size:%uKB\n", iommu_get_port_name(port[k]), port[k], size[k]); } } if (*err_size) pr_notice(" *********** suspect:%s(%d) used:%uKB***********\n", iommu_get_port_name(*err_port), *err_port, *err_size); } void __m4u_dump_pgtable(struct seq_file *s, unsigned int level, bool lock, unsigned long target) { struct m4u_client_t *client = ion_m4u_client; struct list_head *pListHead; struct m4u_buf_info_t *pList = NULL; unsigned long p_start = 0, p_end = 0; unsigned long start = 0, end = 0; struct device *dev = NULL; unsigned int larb = 0, port = 0; if (!client) return; if (lock) //lock is not support in irq context mutex_lock(&(client->dataMutex)); else //mvaList may be destroyed at dump goto iova_dump; m4u_clear_port_size(); M4U_PRINT_SEQ(s, "======== pseudo_m4u IOVA List ==========\n"); M4U_PRINT_SEQ(s, " bound IOVA_start ~ IOVA_end PA_start ~ PA_end size(Byte) port(larb-port) name time(ms) process(pid)\n"); list_for_each(pListHead, &(client->mvaList)) { pList = container_of(pListHead, struct m4u_buf_info_t, link); if (!pList) continue; start = pList->mva; end = pList->mva + pList->size - 1; larb = m4u_get_larbid(pList->port); port = m4u_port_2_larb_port(pList->port); m4u_add_port_size(larb, port, pList->size, start); if (target && ((end <= target - MTK_PGTABLE_DUMP_RANGE) || (start >= target + MTK_PGTABLE_DUMP_RANGE))) continue; dev = pseudo_get_larbdev(pList->port); mtk_iommu_iova_to_pa(dev, start, &p_start); mtk_iommu_iova_to_pa(dev, end, &p_end); M4U_PRINT_SEQ(s, ">>> %lu 0x%lx~0x%lx, 0x%lx~0x%lx, %lu, %u(%u-%u), %s, %llu, %s(%d)\n", (start & 0x300000000) >> 32, start, end, p_start, p_end, pList->size, pList->port, larb, port, iommu_get_port_name(pList->port), pList->timestamp, pList->task_comm, pList->pid); } if (lock) mutex_unlock(&(client->dataMutex)); iova_dump: if (level == MTK_PGTABLE_DUMP_LEVEL_FULL) mtk_iommu_dump_iova_space(target); } EXPORT_SYMBOL(__m4u_dump_pgtable); void m4u_dump_pgtable(unsigned int level, unsigned long target) { __m4u_dump_pgtable(NULL, level, false, target); } EXPORT_SYMBOL(m4u_dump_pgtable); int __pseudo_alloc_mva(struct m4u_client_t *client, int port, unsigned long va, unsigned long size, struct sg_table *sg_table, unsigned int flags, unsigned long *retmva) { struct mva_sglist *mva_sg = NULL; struct sg_table *table = NULL; int ret; struct device *dev = pseudo_get_larbdev(port); dma_addr_t dma_addr = ARM_MAPPING_ERROR; dma_addr_t paddr; unsigned int i; unsigned int err_port = 0, err_size = 0; struct scatterlist *s; dma_addr_t offset = 0; struct m4u_buf_info_t *pbuf_info; unsigned long long current_ts = 0; struct task_struct *task; bool free_table = true; if (!dev) { M4U_MSG("dev NULL!\n"); return -1; } if (va && sg_table) { M4U_MSG("va/sg 0x%x are valid:0x%lx, 0x%p, 0x%x, 0x%lx-0x%lx\n", port, va, sg_table, flags, *retmva, size); } else if (!va && !sg_table) { M4U_ERR("err va, err sg\n"); return -EINVAL; } /* this is for ion mm heap and ion fb heap usage. */ if (sg_table) { s = sg_table->sgl; if ((flags & M4U_FLAGS_SG_READY) == 0) { struct scatterlist *ng; phys_addr_t phys; int i; table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) { M4U_ERR("%d, table is NULL\n", __LINE__); return -ENOMEM; } ret = sg_alloc_table(table, sg_table->nents, GFP_KERNEL); if (ret) { kfree(table); *retmva = 0; M4U_ERR( "err failed to allocat table, ret=%d, nents=%d\n", ret, sg_table->nents); return ret; } ng = table->sgl; size = 0; for (i = 0; i < sg_table->nents; i++) { phys = sg_phys(s); size += s->length; sg_set_page(ng, sg_page(s), s->length, 0); sg_dma_address(ng) = phys; sg_dma_len(ng) = s->length; s = sg_next(s); ng = sg_next(ng); } if (!size) { M4U_ERR("err sg, please set page\n"); goto ERR_EXIT; } } else { table = sg_table; free_table = false; } } if (!table && va && size) table = pseudo_get_sg(port, va, size); if (!table) { M4U_ERR("err sg of va:0x%lx, size:0x%lx\n", va, size); goto ERR_EXIT; } #if defined(CONFIG_MACH_MT6785) /*just a workaround, since m4u design didn't define VPU_DATA*/ if (!(flags & (M4U_FLAGS_FIX_MVA | M4U_FLAGS_START_FROM))) { M4U_DBG("%s,%d, vpu data, flags=0x%x\n", __func__, __LINE__, flags); if (port == M4U_PORT_VPU) port = M4U_PORT_VPU_DATA; dev = pseudo_get_larbdev(port); } else { M4U_DBG("%s,%d, vpu code, flags=0x%x\n", __func__, __LINE__, flags); } #endif dma_map_sg_attrs(dev, table->sgl, sg_table ? table->nents : table->orig_nents, DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); dma_addr = sg_dma_address(table->sgl); current_ts = sched_clock(); if (!dma_addr || dma_addr == ARM_MAPPING_ERROR) { unsigned long base, max; int domain, owner; paddr = sg_phys(table->sgl); domain = mtk_iommu_get_iova_space(dev, &base, &max, &owner, NULL); M4U_ERR( "err %s, domain:%d(%s(%d):0x%lx~0x%lx) iova:0x%pad+0x%lx, pa=0x%pad, f:0x%x, n:%d-%d, sg_tab:0x%llx\n", iommu_get_port_name(port), domain, iommu_get_port_name(owner), owner, base, max, &dma_addr, size, &paddr, flags, table->nents, table->orig_nents, (unsigned long long)sg_table); __m4u_dump_pgtable(NULL, 1, true, 0); if (owner < 0) m4u_find_max_port_size(base, max, &err_port, &err_size); else { err_port = owner; err_size = (unsigned int)(-1); } report_custom_iommu_leakage( iommu_get_port_name(err_port), err_size); goto ERR_EXIT; } /* local table should copy to buffer->sg_table */ if (sg_table) { for_each_sg(sg_table->sgl, s, sg_table->nents, i) { sg_dma_address(s) = dma_addr + offset; offset += s->length; } } *retmva = dma_addr; mva_sg = kzalloc(sizeof(*mva_sg), GFP_KERNEL); if (!mva_sg) { M4U_ERR("alloc mva_sg fail\n"); goto ERR_EXIT; } mva_sg->table = table; mva_sg->mva = *retmva; pseudo_add_sgtable(mva_sg); paddr = sg_phys(table->sgl); M4U_INFO("%s, p:%d(%d-%d) pa=0x%pad iova=0x%lx s=0x%lx n=%d", iommu_get_port_name(port), port, MTK_IOMMU_TO_LARB(port), MTK_IOMMU_TO_PORT(port), &paddr, *retmva, size, table->nents); /* pbuf_info for userspace compatible */ pbuf_info = pseudo_alloc_buf_info(); if (!pbuf_info) { M4U_ERR("alloc pbuf_info fail\n"); goto ERR_EXIT; } pbuf_info->va = va; pbuf_info->port = port; pbuf_info->size = size; pbuf_info->flags = flags; pbuf_info->sg_table = sg_table; pbuf_info->mva = *retmva; pbuf_info->mva_align = *retmva; pbuf_info->size_align = size; do_div(current_ts, 1000000); pbuf_info->timestamp = current_ts; task = current->group_leader; get_task_comm(pbuf_info->task_comm, task); pbuf_info->pid = task_pid_nr(task); pseudo_client_add_buf(client, pbuf_info); mtk_iommu_trace_log(IOMMU_ALLOC, dma_addr, size, flags | (port << 16)); return 0; ERR_EXIT: if (table && #if BITS_PER_LONG == 32 sg_phys(table->sgl) >= (1ULL << MTK_PHYS_ADDR_BITS)) #else sg_phys(table->sgl) >= (1UL << MTK_PHYS_ADDR_BITS)) #endif ret = -ERANGE; else ret = -EINVAL; if (table && free_table) { M4U_ERR("nent:%u--%u, len:0x%lx\n", table->nents, table->orig_nents, (unsigned long)sg_dma_len(table->sgl));/* debug memory corruption */ sg_free_table(table); kfree(table); } kfree(mva_sg); *retmva = 0; return ret; } /* interface for ion */ struct m4u_client_t *pseudo_create_client(void) { struct m4u_client_t *client; client = kmalloc(sizeof(struct m4u_client_t), GFP_ATOMIC); if (!client) return NULL; mutex_init(&(client->dataMutex)); mutex_lock(&(client->dataMutex)); client->open_pid = current->pid; client->open_tgid = current->tgid; INIT_LIST_HEAD(&(client->mvaList)); mutex_unlock(&(client->dataMutex)); return client; } struct m4u_client_t *pseudo_get_m4u_client(void) { if (!ion_m4u_client) { ion_m4u_client = pseudo_create_client(); if (IS_ERR_OR_NULL(ion_m4u_client)) { M4U_ERR("err client\n"); ion_m4u_client = NULL; return NULL; } ion_m4u_client->count++; } //M4U_MSG("user count:%d\n", ion_m4u_client->count); return ion_m4u_client; } EXPORT_SYMBOL(pseudo_get_m4u_client); void pseudo_put_m4u_client(void) { if (IS_ERR_OR_NULL(ion_m4u_client)) { M4U_ERR("err client\n"); return; } if (ion_m4u_client->count <= 0) { M4U_ERR("client count(%ld) not match\n", ion_m4u_client->count); return; } ion_m4u_client->count--; //M4U_MSG("user count:%d\n", ion_m4u_client->count); } EXPORT_SYMBOL(pseudo_put_m4u_client); int pseudo_alloc_mva_sg(struct port_mva_info_t *port_info, struct sg_table *sg_table) { unsigned int flags = 0; int ret, offset; unsigned long mva = 0; unsigned long va_align; unsigned long *pMva; unsigned long mva_align; unsigned long size_align = port_info->buf_size; struct m4u_client_t *client; client = ion_m4u_client; if (!client) { M4U_ERR("failed to get ion_m4u_client\n"); return -1; } if (port_info->flags & M4U_FLAGS_FIX_MVA) flags = M4U_FLAGS_FIX_MVA; if (port_info->flags & M4U_FLAGS_SG_READY) flags |= M4U_FLAGS_SG_READY; else port_info->va = 0; va_align = port_info->va; pMva = &port_info->mva; mva_align = *pMva; /* align the va to allocate continues iova. */ offset = m4u_va_align(&va_align, &size_align); ret = __pseudo_alloc_mva(client, port_info->emoduleid, va_align, size_align, sg_table, flags, &mva_align); if (ret) { M4U_ERR( "error alloc mva: port %d, 0x%x, 0x%lx, 0x%lx, 0x%lx, ret=%d\n", port_info->emoduleid, flags, port_info->va, mva_align, port_info->buf_size, ret); mva = 0; return ret; } mva = mva_align + offset; *pMva = mva; #if 0 M4U_MSG("%s:port(%d), flags(%d), va(0x%lx), mva=0x%x, size 0x%x\n", __func__, port_info->emoduleid, flags, port_info->va, mva, port_info->buf_size); #endif return 0; } static struct sg_table *pseudo_del_sgtable(unsigned long mva) { struct mva_sglist *entry, *tmp; struct sg_table *table = NULL; unsigned long flags = 0; spin_lock_irqsave(&pseudo_list_lock, flags); list_for_each_entry_safe(entry, tmp, &pseudo_sglist, list) { if (entry->mva == mva) { list_del(&entry->list); spin_unlock_irqrestore(&pseudo_list_lock, flags); table = entry->table; M4U_DBG("mva is 0x%lx, entry->mva is 0x%lx\n", mva, entry->mva); kfree(entry); return table; } } spin_unlock_irqrestore(&pseudo_list_lock, flags); return NULL; } /* put ref count on all pages in sgtable */ int pseudo_put_sgtable_pages(struct sg_table *table, int nents) { int i; struct scatterlist *sg; for_each_sg(table->sgl, sg, nents, i) { struct page *page = sg_page(sg); if (IS_ERR(page)) return 0; if (page) put_page(page); } return 0; } /* the caller should make sure the mva offset have been eliminated. */ int __pseudo_dealloc_mva(struct m4u_client_t *client, int port, unsigned long BufAddr, const unsigned long size, const unsigned long mva, struct sg_table *sg_table) { struct sg_table *table = NULL; struct device *dev = pseudo_get_larbdev(port); unsigned long addr_align = mva; unsigned long size_align = size; int offset; if (!dev) { M4U_MSG("dev is NULL\n"); return -EINVAL; } M4U_INFO("larb%d, port%d, addr=0x%lx, size=0x%lx, iova=0x%lx\n", MTK_IOMMU_TO_LARB(port), MTK_IOMMU_TO_PORT(port), BufAddr, size, mva); /* for ion sg alloc, we did not align the mva in allocation. */ /* if (!sg_table) */ offset = m4u_va_align(&addr_align, &size_align); if (sg_table) { struct m4u_buf_info_t *m4u_buf_info; m4u_buf_info = pseudo_client_find_buf(client, mva, 1); table = pseudo_del_sgtable(addr_align); if (!table) { M4U_ERR("err table of mva 0x%lx-0x%lx\n", mva, addr_align); M4U_ERR("%s addr=0x%lx,size=0x%lx\n", m4u_get_module_name(port), BufAddr, size); dump_stack(); return -EINVAL; } if (sg_page(table->sgl) != sg_page(sg_table->sgl)) { M4U_ERR("error, sg\n"); return -EINVAL; } if (m4u_buf_info) { BufAddr = m4u_buf_info->va; pseudo_free_buf_info(m4u_buf_info); } } if (!table) table = pseudo_del_sgtable(addr_align); mtk_iommu_trace_log(IOMMU_DEALLOC, mva, size, (port << 16)); if (table) { dma_unmap_sg_attrs(dev, table->sgl, table->orig_nents, DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); } else { M4U_ERR("can't find the sgtable and would return error\n"); return -EINVAL; } if (BufAddr) { /* from user space */ if (BufAddr < PAGE_OFFSET) { if (!((BufAddr >= VMALLOC_START) && (BufAddr <= VMALLOC_END))) { pseudo_put_sgtable_pages( table, table->nents); } } } if (table) { sg_free_table(table); kfree(table); } M4UTRACE(); return 0; } int pseudo_dealloc_mva(struct m4u_client_t *client, int port, unsigned long mva) { struct m4u_buf_info_t *pMvaInfo; int offset, ret; pMvaInfo = pseudo_client_find_buf(client, mva, 1); if (!pMvaInfo) return -ENOMEM; offset = m4u_va_align(&pMvaInfo->va, &pMvaInfo->size); pMvaInfo->mva -= offset; ret = __pseudo_dealloc_mva(client, port, pMvaInfo->va, pMvaInfo->size, mva, NULL); #if 0 M4U_DBG("port %d, flags 0x%x, va 0x%lx, mva = 0x%lx, size 0x%lx\n", port, pMvaInfo->flags, pMvaInfo->va, mva, pMvaInfo->size); #endif if (ret) return ret; pseudo_free_buf_info(pMvaInfo); return ret; } int pseudo_dealloc_mva_sg(int eModuleID, struct sg_table *sg_table, const unsigned int BufSize, const unsigned long mva) { if (!sg_table) { M4U_MSG("sg_table is NULL\n"); return -EINVAL; } return __pseudo_dealloc_mva(ion_m4u_client, eModuleID, 0, BufSize, mva, sg_table); } int pseudo_destroy_client(struct m4u_client_t *client) { struct m4u_buf_info_t *pMvaInfo; unsigned long mva, size; int port; while (1) { mutex_lock(&(client->dataMutex)); if (list_empty(&client->mvaList)) { mutex_unlock(&(client->dataMutex)); break; } pMvaInfo = container_of(client->mvaList.next, struct m4u_buf_info_t, link); M4U_MSG ("warn:clean,%s,va=0x%lx,mva=0x%lx,s=%lu,c:%ld\n", iommu_get_port_name(pMvaInfo->port), pMvaInfo->va, pMvaInfo->mva, pMvaInfo->size, ion_m4u_client->count); port = pMvaInfo->port; mva = pMvaInfo->mva; size = pMvaInfo->size; mutex_unlock(&(client->dataMutex)); /* m4u_dealloc_mva will lock client->dataMutex again */ pseudo_dealloc_mva(client, port, mva); } kfree(client); M4U_MSG("client has been destroyed\n"); return 0; } static int m4u_fill_sgtable_user(struct vm_area_struct *vma, unsigned long va, int page_num, struct scatterlist **pSg, int has_page) { unsigned long va_align; phys_addr_t pa = 0; int i; long ret = 0; struct scatterlist *sg = *pSg; struct page *pages = NULL; int gup_flags; va_align = round_down(va, PAGE_SIZE); gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; if (vma->vm_flags & VM_LOCKONFAULT) gup_flags &= ~FOLL_POPULATE; /* * We want to touch writable mappings with a write fault in order * to break COW, except for shared mappings because these don't COW * and we would not want to dirty them for nothing. */ if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE) gup_flags |= FOLL_WRITE; /* * We want mlock to succeed for regions that have any permissions * other than PROT_NONE. */ if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) gup_flags |= FOLL_FORCE; for (i = 0; i < page_num; i++) { int fault_cnt; unsigned long va_tmp = va_align+i*PAGE_SIZE; pa = 0; for (fault_cnt = 0; fault_cnt < 3000; fault_cnt++) { if (has_page) { ret = get_user_pages(va_tmp, 1, gup_flags, &pages, NULL); if (ret == 1) pa = page_to_phys(pages) | (va_tmp & ~PAGE_MASK); } else { pa = m4u_user_v2p(va_tmp); if (!pa) { handle_mm_fault(vma, va_tmp, (vma->vm_flags & VM_WRITE) ? FAULT_FLAG_WRITE : 0); } } if (pa) { /* Add one line comment for coding style */ break; } cond_resched(); } if (!pa || !sg) { struct vm_area_struct *vma_temp; M4U_MSG("%s: fail(0x%lx) va=0x%lx,page_num=0x%x\n", __func__, ret, va, page_num); M4U_MSG("%s: fail_va=0x%lx,pa=0x%lx,sg=0x%p,i=%d\n", __func__, va_tmp, (unsigned long)pa, sg, i); vma_temp = find_vma(current->mm, va_tmp); if (vma_temp != NULL) { M4U_MSG("vma start=0x%lx,end=%lx,flag=%lx\n", vma->vm_start, vma->vm_end, vma->vm_flags); M4U_MSG("temp start=0x%lx,end=%lx,flag=%lx\n", vma_temp->vm_start, vma_temp->vm_end, vma_temp->vm_flags); } return -1; } if (fault_cnt > 2) M4U_MSG("warning: handle_mm_fault for %d times\n", fault_cnt); /* debug check... */ if ((pa & (PAGE_SIZE - 1)) != 0) { M4U_MSG("pa error, pa:0x%lx, va:0x%lx, align:0x%lx\n", (unsigned long)pa, va_tmp, va_align); } if (has_page) { struct page *page; page = phys_to_page(pa); sg_set_page(sg, page, PAGE_SIZE, 0); #ifdef CONFIG_NEED_SG_DMA_LENGTH sg->dma_length = sg->length; #endif } else { sg_dma_address(sg) = pa; sg_dma_len(sg) = PAGE_SIZE; } sg = sg_next(sg); } *pSg = sg; return 0; } static int m4u_create_sgtable_user(unsigned long va_align, struct sg_table *table) { int ret = 0; struct vm_area_struct *vma; struct scatterlist *sg = table->sgl; unsigned int left_page_num = table->nents; unsigned long va = va_align; down_read(¤t->mm->mmap_sem); while (left_page_num) { unsigned int vma_page_num; vma = find_vma(current->mm, va); if (vma == NULL || vma->vm_start > va) { M4U_MSG("cannot find vma: va=0x%lx, vma=0x%p\n", va, vma); if (vma != NULL) { M4U_MSG("start=0x%lx,end=0x%lx,flag=0x%lx\n", vma->vm_start, vma->vm_end, vma->vm_flags); } ///m4u_dump_mmaps(va); ret = -1; goto out; } else { /* M4U_DBG("%s va: 0x%lx, vma->vm_start=0x%lx, */ /* vma->vm_end=0x%lx\n",*/ /*__func__, va, vma->vm_start, vma->vm_end); */ } vma_page_num = (vma->vm_end - va) / PAGE_SIZE; vma_page_num = min(vma_page_num, left_page_num); if ((vma->vm_flags) & VM_PFNMAP) { /* ion va or ioremap vma has this flag */ /* VM_PFNMAP: Page-ranges managed without */ /* "struct page", just pure PFN */ M4U_MSG("VM_PFNMAP forbiden! va=0x%lx, page_num=0x%x\n", va, vma_page_num); ret = -2; goto out; } else { /* Add one line comment for avoid kernel coding style*/ /* WARNING:BRACES: */ ret = m4u_fill_sgtable_user(vma, va, vma_page_num, &sg, 1); if (-1 == ret) { struct vm_area_struct *vma_temp; vma_temp = find_vma(current->mm, va_align); if (!vma_temp) { M4U_MSG("vma NUll for va 0x%lx\n", va_align); goto out; } M4U_MSG("start=0x%lx,end=0x%lx,flag=0x%lx\n", vma_temp->vm_start, vma_temp->vm_end, vma_temp->vm_flags); } } if (ret) { /* Add one line comment for avoid kernel coding */ /* style, WARNING:BRACES: */ goto out; } left_page_num -= vma_page_num; va += vma_page_num * PAGE_SIZE; } out: up_read(¤t->mm->mmap_sem); return ret; } struct sg_table *m4u_create_sgtable(unsigned long va, unsigned long size) { struct sg_table *table; int ret, i, page_num; unsigned long va_align; phys_addr_t pa; struct scatterlist *sg; struct page *page; unsigned int psize = PAGE_SIZE; page_num = M4U_GET_PAGE_NUM(va, size); va_align = round_down(va, PAGE_SIZE); table = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (!table) { M4U_MSG("table kmalloc fail:va=0x%lx,size=0x%lx,page_num=%u\n", va, size, page_num); return ERR_PTR(-ENOMEM); } ret = sg_alloc_table(table, page_num, GFP_KERNEL); if (ret) { kfree(table); M4U_MSG("alloc_sgtable fail: va=0x%lx,size=0x%lx,page_num=%u\n", va, size, page_num); return ERR_PTR(-ENOMEM); } if (va < PAGE_OFFSET) { /* from user space */ if (va >= VMALLOC_START && va <= VMALLOC_END) { /* vmalloc */ M4U_MSG(" from user space vmalloc, va = 0x%lx\n", va); for_each_sg(table->sgl, sg, table->nents, i) { page = vmalloc_to_page((void *)(va_align + i * psize)); if (!page) { M4U_MSG("va_to_page fail, va=0x%lx\n", va_align + i * psize); goto err; } sg_set_page(sg, page, psize, 0); sg_dma_len(sg) = psize; } } else { ret = m4u_create_sgtable_user(va_align, table); if (ret) { M4U_ERR("error va=0x%lx, size=%lu\n", va, size); goto err; } } } else { /* from kernel sp */ if (va >= VMALLOC_START && va <= VMALLOC_END) { /* vmalloc */ M4U_MSG(" from kernel space vmalloc, va = 0x%lx", va); for_each_sg(table->sgl, sg, table->nents, i) { page = vmalloc_to_page((void *)(va_align + i * psize)); if (!page) { M4U_MSG("va_to_page fail, va=0x%lx\n", va_align + i * psize); goto err; } sg_set_page(sg, page, psize, 0); sg_dma_len(sg) = psize; } } else { /* kmalloc to-do: use one entry sgtable. */ for_each_sg(table->sgl, sg, table->nents, i) { pa = virt_to_phys((void *)(va_align + i * psize)); page = phys_to_page(pa); sg_set_page(sg, page, psize, 0); sg_dma_len(sg) = psize; } } } return table; err: sg_free_table(table); kfree(table); return ERR_PTR(-EFAULT); } int m4u_dealloc_mva_sg(int eModuleID, struct sg_table *sg_table, const unsigned long BufSize, const unsigned long mva) { return pseudo_dealloc_mva_sg(eModuleID, sg_table, BufSize, mva); } int m4u_alloc_mva_sg(struct port_mva_info_t *port_info, struct sg_table *sg_table) { return pseudo_alloc_mva_sg(port_info, sg_table); } static int pseudo_open(struct inode *inode, struct file *file) { struct m4u_client_t *client; M4U_DBG("%s process : %s\n", __func__, current->comm); client = pseudo_get_m4u_client(); if (IS_ERR_OR_NULL(client)) { M4U_MSG("createclientfail\n"); return -ENOMEM; } file->private_data = client; return 0; } int m4u_config_port(struct M4U_PORT_STRUCT *pM4uPort) { int ret; ret = pseudo_config_port(pM4uPort, 0); return ret; } #if 0 void pseudo_m4u_bank_irq_debug(unsigned int domain) { int i, j, ret = 0; unsigned int count = 0; u32 reg = 0; for (i = 0; i < SMI_LARB_NR; i++) { ret = larb_clock_on(i, 1); if (ret < 0) { M4U_MSG("enable larb%d fail\n", i); continue; } count = mtk_iommu_get_larb_port_count(i); for (j = 0; j < count; j++) { #ifdef SMI_LARB_SEC_CON_EN pseudo_set_reg_by_mask(pseudo_larbbase[i], SMI_LARB_SEC_CONx(j), F_SMI_DOMN(0x7), F_SMI_DOMN(0x2)); reg = pseudo_readreg32(pseudo_larbbase[i], SMI_LARB_SEC_CONx(j)); #else ret = mtk_iommu_set_sec_larb(i, j, 0, 0x2); if (!ret) reg = mtk_iommu_dump_sec_larb(i, j); else pr_notice("%s, atf set sec larb fail, larb:%d, port:%d\n", __func__, i, j); #endif pr_debug("%s, switch larb%d to dom:0x%x\n", __func__, i, reg); } larb_clock_off(i, 1); } } #endif void pseudo_m4u_db_debug(unsigned int m4uid, struct seq_file *s) { int i, j; if (m4uid < MTK_IOMMU_M4U_COUNT) { M4U_PRINT_SEQ(s, "=========================iommu%d start HW register dump=============================\n", m4uid); for (i = 0; i < MTK_IOMMU_MMU_COUNT; i++) mtk_dump_main_tlb(m4uid, i, s); mtk_dump_pfh_tlb(m4uid, s); mtk_dump_victim_tlb(m4uid, s); __mtk_dump_reg_for_hang_issue(m4uid, s); M4U_PRINT_SEQ(s, "=========================iommu%d finish HW register dump============================\n", m4uid); } else { for (i = 0; i < MTK_IOMMU_M4U_COUNT; i++) { M4U_PRINT_SEQ(s, "====================================iommu%d start HW register dump===============================\n", i); for (j = 0; j < MTK_IOMMU_MMU_COUNT; j++) mtk_dump_main_tlb(i, j, s); mtk_dump_pfh_tlb(i, s); mtk_dump_victim_tlb(i, s); __mtk_dump_reg_for_hang_issue(i, s); M4U_PRINT_SEQ(s, "====================================iommu%d finish HW register dump==============================\n", i); } } pseudo_dump_all_port_status(s); __m4u_dump_pgtable(s, MTK_PGTABLE_DUMP_LEVEL_ION, true, 0); } EXPORT_SYMBOL(pseudo_m4u_db_debug); static int pseudo_release(struct inode *inode, struct file *file) { pseudo_put_m4u_client(); return 0; } static int pseudo_flush(struct file *a_pstFile, fl_owner_t a_id) { return 0; } /***********************************************************/ /** map mva buffer to kernel va buffer * this function should ONLY used for DEBUG ************************************************************/ int m4u_mva_map_kernel(unsigned long mva, unsigned long size, unsigned long *map_va, unsigned long *map_size, struct sg_table *table) { struct scatterlist *sg; int i, j, k, ret = 0; struct page **pages; unsigned int page_num; void *kernel_va; unsigned int kernel_size; if (!table) { M4U_MSG("invalid sg table\n"); return -1; } page_num = M4U_GET_PAGE_NUM(mva, size); pages = vmalloc(sizeof(struct page *) * page_num); if (pages == NULL) { M4U_MSG("mva_map_kernel:error to vmalloc for %d\n", (unsigned int)sizeof(struct page *) * page_num); return -1; } k = 0; for_each_sg(table->sgl, sg, table->nents, i) { struct page *page_start; int pages_in_this_sg = PAGE_ALIGN(sg_dma_len(sg)) / PAGE_SIZE; #ifdef CONFIG_NEED_SG_DMA_LENGTH if (sg_dma_address(sg) == 0) pages_in_this_sg = PAGE_ALIGN(sg->length) / PAGE_SIZE; #endif page_start = sg_page(sg); for (j = 0; j < pages_in_this_sg; j++) { pages[k++] = page_start++; if (k >= page_num) goto get_pages_done; } } get_pages_done: if (k < page_num) { /* this should not happen, because we have * checked the size before. */ M4U_MSG( "mva_map_kernel:only get %d pages: mva=0x%lx, size=0x%lx, pg_num=%u\n", k, mva, size, page_num); ret = -1; goto error_out; } kernel_va = 0; kernel_size = 0; kernel_va = vmap(pages, page_num, VM_MAP, PAGE_KERNEL); if (kernel_va == 0 || (uintptr_t)kernel_va & M4U_PAGE_MASK) { M4U_MSG( "mva_map_kernel:vmap fail: page_num=%d, kernel_va=0x%p\n", page_num, kernel_va); ret = -2; goto error_out; } kernel_va += ((unsigned long)mva & (M4U_PAGE_MASK)); *map_va = (unsigned long)kernel_va; *map_size = size; error_out: vfree(pages); M4U_DBG("mva=0x%lx,size=0x%lx,map_va=0x%lx,map_size=0x%lx\n", mva, size, *map_va, *map_size); return ret; } EXPORT_SYMBOL(m4u_mva_map_kernel); int m4u_mva_unmap_kernel(unsigned long mva, unsigned long size, unsigned long map_va) { M4U_DBG( "mva_unmap_kernel:mva=0x%lx,size=0x%lx,va=0x%lx\n", mva, size, map_va); vunmap((void *)(map_va & (~M4U_PAGE_MASK))); return 0; } EXPORT_SYMBOL(m4u_mva_unmap_kernel); #ifdef PSEUDO_M4U_TEE_SERVICE_ENABLE static DEFINE_MUTEX(gM4u_sec_init); bool m4u_tee_en; static int __m4u_sec_init(void) { int ret, i, count = 0; unsigned long pt_pa_nonsec; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4UTL_INIT); if (!ctx) return -EFAULT; for (i = 0; i < SMI_LARB_NR; i++) { ret = larb_clock_on(i, 1); if (ret < 0) { M4U_MSG("enable larb%d fail, ret:%d\n", i, ret); count = i; goto out; } } count = SMI_LARB_NR; if (mtk_iommu_get_pgtable_base_addr((void *)&pt_pa_nonsec)) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4UTL_INIT; ctx->m4u_msg->init_param.nonsec_pt_pa = pt_pa_nonsec; ctx->m4u_msg->init_param.l2_en = M4U_L2_ENABLE; ctx->m4u_msg->init_param.sec_pt_pa = 0; /* m4u_alloc_sec_pt_for_debug(); */ M4ULOG_HIGH("%s call CMD_M4UTL_INIT, nonsec_pt_pa: 0x%lx\n", __func__, pt_pa_nonsec); ret = m4u_exec_cmd(ctx); if (ret) { M4U_ERR("m4u exec command fail\n"); goto out; } /*ret = ctx->m4u_msg->rsp;*/ out: if (count) { for (i = 0; i < count; i++) larb_clock_off(i, 1); } m4u_sec_ctx_put(ctx); return ret; } /* ------------------------------------------------------------- */ #ifdef __M4U_SECURE_SYSTRACE_ENABLE__ static int dr_map(unsigned long pa, size_t size) { int ret; mutex_lock(&m4u_dci_mutex); if (!m4u_dci_msg) { M4U_ERR("error: m4u_dci_msg==null\n"); ret = -1; goto out; } memset(m4u_dci_msg, 0, sizeof(struct m4u_msg)); m4u_dci_msg->cmd = CMD_M4U_SYSTRACE_MAP; m4u_dci_msg->systrace_param.pa = pa; m4u_dci_msg->systrace_param.size = size; ret = m4u_exec_cmd(&m4u_dci_session, m4u_dci_msg); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = m4u_dci_msg->rsp; out: mutex_unlock(&m4u_dci_mutex); return ret; } static int dr_unmap(unsigned long pa, size_t size) { int ret; mutex_lock(&m4u_dci_mutex); if (!m4u_dci_msg) { M4U_ERR("error: m4u_dci_msg==null\n"); ret = -1; goto out; } memset(m4u_dci_msg, 0, sizeof(struct m4u_msg)); m4u_dci_msg->cmd = CMD_M4U_SYSTRACE_UNMAP; m4u_dci_msg->systrace_param.pa = pa; m4u_dci_msg->systrace_param.size = size; ret = m4u_exec_cmd(&m4u_dci_session, m4u_dci_msg); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = m4u_dci_msg->rsp; out: mutex_unlock(&m4u_dci_mutex); return ret; } static int dr_transact(void) { int ret; mutex_lock(&m4u_dci_mutex); if (!m4u_dci_msg) { M4U_ERR("error: m4u_dci_msg==null\n"); ret = -1; goto out; } memset(m4u_dci_msg, 0, sizeof(struct m4u_msg)); m4u_dci_msg->cmd = CMD_M4U_SYSTRACE_TRANSACT; m4u_dci_msg->systrace_param.pa = 0; m4u_dci_msg->systrace_param.size = 0; ret = m4u_exec_cmd(&m4u_dci_session, m4u_dci_msg); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = m4u_dci_msg->rsp; out: mutex_unlock(&m4u_dci_mutex); return ret; } #endif static int m4u_sec_init_nolock(void) { int ret; #if defined(CONFIG_TRUSTONIC_TEE_SUPPORT) && \ !defined(CONFIG_MTK_TEE_GP_SUPPORT) enum mc_result mcRet; #endif M4U_MSG("%s: start\n", __func__); if (m4u_tee_en) { M4U_MSG("warning: re-initiation, %d\n", m4u_tee_en); goto m4u_sec_reinit; } #if defined(CONFIG_TRUSTONIC_TEE_SUPPORT) && \ !defined(CONFIG_MTK_TEE_GP_SUPPORT) /* Allocating WSM for DCI */ mcRet = mc_malloc_wsm(MC_DEVICE_ID_DEFAULT, 0, sizeof(struct m4u_msg), (uint8_t **) &m4u_dci_msg, 0); if (mcRet != MC_DRV_OK) { M4U_MSG("tz_m4u: mc_malloc_wsm returned: %d\n", mcRet); return -1; } /* Open session the trustlet */ m4u_dci_session.device_id = MC_DEVICE_ID_DEFAULT; mcRet = mc_open_session(&m4u_dci_session, &m4u_drv_uuid, (uint8_t *) m4u_dci_msg, (uint32_t) sizeof(struct m4u_msg)); if (mcRet != MC_DRV_OK) { M4U_MSG("tz_m4u: mc_open_session returned: %d\n", mcRet); return -1; } M4U_DBG("tz_m4u: open DCI session returned: %d\n", mcRet); { mdelay(100); /* volatile int i, j; * for (i = 0; i < 10000000; i++) * j++; */ } #endif m4u_sec_set_context(); if (!m4u_tee_en) { ret = m4u_sec_context_init(); if (ret) return ret; m4u_tee_en = 1; } else { M4U_MSG("warning: reinit sec m4u en=%d\n", m4u_tee_en); } m4u_sec_reinit: ret = __m4u_sec_init(); if (ret < 0) { m4u_tee_en = 0; m4u_sec_context_deinit(); M4U_MSG("%s:init fail,ret=0x%x\n", __func__, ret); return ret; } /* don't deinit ta because of multiple init operation */ return 0; } int m4u_sec_init(void) { int ret = 0; mutex_lock(&gM4u_sec_init); ret = m4u_sec_init_nolock(); mutex_unlock(&gM4u_sec_init); return ret; } int m4u_config_port_tee(struct M4U_PORT_STRUCT *pM4uPort) /* native */ { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_CFG_PORT); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_CFG_PORT; ctx->m4u_msg->port_param.port = pM4uPort->ePortID; ctx->m4u_msg->port_param.virt = pM4uPort->Virtuality; ctx->m4u_msg->port_param.direction = pM4uPort->Direction; ctx->m4u_msg->port_param.distance = pM4uPort->Distance; ctx->m4u_msg->port_param.sec = 0; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } #if 0 int m4u_config_port_array_tee(unsigned char *port_array) /* native */ { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_CFG_PORT_ARRAY); if (!ctx) return -EFAULT; memset(ctx->m4u_msg, 0, sizeof(*ctx->m4u_msg)); memcpy(ctx->m4u_msg->port_array_param.m4u_port_array, port_array, sizeof(ctx->m4u_msg->port_array_param.m4u_port_array)); ctx->m4u_msg->cmd = CMD_M4U_CFG_PORT_ARRAY; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } #endif /*#ifdef TO_BE_IMPL*/ int m4u_larb_backup_sec(unsigned int larb_idx) { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_LARB_BACKUP); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_LARB_BACKUP; ctx->m4u_msg->larb_param.larb_idx = larb_idx; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } int m4u_larb_restore_sec(unsigned int larb_idx) { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_LARB_RESTORE); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_LARB_RESTORE; ctx->m4u_msg->larb_param.larb_idx = larb_idx; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } static int m4u_reg_backup_sec(void) { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_REG_BACKUP); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_REG_BACKUP; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } static int m4u_reg_restore_sec(void) { int ret; struct m4u_sec_context *ctx; ctx = m4u_sec_ctx_get(CMD_M4U_REG_RESTORE); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_REG_RESTORE; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } static void m4u_early_suspend(void) { if (mtk_iommu_power_support()) { M4U_MSG("%s , iommu pg callback supported\n", __func__); return; } M4U_MSG("%s +, %d\n", __func__, m4u_tee_en); //smi_debug_bus_hang_detect(false, M4U_DEV_NAME); if (m4u_tee_en) m4u_reg_backup_sec(); M4U_MSG("%s -\n", __func__); } static void m4u_late_resume(void) { if (mtk_iommu_power_support()) { M4U_MSG("%s , iommu pg callback supported\n", __func__); return; } M4U_MSG("%s +, %d\n", __func__, m4u_tee_en); //smi_debug_bus_hang_detect(false, M4U_DEV_NAME); if (m4u_tee_en) m4u_reg_restore_sec(); M4U_MSG("%s -\n", __func__); } static struct notifier_block m4u_fb_notifier; static int m4u_fb_notifier_callback( struct notifier_block *self, unsigned long event, void *data) { struct fb_event *evdata = data; int blank; M4U_MSG("%s %ld, %d\n", __func__, event, FB_EVENT_BLANK); if (event != FB_EVENT_BLANK) return 0; blank = *(int *)evdata->data; switch (blank) { case FB_BLANK_UNBLANK: case FB_BLANK_NORMAL: m4u_late_resume(); break; case FB_BLANK_VSYNC_SUSPEND: case FB_BLANK_HSYNC_SUSPEND: break; case FB_BLANK_POWERDOWN: m4u_early_suspend(); break; default: return -EINVAL; } return 0; } int m4u_map_nonsec_buf(int port, unsigned long mva, unsigned long size) { int ret; struct m4u_sec_context *ctx; return -EPERM; /* Not allow */ if ((mva > DMA_BIT_MASK(32)) || (mva + size > DMA_BIT_MASK(32))) { M4U_MSG("%s invalid mva:0x%lx, size:0x%lx\n", __func__, mva, size); return -EFAULT; } ctx = m4u_sec_ctx_get(CMD_M4U_MAP_NONSEC_BUFFER); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_MAP_NONSEC_BUFFER; ctx->m4u_msg->buf_param.mva = mva; ctx->m4u_msg->buf_param.size = size; ctx->m4u_msg->buf_param.port = port; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } int m4u_unmap_nonsec_buffer(unsigned long mva, unsigned long size) { int ret; struct m4u_sec_context *ctx; return -EPERM; if ((mva > DMA_BIT_MASK(32)) || (mva + size > DMA_BIT_MASK(32))) { M4U_MSG("%s invalid mva:0x%lx, size:0x%lx\n", __func__, mva, size); return -EFAULT; } ctx = m4u_sec_ctx_get(CMD_M4U_UNMAP_NONSEC_BUFFER); if (!ctx) return -EFAULT; ctx->m4u_msg->cmd = CMD_M4U_UNMAP_NONSEC_BUFFER; ctx->m4u_msg->buf_param.mva = mva; ctx->m4u_msg->buf_param.size = size; ret = m4u_exec_cmd(ctx); if (ret) { M4U_MSG("m4u exec command fail\n"); ret = -1; goto out; } ret = ctx->m4u_msg->rsp; out: m4u_sec_ctx_put(ctx); return ret; } #endif #ifdef M4U_GZ_SERVICE_ENABLE static DEFINE_MUTEX(gM4u_gz_sec_init); static bool m4u_gz_en[SEC_ID_COUNT]; static int __m4u_gz_sec_init(int mtk_iommu_sec_id) { int ret, i, count = 0; unsigned long pt_pa_nonsec; struct m4u_gz_sec_context *ctx; ctx = m4u_gz_sec_ctx_get(); if (!ctx) return -EFAULT; for (i = 0; i < SMI_LARB_NR; i++) { ret = larb_clock_on(i, 1); if (ret < 0) { M4U_MSG("enable larb%d fail, ret:%d\n", i, ret); count = i; goto out; } } count = SMI_LARB_NR; if (mtk_iommu_get_pgtable_base_addr((void *)&pt_pa_nonsec)) return -EFAULT; ctx->gz_m4u_msg->cmd = CMD_M4UTY_INIT; ctx->gz_m4u_msg->iommu_sec_id = mtk_iommu_sec_id; ctx->gz_m4u_msg->init_param.nonsec_pt_pa = pt_pa_nonsec; ctx->gz_m4u_msg->init_param.l2_en = M4U_L2_ENABLE; ctx->gz_m4u_msg->init_param.sec_pt_pa = 0; M4ULOG_HIGH("[MTEE]%s: mtk_iommu_sec_id:%d, nonsec_pt_pa: 0x%lx\n", __func__, mtk_iommu_sec_id, pt_pa_nonsec); ret = m4u_gz_exec_cmd(ctx); if (ret) { M4U_ERR("[MTEE]m4u exec command fail\n"); goto out; } out: if (count) { for (i = 0; i < count; i++) larb_clock_off(i, 1); } m4u_gz_sec_ctx_put(ctx); return ret; } int m4u_gz_sec_init(int mtk_iommu_sec_id) { int ret; M4U_MSG("[MTEE]%s: start\n", __func__); if (m4u_gz_en[mtk_iommu_sec_id]) { M4U_MSG("warning: re-initiation, %d\n", m4u_gz_en[mtk_iommu_sec_id]); goto m4u_gz_sec_reinit; } m4u_gz_sec_set_context(); if (!m4u_gz_en[mtk_iommu_sec_id]) { ret = m4u_gz_sec_context_init(); if (ret) return ret; m4u_gz_en[mtk_iommu_sec_id] = 1; } else { M4U_MSG("[MTEE]warning: reinit sec m4u_gz_en[%d]=%d\n", mtk_iommu_sec_id, m4u_gz_en[mtk_iommu_sec_id]); } m4u_gz_sec_reinit: ret = __m4u_gz_sec_init(mtk_iommu_sec_id); if (ret < 0) { m4u_gz_en[mtk_iommu_sec_id] = 0; m4u_gz_sec_context_deinit(); M4U_MSG("[MTEE]%s:init fail,ret=0x%x\n", __func__, ret); return ret; } /* don't deinit ta because of multiple init operation */ return 0; } int m4u_map_gz_nonsec_buf(int iommu_sec_id, int port, unsigned long mva, unsigned long size) { return -EPERM; /* Not allow */ } int m4u_unmap_gz_nonsec_buffer(int iommu_sec_id, unsigned long mva, unsigned long size) { return -EPERM; /* Not allow */ } #endif /* * inherent this from original m4u driver, we use this to make sure * we could still support * userspace ioctl commands. */ static long pseudo_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { int ret = 0; struct M4U_PORT_STRUCT m4u_port; switch (cmd) { case MTK_M4U_T_CONFIG_PORT: { ret = copy_from_user(&m4u_port, (void *)arg, sizeof(struct M4U_PORT_STRUCT )); if (ret) { M4U_MSG("copy_from_user failed:%d\n", ret); return -EFAULT; } ret = pseudo_config_port(&m4u_port, 1); } break; #ifdef PSEUDO_M4U_TEE_SERVICE_ENABLE case MTK_M4U_T_SEC_INIT: { M4U_MSG( "MTK M4U ioctl : MTK_M4U_T_SEC_INIT command!! 0x%x\n", cmd); ret = m4u_sec_init(); } break; #endif #ifdef M4U_GZ_SERVICE_ENABLE case MTK_M4U_GZ_SEC_INIT: { int mtk_iommu_sec_id = 0; M4U_MSG( "MTK M4U ioctl : MTK_M4U_GZ_SEC_INIT command!! 0x%x, arg:%d\n", cmd, arg); mtk_iommu_sec_id = arg; if (mtk_iommu_sec_id < 0 || mtk_iommu_sec_id >= SEC_ID_COUNT) return -EFAULT; mutex_lock(&gM4u_gz_sec_init); ret = m4u_gz_sec_init(mtk_iommu_sec_id); mutex_unlock(&gM4u_gz_sec_init); } break; #endif default: M4U_MSG("MTK M4U ioctl:No such command(0x%x)!!\n", cmd); ret = -EPERM; break; } return ret; } #if IS_ENABLED(CONFIG_COMPAT) static int compat_get_module_struct( struct COMPAT_M4U_MOUDLE_STRUCT __user *data32, struct M4U_MOUDLE_STRUCT __user *data) { compat_uint_t u; compat_ulong_t l; int err; err = get_user(u, &(data32->port)); err |= put_user(u, &(data->port)); err |= get_user(l, &(data32->BufAddr)); err |= put_user(l, &(data->BufAddr)); err |= get_user(u, &(data32->BufSize)); err |= put_user(u, &(data->BufSize)); err |= get_user(u, &(data32->prot)); err |= put_user(u, &(data->prot)); err |= get_user(u, &(data32->MVAStart)); err |= put_user(u, &(data->MVAStart)); err |= get_user(u, &(data32->MVAEnd)); err |= put_user(u, &(data->MVAEnd)); err |= get_user(u, &(data32->flags)); err |= put_user(u, &(data->flags)); return err; } static int compat_put_module_struct( struct COMPAT_M4U_MOUDLE_STRUCT __user *data32, struct M4U_MOUDLE_STRUCT __user *data) { compat_uint_t u; compat_ulong_t l; int err; err = get_user(u, &(data->port)); err |= put_user(u, &(data32->port)); err |= get_user(l, &(data->BufAddr)); err |= put_user(l, &(data32->BufAddr)); err |= get_user(u, &(data->BufSize)); err |= put_user(u, &(data32->BufSize)); err |= get_user(u, &(data->prot)); err |= put_user(u, &(data32->prot)); err |= get_user(u, &(data->MVAStart)); err |= put_user(u, &(data32->MVAStart)); err |= get_user(u, &(data->MVAEnd)); err |= put_user(u, &(data32->MVAEnd)); err |= get_user(u, &(data->flags)); err |= put_user(u, &(data32->flags)); return err; } long pseudo_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { long ret; if (!filp->f_op || !filp->f_op->unlocked_ioctl) return -ENOTTY; switch (cmd) { case COMPAT_MTK_M4U_T_ALLOC_MVA: { struct COMPAT_M4U_MOUDLE_STRUCT __user *data32; struct M4U_MOUDLE_STRUCT __user *data; int err; int module_size = sizeof(struct M4U_MOUDLE_STRUCT); data32 = compat_ptr(arg); data = compat_alloc_user_space(module_size); if (data == NULL) return -EFAULT; err = compat_get_module_struct(data32, data); if (err) return err; ret = filp->f_op->unlocked_ioctl(filp, MTK_M4U_T_ALLOC_MVA, (unsigned long)data); err = compat_put_module_struct(data32, data); if (err) return err; } break; case COMPAT_MTK_M4U_T_DEALLOC_MVA: { struct COMPAT_M4U_MOUDLE_STRUCT __user *data32; struct M4U_MOUDLE_STRUCT __user *data; int err; int module_size = sizeof(struct M4U_MOUDLE_STRUCT); data32 = compat_ptr(arg); data = compat_alloc_user_space(module_size); if (data == NULL) return -EFAULT; err = compat_get_module_struct(data32, data); if (err) return err; ret = filp->f_op->unlocked_ioctl(filp, MTK_M4U_T_DEALLOC_MVA, (unsigned long)data); } break; #ifdef PSEUDO_M4U_TEE_SERVICE_ENABLE case COMPAT_MTK_M4U_T_SEC_INIT: { M4U_MSG( "MTK_M4U_T_SEC_INIT command!! 0x%x\n", cmd); ret = m4u_sec_init(); } break; #endif #ifdef M4U_GZ_SERVICE_ENABLE case MTK_M4U_GZ_SEC_INIT: { int mtk_iommu_sec_id = arg; M4U_MSG( "MTK_M4U_GZ_SEC_INIT command!! 0x%x, arg: %d\n", cmd, arg); if (mtk_iommu_sec_id < 0 || mtk_iommu_sec_id >= SEC_ID_COUNT) return -EFAULT; mutex_lock(&gM4u_gz_sec_init); ret = m4u_gz_sec_init(mtk_iommu_sec_id); mutex_unlock(&gM4u_gz_sec_init); } break; #endif default: M4U_MSG("compat ioctl:No such command(0x%x)!!\n", cmd); ret = -ENOIOCTLCMD; break; } return ret; } #else #define pseudo_compat_ioctl NULL #endif static const struct file_operations pseudo_fops = { .owner = THIS_MODULE, .open = pseudo_open, .release = pseudo_release, .flush = pseudo_flush, .unlocked_ioctl = pseudo_ioctl, .compat_ioctl = pseudo_compat_ioctl, }; static int pseudo_probe(struct platform_device *pdev) { unsigned int i, j; #ifndef CONFIG_FPGA_EARLY_PORTING unsigned int count = 0; #endif int ret = 0; struct device_node *node = NULL; #if defined(CONFIG_MTK_SMI_EXT) M4U_MSG("%s: %d\n", __func__, smi_mm_first_get()); if (!smi_mm_first_get()) { M4U_MSG("SMI not start probe\n"); return -EPROBE_DEFER; } #endif for (i = 0; i < SMI_LARB_NR; i++) { /* wait for larb probe done. */ /* if (mtk_smi_larb_ready(i) == 0) { * M4U_MSG("pseudo_probe - smi not ready\n"); * return -EPROBE_DEFER; * } */ /* wait for pseudo larb probe done. */ if (!pseudo_dev_larb[i].dev && strcmp(pseudo_larbname[i], "m4u_none")) { M4U_MSG("%s: dev(%d) not ready\n", __func__, i); #ifndef CONFIG_FPGA_EARLY_PORTING return -EPROBE_DEFER; #endif } } #ifdef M4U_MTEE_SERVICE_ENABLE { /* init the sec_mem_size to 400M to avoid build error. */ unsigned int sec_mem_size = 400 * 0x100000; /*reserve mva range for sec */ struct device *dev = &pdev->dev; pseudo_session_init(); pseudo_sec_init(0, M4U_L2_ENABLE, &sec_mem_size); } #endif pseudo_mmu_dev = kzalloc(sizeof(struct m4u_device), GFP_KERNEL); if (!pseudo_mmu_dev) { M4U_MSG("kmalloc for m4u_device fail\n"); return -ENOMEM; } pseudo_mmu_dev->m4u_dev_proc_entry = proc_create("m4u", 0000, NULL, &pseudo_fops); if (!pseudo_mmu_dev->m4u_dev_proc_entry) { M4U_ERR("proc m4u create error\n"); return -ENODEV; } pseudo_debug_init(pseudo_mmu_dev); node = of_find_compatible_node(NULL, NULL, "mediatek,iommu_v0"); if (node == NULL) M4U_ERR("init iommu_v0 error\n"); pseudo_mmubase[0] = (unsigned long)of_iomap(node, 0); for (i = 0; i < SMI_LARB_NR; i++) { node = of_find_compatible_node(NULL, NULL, pseudo_larbname[i]); if (node == NULL) { pseudo_larbbase[i] = 0x0; M4U_ERR("cannot find larb %d, skip it\n", i); continue; } pseudo_larbbase[i] = (unsigned long)of_iomap(node, 0); /* set mm engine domain to 0x4 (default value) */ ret = larb_clock_on(i, 1); if (ret < 0) { M4U_MSG("larb%d clock on fail\n", i); continue; } #ifndef CONFIG_FPGA_EARLY_PORTING count = mtk_iommu_get_larb_port_count(i); for (j = 0; j < count; j++) { #ifdef SMI_LARB_SEC_CON_EN pseudo_set_reg_by_mask(pseudo_larbbase[i], SMI_LARB_SEC_CONx(j), F_SMI_DOMN(0x7), F_SMI_DOMN(0x4)); #endif // MDP path config if (m4u_port_id_of_mdp(i, j)) pseudo_set_reg_by_mask( pseudo_larbbase[i], SMI_LARB_NON_SEC_CONx(j), F_SMI_MMU_EN, 0x1); } #else j = 0; #endif larb_clock_off(i, 1); M4U_DBG("init larb%d=0x%lx\n", i, pseudo_larbbase[i]); } #ifdef PSEUDO_M4U_TEE_SERVICE_ENABLE if (!mtk_iommu_power_support()) { m4u_fb_notifier.notifier_call = m4u_fb_notifier_callback; ret = fb_register_client(&m4u_fb_notifier); if (ret) M4U_MSG("err fb_notifier failed! ret(%d)\n", ret); else M4U_MSG("register fb_notifier OK!\n"); } else M4U_MSG("pg_callback replace fb_notifier\n"); #endif pseudo_get_m4u_client(); if (IS_ERR_OR_NULL(ion_m4u_client)) { M4U_MSG("createclientfail\n"); return -ENOMEM; } spin_lock_init(&pseudo_list_lock); M4U_MSG("%s done\n", __func__); return 0; } static int pseudo_port_probe(struct platform_device *pdev) { int larbid = -1; unsigned int fake_nr, i; int ret; struct device *dev; struct device_dma_parameters *dma_param; /* dma will split the iova into max size to 65535 byte by default */ /* if we do not set this.*/ dma_param = kzalloc(sizeof(*dma_param), GFP_KERNEL); if (!dma_param) return -ENOMEM; /* set the iova to 256MB for one time map, should be suffice for ION */ dma_param->max_segment_size = 0x10000000; dev = &pdev->dev; dev->dma_parms = dma_param; #if (CONFIG_MTK_IOMMU_PGTABLE_EXT > 32) /* correct iova limit of pseudo device by update dma mask */ *dev->dma_mask = DMA_BIT_MASK(35); dev->coherent_dma_mask = DMA_BIT_MASK(35); #endif ret = of_property_read_u32(dev->of_node, "mediatek,larbid", &larbid); if (ret) goto out; fake_nr = ARRAY_SIZE(pseudo_dev_larb_fake); if (larbid >= 0 && larbid < SMI_LARB_NR) { pseudo_dev_larb[larbid].larbid = larbid; pseudo_dev_larb[larbid].dev = dev; pseudo_dev_larb[larbid].name = pseudo_larbname[larbid]; if (pseudo_dev_larb[larbid].mmuen) return 0; } else { for (i = 0; i < fake_nr; i++) { if (!pseudo_dev_larb_fake[i].dev && larbid == pseudo_dev_larb_fake[i].larbid) { pseudo_dev_larb_fake[i].dev = dev; break; } } if (i == fake_nr) { M4U_ERR("%s, pseudo not matched of dev larb%d\n", __func__, larbid); ret = -ENOMEM; goto out; } } M4U_MSG("done, larbid:%u, mask:0x%llx)\n", larbid, dev->coherent_dma_mask); return 0; out: kfree(dma_param); return ret; } /* * func: get the IOVA space of target port * port: user input the target port id * base: return the start addr of IOVA space * max: return the end addr of IOVA space * list: return the reserved region list * check the usage of struct iommu_resv_region */ int pseudo_get_iova_space(int port, unsigned long *base, unsigned long *max, struct list_head *list) { struct device *dev = pseudo_get_larbdev(port); int owner = -1; if (!dev) return -5; if (mtk_iommu_get_iova_space(dev, base, max, &owner, list) < 0) return -1; return 0; } /* * func: free the memory allocated by pseudo_get_iova_space() * list: user input the reserved region list returned by * pseudo_get_iova_space() */ void pseudo_put_iova_space(int port, struct list_head *list) { mtk_iommu_put_iova_space(NULL, list); } static int __pseudo_dump_iova_reserved_region(struct device *dev, struct seq_file *s) { unsigned long base, max; int domain, owner; struct iommu_resv_region *region; LIST_HEAD(resv_regions); domain = mtk_iommu_get_iova_space(dev, &base, &max, &owner, &resv_regions); if (domain < 0) { pr_notice("%s, %d, failed to get iova space\n", __func__, __LINE__); return domain; } M4U_PRINT_SEQ(s, "domain:%d, owner:%s(%d), from:0x%lx, to:0x%lx\n", domain, iommu_get_port_name(owner), owner, base, max); list_for_each_entry(region, &resv_regions, list) M4U_PRINT_SEQ(s, ">> reserved: 0x%llx ~ 0x%llx\n", region->start, region->start + region->length - 1); mtk_iommu_put_iova_space(dev, &resv_regions); return 0; } /* * dump the reserved region of IOVA domain * this is the initialization result of mtk_domain_array[] * and the mapping relationship between * pseudo devices of mtk_domain_array[] */ int pseudo_dump_iova_reserved_region(struct seq_file *s) { struct device *dev; unsigned int i, fake_nr; for (i = 0; i < SMI_LARB_NR; i++) { dev = pseudo_dev_larb[i].dev; if (!dev) continue; M4U_PRINT_SEQ(s, "======== %s ==========\n", pseudo_dev_larb[i].name); __pseudo_dump_iova_reserved_region(dev, s); } fake_nr = ARRAY_SIZE(pseudo_dev_larb_fake); for (i = 0; i < fake_nr; i++) { M4U_PRINT_SEQ(s, "======== %s ==========\n", pseudo_dev_larb_fake[i].name); dev = pseudo_dev_larb_fake[i].dev; if (!dev) continue; __pseudo_dump_iova_reserved_region(dev, s); } return 0; } EXPORT_SYMBOL(pseudo_dump_iova_reserved_region); int pseudo_m4u_sec_init(int mtk_iommu_sec_id) { int ret = 0; #if defined(M4U_GZ_SERVICE_ENABLE) if (mtk_iommu_sec_id >= 0 && mtk_iommu_sec_id < SEC_ID_COUNT) { mutex_lock(&gM4u_gz_sec_init); ret = m4u_gz_sec_init(mtk_iommu_sec_id); mutex_unlock(&gM4u_gz_sec_init); } #elif defined(PSEUDO_M4U_TEE_SERVICE_ENABLE) ret = m4u_sec_init(); #endif return ret; } static int pseudo_remove(struct platform_device *pdev) { if (pseudo_mmu_dev->m4u_dev_proc_entry) proc_remove(pseudo_mmu_dev->m4u_dev_proc_entry); pseudo_put_m4u_client(); M4U_MSG("client user count:%ld\n", ion_m4u_client->count); pseudo_destroy_client(ion_m4u_client); return 0; } static int pseudo_port_remove(struct platform_device *pdev) { return 0; } static int pseudo_suspend(struct platform_device *pdev, pm_message_t mesg) { return 0; } static int pseudo_resume(struct platform_device *pdev) { return 0; } static struct platform_driver pseudo_driver = { .probe = pseudo_probe, .remove = pseudo_remove, .suspend = pseudo_suspend, .resume = pseudo_resume, .driver = { .name = M4U_DEVNAME, .of_match_table = mtk_pseudo_of_ids, .owner = THIS_MODULE, } }; static struct platform_driver pseudo_port_driver = { .probe = pseudo_port_probe, .remove = pseudo_port_remove, .suspend = pseudo_suspend, .resume = pseudo_resume, .driver = { .name = "pseudo_port_device", .of_match_table = mtk_pseudo_port_of_ids, .owner = THIS_MODULE, } }; static int __init mtk_pseudo_init(void) { if (platform_driver_register(&pseudo_port_driver)) { M4U_MSG("failed to register pseudo port driver"); platform_driver_unregister(&pseudo_driver); return -ENODEV; } if (platform_driver_register(&pseudo_driver)) { M4U_MSG("failed to register pseudo driver"); return -ENODEV; } return 0; } static void __exit mtk_pseudo_exit(void) { platform_driver_unregister(&pseudo_driver); platform_driver_unregister(&pseudo_port_driver); } module_init(mtk_pseudo_init); module_exit(mtk_pseudo_exit); MODULE_DESCRIPTION("MTK pseudo m4u v2 driver based on iommu"); MODULE_LICENSE("GPL");