/* ================================================================== >>>>>>>>>>>>>>>>>>>>>>> COPYRIGHT NOTICE <<<<<<<<<<<<<<<<<<<<<<<<< ------------------------------------------------------------------ Copyright (c) 2019-2024 by Lattice Semiconductor Corporation ALL RIGHTS RESERVED ------------------------------------------------------------------ DISCLAIMER: LATTICE MAKES NO WARRANTIES ON THIS FILE OR ITS CONTENTS, WHETHER EXPRESSED, IMPLIED, STATUTORY, OR IN ANY PROVISION OF THE LATTICE PROPEL LICENSE AGREEMENT OR COMMUNICATION WITH LICENSEE, AND LATTICE SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. LATTICE DOES NOT WARRANT THAT THE FUNCTIONS CONTAINED HEREIN WILL MEET LICENSEE 'S REQUIREMENTS, OR THAT LICENSEE' S OPERATION OF ANY DEVICE, SOFTWARE OR SYSTEM USING THIS FILE OR ITS CONTENTS WILL BE UNINTERRUPTED OR ERROR FREE, OR THAT DEFECTS HEREIN WILL BE CORRECTED. LICENSEE ASSUMES RESPONSIBILITY FOR SELECTION OF MATERIALS TO ACHIEVE ITS INTENDED RESULTS, AND FOR THE PROPER INSTALLATION, USE, AND RESULTS OBTAINED THEREFROM. LICENSEE ASSUMES THE ENTIRE RISK OF THE FILE AND ITS CONTENTS PROVING DEFECTIVE OR FAILING TO PERFORM PROPERLY AND IN SUCH EVENT, LICENSEE SHALL ASSUME THE ENTIRE COST AND RISK OF ANY REPAIR, SERVICE, CORRECTION, OR ANY OTHER LIABILITIES OR DAMAGES CAUSED BY OR ASSOCIATED WITH THE SOFTWARE.IN NO EVENT SHALL LATTICE BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS FILE OR ITS CONTENTS, EVEN IF LATTICE HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. LATTICE 'S SOLE LIABILITY, AND LICENSEE' S SOLE REMEDY, IS SET FORTH ABOVE. LATTICE DOES NOT WARRANT OR REPRESENT THAT THIS FILE, ITS CONTENTS OR USE THEREOF DOES NOT INFRINGE ON THIRD PARTIES' INTELLECTUAL PROPERTY RIGHTS, INCLUDING ANY PATENT. IT IS THE USER' S RESPONSIBILITY TO VERIFY THE USER SOFTWARE DESIGN FOR CONSISTENCY AND FUNCTIONALITY THROUGH THE USE OF FORMAL SOFTWARE VALIDATION METHODS. ------------------------------------------------------------------ ================================================================== */ #include #include #include #include #include #include /* copy_to_user */ #include "version.h" #include "pcie_dma_regs.h" #include "pcie_dma.h" #ifdef pr_fmt #undef pr_fmt #define pr_fmt(fmt) KBUILD_MODNAME " %s: " fmt, __func__ #endif /** * @brief Waits for a condition to become true or a timeout to elapse with lock held. * * This macro will wait for a given condition to become true while holding a specified lock. * If the condition is already true, it returns immediately. Otherwise, it unlocks the given lock, * schedules a timeout, and then reacquires the lock before checking the condition again. * * @param[in] wq_head Wait queue head to wait on. * @param[in] condition Condition to be checked. * @param[in,out] lock Lock to be held during wait. * @param[in] timeout Timeout value in jiffies. * @param[in] state Task state to set for the waiting task. * * @return The remaining time in jiffies, or 0 if the condition was met before the timeout elapsed. */ #define __wait_event_lock_timeout(wq_head, condition, lock, timeout, state) \ ___wait_event(wq_head, ___wait_cond_timeout(condition), \ state, 0, timeout, \ spin_unlock_irq(&lock); \ __ret = schedule_timeout(__ret); \ spin_lock_irq(&lock)); /** * @brief Waits for a condition to become true or a timeout to elapse with lock held, uninterruptible. * * This macro is a wrapper around `__wait_event_lock_timeout` that sets the task state to * TASK_UNINTERRUPTIBLE. It waits for a condition to become true or a timeout to elapse while * holding a specified lock. If the condition is true at the start, it returns the original timeout value. * * @param[in] wq_head Wait queue head to wait on. * @param[in] condition Condition to be checked. * @param[in,out] lock Lock to be held during wait. * @param[in] timeout Timeout value in jiffies. * * @return The remaining time in jiffies, or the original timeout value if the condition was true initially. */ #define wait_event_lock_timeout(wq_head, condition, lock, timeout) \ ({ \ long __ret = timeout; \ if (!___wait_cond_timeout(condition)) \ __ret = __wait_event_lock_timeout( \ wq_head, condition, lock, timeout, \ TASK_UNINTERRUPTIBLE); \ __ret; \ }) /** * PROTOTYPE DEC */ static int dma_debug = 0; module_param(dma_debug, int, 0); MODULE_PARM_DESC(debug, "dma enable debugging (0-1)"); /** * List of boards we will attempt to find and associate with the driver. */ static const char *BoardName[4] = {"LSC", "LSC", "DMA", "DMA"}; static const char *DemoName[3] = {"DMA", "DMA", "BASIC"}; static unsigned long counter = 0; static const char *ch0_msi_name[] = {"pciedmaCh0DoneMSI", ""}; static struct PCIEDMA global_pciedma; static struct pci_device_id lattice_pci_id_tbl[] = { //{0x1204, 0x9C3e,0x19AA,0xE004,}, //{0x1204, 0x0001,0x1204,0x0001,}, {0x1204, 0x9c25,0x19AA,0xE004,}, {} /* Terminating entry */ }; MODULE_DEVICE_TABLE(pci, lattice_pci_id_tbl); /*Info structure which use to store the driver info*/ pcie_board_t pBrd; char gDrvrName[] = "pciedma"; /** * Read a 8 bit hardware register and return the word. * * @param pBrd pointer to the Board to get hardware BAR address * @param offset in board's mapped control BAR to read from * @return 8 bit value read from the hardware register */ static uint8_t rdReg8(pcie_board_t *pBrd, uint16_t offset) { return(readb(pBrd->ctrlBARaddr + offset)); } /** * Read a 16 bit hardware register and return the word. * * @param pBrd pointer to the Board to get hardware BAR address * @param offset in board's mapped control BAR to read from * @return 16 bit value read from the hardware register */ static uint16_t rdReg16(pcie_board_t *pBrd, uint16_t offset) { return(readw(pBrd->ctrlBARaddr + offset)); } /** * Read a 32 bit hardware register and return the word. * * @param pBrd pointer to the Board to get hardware BAR address * @param offset in board's mapped control BAR to read from * @return 32 bit value read from the hardware register */ static uint32_t rdReg32(pcie_board_t *pBrd, uint16_t offset) { return (readl(pBrd->ctrlBARaddr + offset)); } /** * Write a 8 bit value to a hardware register. * * @param pBrd pointer to Board to get hardware BAR address * @param offset in the board's mapped control BAR to write to * @param val the 8 bit value to write */ static void wrReg8(pcie_board_t *pBrd, uint16_t offset, uint8_t val) { writeb(val, pBrd->ctrlBARaddr + offset); } /** * Write a 16 bit value to a hardware register. * * @param pBrd pointer to Board to get hardware BAR address * @param offset in the board's mapped control BAR to write to * @param val the 16 bit value to write */ static void wrReg16(pcie_board_t *pBrd, uint16_t offset, uint16_t val) { writew(val, pBrd->ctrlBARaddr + offset); } /** * Write a 32 bit value to a hardware register. * * @param pBrd pointer to Board to get hardware BAR address * @param offset in the board's mapped control BAR to write to * @param val the long value to write */ static void wrReg32(pcie_board_t *pBrd, uint16_t offset, uint32_t val) { writel(val, pBrd->ctrlBARaddr + offset); wmb(); } /** * Read and then Write a 32 bit value to a hardware register. * * @param pBrd pointer to Board to get hardware BAR address * @param offset in the board's mapped control BAR to write to * @param mask to mask the exact bits to modify * @param val the long value to write */ static void wrReg32Modify(pcie_board_t *pBrd, uint16_t offset, uint32_t mask, uint32_t val) { uint32_t result; result = rdReg32(pBrd, offset); result &= (~mask); result |= (mask & val); mdelay(5); wrReg32(pBrd, offset, result); } /** * rdRegPciCfg - Read a value from the PCI configuration register at the specified offset. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board. * @param offset: Offset in the PCI Config Space Register. * * This function reads the value from the PCI configuration register at the given offset. * If the read operation fails, an error code is returned. * * @return Value read from the register, or ERR (error code) on failure. */ static uint32_t rdRegPciCfg (pcie_board_t *pBrd, uint16_t offset) { uint32_t pciReg; if (pci_read_config_dword(pBrd->pdev, offset, &pciReg) < 0) { printk("%s: rdRegPciCfg Device Configuration Reading failed.",gDrvrName); return ERR; } return pciReg; } /** * wrRegPciCfg - Write a 32-bit value to the PCI configuration register. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board. * @param offset: Offset of the configuration register. * @param val: Value to write to the register. * * This function writes the specified 32-bit value to the PCI configuration register * at the given offset. If the write operation fails, an error code is returned. * * @return 0 on success, ERR (error code) on failure. */ static uint32_t wrRegPciCfg (pcie_board_t *pBrd, uint16_t offset, uint32_t val) { if (pci_write_config_dword(pBrd->pdev, offset, val) < 0) { printk("%s: wrRegPciCfg Device Configuration Writing failed.",gDrvrName); return ERR; } return 0; } /** * @brief IRQ handler for PCIe DMA operations. * * This function handles the interrupts generated by the PCIe DMA operations. * It checks the interrupt vector, updates the DMA status, and wakes up the * waiting queue if the DMA operation is complete. * * @param irq The interrupt request number. * @param arg A pointer to the PCIe board structure. * @return IRQ_HANDLED indicating the interrupt was handled. * * The function performs the following steps: * - Initializes the DMA done statuses to DMA_NOT_DONE. * - Prints debug information if dma_debug is enabled. * - Increments a counter and logs the IRQ vector every 100 interrupts. * - Checks if the interrupt vector matches the expected value. * - If it matches, sets the DMA done statuses to DMA_IS_DONE and wakes up the waiting queue. * - If it does not match, keeps the DMA done statuses as DMA_NOT_DONE. */ static irqreturn_t PcieDmaIrqHandler(int irq, void *arg) { uint32_t read_val = 0; pcie_board_t *pBrd = (pcie_board_t *)arg; pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done = DMA_NOT_DONE; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done = DMA_NOT_DONE; if (dma_debug) { printk("H2F DMA Done Status at IRQ Handler Pre: 0x%x\n",pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done); printk("F2H DMA Done Status at IRQ Handler Pre: 0x%x\n",pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done); } counter++; if (counter % 100 == 0) read_val = (rdReg32(pBrd, H2F_DMA_CTRL)); read_val = (rdReg32(pBrd, F2H_DMA_CTRL)); if (pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[0] == irq) { pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done = DMA_IS_DONE; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done = DMA_IS_DONE; wake_up(&pBrd->DMA.ReadWaitQ); } else { printk("IRQ Vector:%d ", irq); pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done = DMA_NOT_DONE; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done = DMA_NOT_DONE; } return IRQ_HANDLED; } /** * Read PCI configuration registers for the given PCIe board. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board * * This function reads the PCI configuration registers for the specified PCIe board. * It iterates through the registers, retrieves their values, and stores them in the * pBrd->PCICfgRegs array. If an error occurs during reading, an appropriate error * code is returned. * * @return 0 on success, negative error code on failure */ static int ReadPCIConfigRegs(pcie_board_t *pBrd) { int status; int i; for (i = 0; i < 0x100; i = i + 4) { status = pci_read_config_dword(pBrd->pdev, i, &(pBrd->PCICfgRegs[i/4])); if (status) { pr_err("calling pci_read_config_dword(%d)\n", i); return(status); } } if (dma_debug) { pr_info("PCICfgRegs 00: %x %x %x %x\n", pBrd->PCICfgRegs[0], pBrd->PCICfgRegs[1], pBrd->PCICfgRegs[2], pBrd->PCICfgRegs[3]); pr_info("PCICfgRegs 10: %x %x %x %x\n", pBrd->PCICfgRegs[4], pBrd->PCICfgRegs[5], pBrd->PCICfgRegs[6], pBrd->PCICfgRegs[7]); pr_info("PCICfgRegs 20: %x %x %x %x\n", pBrd->PCICfgRegs[8], pBrd->PCICfgRegs[9], pBrd->PCICfgRegs[10], pBrd->PCICfgRegs[11]); pr_info("PCICfgRegs 30: %x %x %x %x\n", pBrd->PCICfgRegs[12], pBrd->PCICfgRegs[13], pBrd->PCICfgRegs[14], pBrd->PCICfgRegs[15]); } return(status); } /** * ParsePCIeLinkCap - Parse PCIe capabilities structures for the given PCIe board. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board. * * This function parses the PCIe capabilities structures from the PCI configuration * registers. It identifies and processes various capability types, such as Power * Management, AGP, VPD, MSI, PCI-X, and PCI Express. The relevant information is * extracted and stored in the pcie_board_t structure. * * @return 0 on success, -1 if no valid PCIe capabilities structure is found. */ static int ParsePCIeLinkCap(pcie_board_t *pBrd) { int i, id, next, index; uint8_t *buf; uint32_t *pPCI; uint8_t *p8; uint16_t *p16; uint32_t *p32; bool found = false; pPCI = pBrd->PCICfgRegs; buf = (uint8_t *)pBrd->PCICfgRegs; if (ReadPCIConfigRegs(pBrd) != 0) { pr_err("error Reading Config Regs!\n"); return(-1); } if (((pPCI[STAT_CMD_REG] & 0x00100000) == 0) || (pPCI[CAP_PTR_REG] == 0)) { pr_err("no PCIe Capabilities Structure!\n"); return(-1); } i = 0; next = (int)pPCI[CAP_PTR_REG] & 0x000000ff; while ((next >= 0x40) && (i < 16)) { ++i; // Loop counter to prevent circular loop index = next; id = buf[next]; p8 = (uint8_t *)&buf[next]; p16 = (uint16_t *)&buf[next]; p32 = (uint32_t *)&buf[next]; next = (int)buf[next + 1]; switch(id) { case 1: // Power Management pr_info("Power Management Capability Structure @ %x\n", index); break; case 2: // AGP Capability pr_info("AGP Capability Structure @ %x\n", index); break; case 3: // VPD (Vital Product Data) Capability pr_info("VPD Capability Structure @ %x\n", index); break; case 4: // Slot ID Capability pr_info("Slot ID Capability Structure @ %x\n", index); break; case 5: // MSI pr_info("MSI Capability Structure @ %x\n", index); break; case 6: // CompactPCI Hot Swap pr_info("CompactPCI Capability Structure @ %x\n", index); break; case 7: // PCI-X pr_info("PCI-X Capability Structure @ %x\n", index); break; case 8: // AMD pr_info("AMD Capability Structure @ %x\n", index); break; case 9: // Vendor Specific pr_info("Vendor Specific Capability Structure @ %x\n", index); break; case 0x0a: // Debug Port pr_info("Debug Port Capability Structure @ %x\n", index); break; case 0x0b: // CompactPCI central resource control pr_info("CompactPCI resource Capability Structure @ %x\n", index); break; case 0x0c: // PCI Hot Plug pr_info("PCI Hot Plug Capability Structure @ %x\n", index); break; case 0x10: // PCI Express pr_info("PCI Express Capability Structure @ %x\n", index); pBrd->PCIeMaxReadReqSize = (128<<((p16[4] & 0x7000)>>12)); pBrd->PCIeMaxPayloadSize = (128<<((p16[4] & 0x0e)>>5)); if (p16[8] & 0x0008) pBrd->PCIeRCBSize = 128; else pBrd->PCIeRCBSize = 64; pBrd->PCIeLinkWidth = ((p16[9] & 0x03f0)>>4); pr_info("MaxPayloadSize = %d\n", pBrd->PCIeMaxPayloadSize); pr_info("MaxReadReqSize = %d\n", pBrd->PCIeMaxReadReqSize); pr_info("RCBSize = %d\n", pBrd->PCIeRCBSize); pr_info("LinkWidth = x%d\n", pBrd->PCIeLinkWidth); // Slot Registers and Root Registers not implemented by our EndPoint core found = true; break; default: return( -1); break; } } if (found) return(0); else return( -1); } /** * Release user buffer pages. * @param info: Pointer to the ubuf_info structure containing page information. * * Releases the user buffer pages previously acquired using GetUbufPages. * After processing all pages, the function frees the memory allocated for the pages array. * */ static void PutUbufPages(struct ubuf_info *info) { int i; for (i = 0; i < info->nr_pages; i++) { set_page_dirty(info->pages[i]); put_page(info->pages[i]); } kfree(info->pages); } /** * Retrieve user buffer pages for the specified address range. * * @param info: Pointer to the ubuf_info structure to store retrieved information. * @param uaddr: User-space address of the buffer. * @param size: Size of the buffer. * * This function retrieves the user buffer pages corresponding to the specified * address range. It calculates the page range, allocates memory for page pointers, * and populates the ubuf_info structure. If successful, it returns 0; otherwise, * an error code is returned. * * @return 0 on success, negative error code on failure. */ static int GetUbufPages(struct ubuf_info *info, unsigned long uaddr, unsigned long size) { unsigned long first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; unsigned long last = ((uaddr+size-1) & PAGE_MASK) >> PAGE_SHIFT; unsigned int flags = FOLL_FORCE | FOLL_WRITE; // TODO int ret, i; info->offset = uaddr & ~PAGE_MASK; info->nr_pages = last-first+1; info->size = size; if (dma_debug) { pr_info("info offset = %d \n", info->offset); pr_info("info nr_pages = %d \n", info->nr_pages); pr_info("info size = %zu \n", info->size); } // info->pages = kmalloc(info->nr_pages * sizeof(struct page *), GFP_KERNEL); info->pages = kmalloc(info->nr_pages * sizeof(struct page *), GFP_DMA); if (!info->pages) return -ENOMEM; if (dma_debug) { pr_info("%#lx + %ld => %d pages\n", uaddr, size, info->nr_pages); } ret = get_user_pages_unlocked(uaddr & PAGE_MASK, info->nr_pages, info->pages, flags); if (ret != info->nr_pages) goto fail_map; return 0; fail_map: pr_err("get_user_pages fail: ret=%d [%d]\n", ret, info->nr_pages); if (ret > 0) { for (i = 0; i < ret; i++) put_page(info->pages[i]); ret = -EINVAL; } kfree(info->pages); return ret; } /** * Free scatter-gather (SG) and user buffer resources. * @param pBrd: Pointer to the PCIe board structure. * * Cleans up all the scatter-gather lists and user buffer information * associated with a specific DMA channel on the PCIe board. * Releasing the resources for both the SG table and * the user buffer info. If the SG table or user buffer info is present, it will: * PutUbufPages to release the user buffer pages. * Free the memory allocated for the ubuf_info structure. * sg_free_table to free the SG table resources. * Free the memory allocated for the sg_table structure. * Sets the pointers in the DMA channel's arrays to NULL, indicating * that the resources have been released. */ static void UbufSgFree(pcie_board_t *pBrd) { int i; struct sg_table *table; struct ubuf_info *info; // for (i = 0; i < MAX_NUM_DESCRIPTORS; i++) for (i = 0; i < pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt; i++) { table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; info = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_info[i]; if (info) { PutUbufPages(info); kfree(info); } if (table) { sg_free_table(table); kfree(table); } pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i] = NULL; pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_info[i] = NULL; } } /** * Unmap scatter-gather (SG) buffers and free associated resources. * @param pBrd: Pointer to the PCIe board structure. * * Unmaps the SG buffers associated with a specific DMA channel on the PCIe board. * unmapping the SG buffers using dma_unmap_sg(). If an SG table is present, it * unmaps the entire table. After unmapping, UbufSgFree() releases the * user buffer resources (including freeing memory and decrementing reference counts). */ static void PcieDmaUbufUnmap(pcie_board_t *pBrd) { int i; struct sg_table *table; pr_info("Unmapping UBuf\n"); //for (i = 0; i < MAX_NUM_DESCRIPTORS; i++) { for (i = 0; i < pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt; i++) { table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; if(table) dma_unmap_sg(&pBrd->pdev->dev, table->sgl, table->nents, DMA_BIDIRECTIONAL); } UbufSgFree(pBrd); } /** * Allocate and initialize scatter-gather (SG) buffers for user space memory. * @param pBrd: Pointer to the PCIe board structure. * @param idx: Index of the descriptor within the DMA channel. * @param addr: User space address of the buffer. * @param len: Size of the buffer in bytes. * * Set up SG buffers for a given user space memory for DMA Operation * buffer to be used in DMA operations. * Allocates memory for the ubuf_info structure to store page information. * Invokes GetUbufPages to acquire user buffer pages and populate the ubuf_info. * Allocates memory for the SG table structure. * sg_alloc_table_from_pages creates the SG table from acquired pages. * Stores the pointers to the SG table and ubuf_info in the DMA channel's descriptor array. */ static int UbufSgAlloc(pcie_board_t *pBrd, int idx, void *addr, size_t len) { unsigned long uaddr = (unsigned long)addr; struct ubuf_info *info; struct sg_table *table; int ret; printk("UbufSgAlloc uaddr: 0x%lx", uaddr); info = kmalloc(sizeof(*info), GFP_DMA); if (!info) { return -ENOMEM; } ret = GetUbufPages(info, uaddr, len); if (ret) goto fail_init_ubuf; table = kmalloc(sizeof(*table), GFP_DMA); if (!table) { ret = -ENOMEM; goto fail_table; } ret = sg_alloc_table_from_pages(table, info->pages, info->nr_pages, info->offset, info->size, GFP_DMA); if (ret){ goto fail_sg; } pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[idx] = table; pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_info[idx] = (void *)info; return 0; fail_sg: kfree(table); fail_table: PutUbufPages(info); fail_init_ubuf: kfree(info); return ret; } /** * PcieDmaUbufMap - Map user buffer pages for DMA operations. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board. * @param uaddr: Array of user-space addresses for the buffer descriptors. * @param len: Length of the buffer. * @param desc_cnt: Number of descriptors. * * This function maps user buffer pages for DMA operations. It allocates scatter-gather * tables, allocates and maps pages, and prepares the buffer for DMA transfers. If any * allocation or mapping fails, appropriate cleanup is performed, and an error code is * returned. * * @return 0 on success, negative error code on failure. */ static int PcieDmaUbufMap(pcie_board_t *pBrd, void **uaddr, int len, int desc_cnt) { struct sg_table *table; int i; //printk("PcieDmaUbufMap function uaddr: 0x%p", uaddr); for (i = 0; i < desc_cnt; i++) { //printk("PcieDmaUbufMap inside for loop uaddr: 0x%p", uaddr[i]); if (UbufSgAlloc(pBrd, i, uaddr[i], len)) { UbufSgFree(pBrd); return -ENOMEM; } table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; if (!dma_map_sg(&pBrd->pdev->dev, table->sgl, table->nents, DMA_BIDIRECTIONAL)) { PcieDmaUbufUnmap(pBrd); return (-ENOMEM); } } return 0; } /** *Free DMA F2H descriptors. * @param pBrd: Pointer to the PCIe board structure. * * Frees the DMA F2H descriptors and associated scatter-gather (SG) tables * for each descriptor in the DMA channel. * Retrieves the pointer to the current descriptor and its SG table. * If both the descriptor and the SG table are present, it frees the coherent * memory allocated for the descriptor using dma_free_coherent(). * Sets the pointer to the descriptor in the DMA channel to NULL. */ static void PcieDmaF2hDescFree(pcie_board_t *pBrd) { dma_f2h_desc_hw_t *dma_f2h_desc; struct sg_table *table; int i; pr_info("Free F2H DMA desc \n"); for (i = 0; i < pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt; i ++) { dma_f2h_desc = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i]; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; if (dma_f2h_desc && table) dma_free_coherent(&pBrd->pdev->dev, (table->nents + 1) * sizeof(dma_f2h_desc_hw_t), dma_f2h_desc, pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[i]); pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i] = NULL; } } /** *Free DMA H2F descriptors. * @param pBrd: Pointer to the PCIe board structure. * * Frees the DMA H2F descriptors and associated scatter-gather (SG) tables * for each descriptor in the DMA channel. * Retrieves the pointer to the current descriptor and its SG table. * If both the descriptor and the SG table are present, it frees the coherent * memory allocated for the descriptor using dma_free_coherent(). * Sets the pointer to the descriptor in the DMA channel to NULL. */ static void PcieDmaH2fDescFree(pcie_board_t *pBrd) { dma_h2f_desc_hw_t *dma_h2f_desc; struct sg_table *table; int i; pr_info("Free H2F DMA desc \n"); for (i = 0; i < pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt; i++) { dma_h2f_desc = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i]; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; if (dma_h2f_desc && table) dma_free_coherent(&pBrd->pdev->dev, (table->nents + 1) * sizeof(dma_h2f_desc_hw_t), dma_h2f_desc, pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[i]); pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i] = NULL; } } /** * Start DMA Operation for F2H DMA Transfer * Checks if the DMA Request is already cleared from prior request before * writing 1 to the Request Bit * @param pBrd pointer to device extension to get hardware BAR address * @return 0 on success, negative error code on failure. */ static int PcieDmaF2hStartTransfer(pcie_board_t *pBrd) { uint32_t read_val; read_val = (rdReg32(pBrd, F2H_DMA_CTRL)) & START_DMA_MASK; if(read_val != START_DMA_IS_CLEAR) { pr_err("DMA REQUEST BIT NOT CLEARED YET\n"); return ERR; } else { wrReg32(pBrd, F2H_DMA_CTRL, START_DMA); } return 0; } /** * Start DMA Operation for H2F DMA Transfer * Checks if the DMA Request is already cleared from prior request before * writing 1 to the Request Bit * @param pBrd pointer to device extension to get hardware BAR address * @return 0 on success, negative error code on failure. */ static int PcieDmaH2fStartTransfer(pcie_board_t *pBrd) { uint32_t read_val; read_val = (rdReg32(pBrd, H2F_DMA_CTRL)) & START_DMA_MASK; if(read_val != START_DMA_IS_CLEAR) { pr_err("DMA REQUEST BIT NOT CLEARED YET\n"); return ERR; } else { wrReg32(pBrd, H2F_DMA_CTRL, START_DMA); } return 0; } /** * Stop the DMA F2H transfer. * @param pBrd: Pointer to the PCIe board structure. * * Stops the DMA F2H transfer by updating the control * field of the last descriptor in the DMA channel. * @return 0 on success, negative error code on failure. */ static int PcieDmaF2hStopTransfer(pcie_board_t *pBrd) { struct sg_table *table; dma_f2h_desc_hw_t *desc_entry; unsigned int num_sg = 0; int num_sg_max; int cont_desc_rem = 0; desc_entry = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[0]; num_sg = pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_num_sg_0; num_sg_max = num_sg - 1; pr_info("F2H DMA Stop \n"); if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max].f2h_desc_ctrl = (CONT_DESC_64 << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else { desc_entry[num_sg_max].f2h_desc_ctrl = (num_sg << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } mdelay(1); return 0; } /** * Stop the DMA H2F transfer. * @param pBrd: Pointer to the PCIe board structure. * * Stops the DMA H2F transfer by updating the control * field of the last descriptor in the DMA channel. * @return 0 on success, negative error code on failure. */ static int PcieDmaH2fStopTransfer(pcie_board_t *pBrd) { struct sg_table *table; dma_h2f_desc_hw_t *desc_entry; unsigned int num_sg = 0; int num_sg_max; int cont_desc_rem = 0; desc_entry = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[0]; num_sg = pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_num_sg_0; num_sg_max = num_sg - 1; pr_info("H2F DMA Stop \n"); if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max].h2f_desc_ctrl = (CONT_DESC_64 << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else { desc_entry[num_sg_max].h2f_desc_ctrl = (num_sg << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } mdelay(1); return 0; } /** * Populate DMA F2H descriptors. * @param pBrd: Pointer to the PCIe board structure. * @param num_desc: Index of the descriptor to be populated. * * Populates the DMA F2H descriptors with the necessary control flags, * lengths, and addresses for a DMA transfer from the host. It retrieves the SG table * and the descriptor entry for the specified descriptor index. The function then iterates * over each scatterlist entry in the SG table, setting up the descriptor entries with: * - Descriptor control flags based on the number of SG entries. * - DMA length from the scatterlist. * - Physical addresses for the next descriptor, source, and destination. */ static void PcieDmaF2hDescFill(pcie_board_t *pBrd, uint16_t num_desc) { struct sg_table *table; struct scatterlist *sg; dma_f2h_desc_hw_t *desc_entry; unsigned int num_sg = 0; unsigned long long desc_phy_addr; unsigned long long source_address; uint32_t addr_len_sum = 0; int i; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; num_sg = table->nents; unsigned long long source_address_arr[num_sg]; desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[num_desc]; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[num_desc]; /** * Store an incremental sum of the DMA Length to be used as Descriptor Address * for F2H Transfer */ for_each_sg(table->sgl, sg, num_sg, i) { addr_len_sum = addr_len_sum + sg_dma_len(sg); source_address_arr[i] = addr_len_sum; } for_each_sg(table->sgl, sg, num_sg, i) { if ((i == 0) && (num_desc == 0)) { source_address = 0; } else if ((i > 0) && (num_desc == 0)) { source_address = source_address_arr[(i-1)]; } if ((i == 0) && (num_desc > 0)) { source_address = num_desc * pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_size; } else if ((i > 0) && (num_desc > 0)) { source_address = num_desc * pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_size + source_address_arr[(i-1)]; } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[i].f2h_desc_ctrl = (CONT_DESC_64 << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } else { desc_entry[i].f2h_desc_ctrl = (num_sg << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[i].f2h_dma_len = sg_dma_len(sg); desc_entry[i].f2h_next_desc_addr_lo = (desc_phy_addr & 0xffffffff) + (32 * (i+1)); desc_entry[i].f2h_next_desc_addr_hi = desc_phy_addr >> 32; desc_entry[i].f2h_src_addr_lo = source_address & 0xffffffff; desc_entry[i].f2h_src_addr_hi = source_address >> 32; desc_entry[i].f2h_dest_addr_lo = sg_dma_address(sg); desc_entry[i].f2h_dest_addr_hi = sg_dma_address(sg) >> 32; if(dma_debug) { if(i < 50) { printk("==========Descriptor %d | Entry %d========================================\n", num_desc, i); printk("Entry[%d] Descriptor Control DW0: 0x%x\n", i, desc_entry[i].f2h_desc_ctrl); printk("Entry[%d] DMA Length DW1: 0x%x\n", i, desc_entry[i].f2h_dma_len); printk("Entry[%d] Next Descriptor Address Low DW2: 0x%x\n", i, desc_entry[i].f2h_next_desc_addr_lo); printk("Entry[%d] Next Descriptor Address High DW3: 0x%x\n", i, desc_entry[i].f2h_next_desc_addr_hi); printk("Entry[%d] Source Address Low DW4: 0x%x\n", i, desc_entry[i].f2h_src_addr_lo); printk("Entry[%d] Source Address High DW5: 0x%x\n", i, desc_entry[i].f2h_src_addr_hi); printk("Entry[%d] Destination Address Low DW6: 0x%x\n", i, desc_entry[i].f2h_dest_addr_lo); printk("Entry[%d] Destination Address High DW7: 0x%x\n", i, desc_entry[i].f2h_dest_addr_hi); } } } } /** * Configure the last entry of the DMA F2H Descriptor. * * Sets up the descriptor entries for a F2H DMA transfer . * It handles the configuration of the last descriptor entry based on the number of * scatter-gather entries, the maximum number of descriptors, and the descriptor mode. * * @param pBrd: Pointer to the PCIe board structure. * @param num_desc: The index of the current descriptor being processed. * @param max_desc: The total number of descriptors available. * * Processes the scatter-gather table for the current descriptor and * calculates the remaining continuous descriptors. It then sets the control bits * for the last descriptor entry, including enabling interrupts and marking the end * of the packet (EOP) if necessary. If the descriptor mode is circular, it links * the last descriptor back to the first. */ static void PcieDmaF2hLastEntry(pcie_board_t *pBrd, uint16_t num_desc, uint16_t max_desc) { struct sg_table *table; struct sg_table *next_table; dma_f2h_desc_hw_t *desc_entry; unsigned int num_sg = 0; unsigned int num_sg_next = 0; unsigned long long desc_phy_addr; int next_desc; int num_sg_max; int cont_desc_rem = 0; next_desc = num_desc + 1; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; if (num_desc < (max_desc -1)) { next_table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[next_desc]; } else { next_table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; } desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[num_desc]; num_sg = table->nents; num_sg_next = next_table->nents; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[num_desc]; num_sg_max = num_sg - 1; if ((num_sg) >= CONT_DESC_MAX) { cont_desc_rem = (num_sg) % CONT_DESC_MAX; } else if (num_desc < (max_desc - 1)) { cont_desc_rem = num_sg_next; } else if (num_desc == (max_desc -1)) { cont_desc_rem = num_sg; } if ((cont_desc_rem == 0) && (max_desc >= 1) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 0 is true\n"); printk("No Contiguous Descriptor remain, \nThere are 1 or more Descriptors, \nThis Descriptor is the Last Descriptor\n"); } desc_entry[num_sg_max].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else if ((cont_desc_rem == 0) && (max_desc > 1) && (num_desc < (max_desc -1))) { if (dma_debug) { printk("Condition 1 is true\n"); printk("No Contiguous Descriptor remain, \nThere are more than 1 Descriptors, \nThis Descriptor is NOT the Last Descriptor\n"); } desc_entry[num_sg_max].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].f2h_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[next_desc]) & 0xffffffff; desc_entry[num_sg_max].f2h_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[next_desc]) >> 32; } else if ((cont_desc_rem > 0) && (max_desc >= 1) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 2 is true\n"); printk("There are Contiguous Descriptor remaining, \nThere are 1 or more Descriptors, \nThis Descriptor is the Last Descriptor\n"); } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max - cont_desc_rem].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[num_sg_max].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else if ((cont_desc_rem > 0) && (max_desc > 1) && (num_desc < (max_desc -1))) { if (dma_debug) { printk("Condition 3 is true\n"); printk("There are Contiguous Descriptor remaining, \nThere are more than 1 Descriptors, \nThis Descriptor is NOT the Last Descriptor\n"); } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max - cont_desc_rem].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[num_sg_max].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].f2h_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[next_desc]) & 0xffffffff; //wlsoh exp desc_entry[num_sg_max].f2h_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[next_desc]) >> 32; //wlsoh exp } if ((pBrd->DMA.chan[DMA_CHANNEL_NUM].desc_mode == DESC_MODE_CIRCULAR) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 4 is true\n"); printk("This is a Circular Descriptor, \nThis Descriptor is the Last Descriptor\n"); } desc_entry[num_sg_max].f2h_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].f2h_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[0]) & 0xffffffff; desc_entry[num_sg_max].f2h_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[0]) >> 32; } if(dma_debug) { printk("==========Override Last Entry/Descriptor==================================\n"); printk("Entry[%d] Descriptor Control DW0: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_desc_ctrl); printk("Entry[%d] DMA Length DW1: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_dma_len); printk("Entry[%d] Next Descriptor Address Low DW2: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_next_desc_addr_lo); printk("Entry[%d] Next Descriptor Address High DW3: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_next_desc_addr_hi); printk("Entry[%d] Source Address Low DW4: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_src_addr_lo); printk("Entry[%d] Source Address High DW5: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_src_addr_hi); printk("Entry[%d] Destination Address Low DW6: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_dest_addr_lo); printk("Entry[%d] Destination Address High DW7: 0x%x\n", num_sg_max, desc_entry[num_sg_max].f2h_dest_addr_hi); printk("==========================================================================\n"); } } /** * PcieDmaF2hDescGen - Generate PCIe DMA F2H descriptors for the given PCIe board. * * @param pBrd: Pointer to the pcie_board_t structure representing the PCIe board * * This function generates F2H (from FPGA to host) descriptors for the specified PCIe board. * It allocates memory for the descriptors, fills in SG-element information, and configures * DMA hardware registers accordingly. The function also handles cases where the number of * entries exceeds the maximum contiguous descriptor limit. * * @return 0 on success, negative error code on failure */ static int PcieDmaF2hDescGen(pcie_board_t *pBrd) { int i; uint32_t read_val = 0; struct sg_table *table; unsigned int num_sg, sum_sg = 0; dma_f2h_desc_hw_t *desc_entry; unsigned long desc_phy_addr; size_t desc_size; int desc_count = 0; int cont_desc_count = 0; int cont_desc_rem = 0; desc_count = pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_total_desc_entry = 0; if (dma_debug) { printk("========== F2H Process Starts Here ==========\n"); } for (i = 0; i < desc_count; i++) { table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; desc_size = (table->nents) * sizeof(dma_f2h_desc_hw_t); pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_total_desc_entry = pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_total_desc_entry + (table->nents); if (dma_debug) { printk("Number of Entries: %d", table->nents); printk("Desc Size: %zu \n", desc_size); } pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i] = dma_alloc_coherent(&pBrd->pdev->dev, desc_size, &pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[i], GFP_KERNEL | __GFP_ZERO); if (!pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i]) { PcieDmaF2hDescFree(pBrd); return -ENOMEM; } if(dma_debug) { printk("Desc[%d]: Desc Virt Addr %px Desc Phy Addr: %llx, Desc Size = %zu\n", i, pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i], pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[i], desc_size); printk("Total Number of F2H Descriptor Entries:%d\n",pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_total_desc_entry); } } for (i = 0; i < desc_count; i++) { desc_size = (table->nents) * sizeof(dma_f2h_desc_hw_t); table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc[i]; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_f2h_desc_handle[i]; num_sg = table->nents; if (i == 0) { pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_num_sg_0 = num_sg; } sum_sg += num_sg; if(dma_debug) printk("=== Descriptor[%d] has %d Number of Entries (nents) ===\n", i, num_sg); cont_desc_count = (num_sg)/CONT_DESC_MAX; cont_desc_rem = (num_sg)%CONT_DESC_MAX; PcieDmaF2hDescFill(pBrd, i); PcieDmaF2hLastEntry(pBrd, i, desc_count); /** * Write the Very First Descriptor Entry Address in to the DMA HW Registers */ if (i == 0) { wrReg32(pBrd, F2H_DESC_ADDR_HIGH, (uint32_t) (desc_phy_addr >> 32)); mdelay(1); wrReg32(pBrd, F2H_DESC_ADDR_LOW, (uint32_t)(desc_phy_addr)); mdelay(1); if ((num_sg) >= CONT_DESC_MAX) { wrReg32(pBrd, F2H_CONT_REMAIN, CONT_DESC_64); } else { wrReg32(pBrd, F2H_CONT_REMAIN, num_sg); } } } if(dma_debug) { printk("===descriptor[%d] cur_desc %px desc_phy_addr 0x%lx ===\n", i, desc_entry, desc_phy_addr); read_val = rdReg32(pBrd, F2H_DESC_ADDR_HIGH); printk("F2H_DESC_ADDR_HIGH: 0x%x VAL: 0x%x \n",F2H_DESC_ADDR_HIGH, read_val); read_val = rdReg32(pBrd, F2H_DESC_ADDR_LOW); printk("F2H_DESC_ADDR_LOW: 0x%x VAL: 0x%x \n",F2H_DESC_ADDR_LOW, read_val); read_val = rdReg32(pBrd, F2H_CONT_REMAIN); printk("F2H_CONT_REMAIN: 0x%x VAL: 0x%x \n",F2H_CONT_REMAIN, read_val); printk("==========================================================================\n"); } return 0; } /** * Populate DMA H2F descriptors. * @param pBrd: Pointer to the PCIe board structure. * @param num_desc: Index of the descriptor to be populated. * * Populates the DMA H2F descriptors with the necessary control flags, * lengths, and addresses for a DMA transfer from the host. It retrieves the SG table * and the descriptor entry for the specified descriptor index. The function then iterates * over each scatterlist entry in the SG table, setting up the descriptor entries with: * - Descriptor control flags based on the number of SG entries. * - DMA length from the scatterlist. * - Physical addresses for the next descriptor, source, and destination. */ static void PcieDmaH2fDescFill(pcie_board_t *pBrd, uint16_t num_desc) { struct sg_table *table; struct scatterlist *sg; dma_h2f_desc_hw_t *desc_entry; unsigned int num_sg = 0; unsigned long long desc_phy_addr; unsigned long long destination_address; uint32_t addr_len_sum = 0; int i; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; num_sg = table->nents; unsigned long long destination_address_arr[num_sg]; desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[num_desc]; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[num_desc]; /** * Store an incremental sum of the DMA Length to be used as Descriptor Address * for H2F Transfer */ for_each_sg(table->sgl, sg, num_sg, i) { addr_len_sum = addr_len_sum + sg_dma_len(sg); destination_address_arr[i] = addr_len_sum; } for_each_sg(table->sgl, sg, num_sg, i) { if ((i == 0) && (num_desc == 0)) { destination_address = 0; } else if ((i > 0) && (num_desc == 0)) { destination_address = destination_address_arr[(i-1)]; } if ((i == 0) && (num_desc > 0)) { destination_address = num_desc * pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_size; } else if ((i > 0) && (num_desc > 0)) { destination_address = num_desc * pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_size + destination_address_arr[(i-1)]; } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[i].h2f_desc_ctrl = (CONT_DESC_64 << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } else { desc_entry[i].h2f_desc_ctrl = (num_sg << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[i].h2f_dma_len = sg_dma_len(sg); desc_entry[i].h2f_next_desc_addr_lo = (desc_phy_addr & 0xffffffff) + (32 * (i+1)); desc_entry[i].h2f_next_desc_addr_hi = desc_phy_addr >> 32; desc_entry[i].h2f_src_addr_lo = sg_dma_address(sg); desc_entry[i].h2f_src_addr_hi = sg_dma_address(sg) >> 32; desc_entry[i].h2f_dest_addr_lo = destination_address & 0xffffffff; desc_entry[i].h2f_dest_addr_hi = destination_address >> 32; if(dma_debug) { if(i < 50) { printk("==========Descriptor %d | Entry %d========================================\n", num_desc, i); printk("Entry[%d] Descriptor Control DW0: 0x%x\n", i, desc_entry[i].h2f_desc_ctrl); printk("Entry[%d] DMA Length DW1: 0x%x\n", i, desc_entry[i].h2f_dma_len); printk("Entry[%d] Next Descriptor Address Low DW2: 0x%x\n", i, desc_entry[i].h2f_next_desc_addr_lo); printk("Entry[%d] Next Descriptor Address High DW3: 0x%x\n", i, desc_entry[i].h2f_next_desc_addr_hi); printk("Entry[%d] Source Address Low DW4: 0x%x\n", i, desc_entry[i].h2f_src_addr_lo); printk("Entry[%d] Source Address High DW5: 0x%x\n", i, desc_entry[i].h2f_src_addr_hi); printk("Entry[%d] Destination Address Low DW6: 0x%x\n", i, desc_entry[i].h2f_dest_addr_lo); printk("Entry[%d] Destination Address High DW7: 0x%x\n", i, desc_entry[i].h2f_dest_addr_hi); //printk("=========================================================================\n"); } } } } /** * Configure the last entry of the DMA H2F Descriptor. * * Sets up the descriptor entries for a H2F DMA transfer . * It handles the configuration of the last descriptor entry based on the number of * scatter-gather entries, the maximum number of descriptors, and the descriptor mode. * * @param pBrd: Pointer to the PCIe board structure. * @param num_desc: The index of the current descriptor being processed. * @param max_desc: The total number of descriptors available. * * Processes the scatter-gather table for the current descriptor and * calculates the remaining continuous descriptors. It then sets the control bits * for the last descriptor entry, including enabling interrupts and marking the end * of the packet (EOP) if necessary. If the descriptor mode is circular, it links * the last descriptor back to the first. */ static void PcieDmaH2fLastEntry(pcie_board_t *pBrd, uint16_t num_desc, uint16_t max_desc) { struct sg_table *table; struct sg_table *next_table; dma_h2f_desc_hw_t *desc_entry; unsigned int num_sg = 0; unsigned int num_sg_next = 0; unsigned long long desc_phy_addr; int next_desc; int num_sg_max; int cont_desc_rem = 0; next_desc = num_desc + 1; table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; if (num_desc < (max_desc -1)) { next_table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[next_desc]; } else { next_table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[num_desc]; } num_sg = table->nents; num_sg_next = next_table->nents; desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[num_desc]; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[num_desc]; num_sg_max = num_sg - 1; if ((num_sg) >= CONT_DESC_MAX) { cont_desc_rem = (num_sg) % CONT_DESC_MAX; } else if (num_desc < (max_desc - 1)) { cont_desc_rem = num_sg_next; } else if (num_desc == (max_desc -1)) { cont_desc_rem = num_sg; } if ((cont_desc_rem == 0) && (max_desc >= 1) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 0 is true\n"); printk("No Contiguous Descriptor remain, \nThere are 1 or more Descriptors, \nThis Descriptor is the Last Descriptor\n"); } desc_entry[num_sg_max].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else if ((cont_desc_rem == 0) && (max_desc > 1) && (num_desc < (max_desc -1))) { if (dma_debug) { printk("Condition 1 is true\n"); printk("No Contiguous Descriptor remain, \nThere are more than 1 Descriptors, \nThis Descriptor is NOT the Last Descriptor\n"); } desc_entry[num_sg_max].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].h2f_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[next_desc]) & 0xffffffff; desc_entry[num_sg_max].h2f_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[next_desc]) >> 32; } else if ((cont_desc_rem > 0) && (max_desc >= 1) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 2 is true\n"); printk("There are Contiguous Descriptor remaining, \nThere are 1 or more Descriptors, \nThis Descriptor is the Last Descriptor\n"); } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max - cont_desc_rem].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[num_sg_max].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | EOP; } else if ((cont_desc_rem > 0) && (max_desc > 1) && (num_desc < (max_desc -1))) { if (dma_debug) { printk("Condition 3 is true\n"); printk("There are Contiguous Descriptor remaining, \nThere are more than 1 Descriptors, \nThis Descriptor is NOT the Last Descriptor\n"); } if ((num_sg) >= CONT_DESC_MAX) { desc_entry[num_sg_max - cont_desc_rem].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; } desc_entry[num_sg_max].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_NOT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].h2f_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[next_desc]) & 0xffffffff; desc_entry[num_sg_max].h2f_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[next_desc]) >> 32; } if ((pBrd->DMA.chan[DMA_CHANNEL_NUM].desc_mode == DESC_MODE_CIRCULAR) && (num_desc == (max_desc -1))) { if (dma_debug) { printk("Condition 4 is true\n"); printk("This is a Circular Descriptor, \nThis Descriptor is the Last Descriptor\n"); } desc_entry[num_sg_max].h2f_desc_ctrl = (cont_desc_rem << CONT_DESC_SHIFT) | (INT_ENABLE << INT_SHIFT) | NOT_EOP; desc_entry[num_sg_max].h2f_next_desc_addr_lo = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[0]) & 0xffffffff; desc_entry[num_sg_max].h2f_next_desc_addr_hi = (pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[0]) >> 32; } if(dma_debug) { printk("==========Override Last Entry/Descriptor==================================\n"); printk("Entry[%d] Descriptor Control DW0: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_desc_ctrl); printk("Entry[%d] DMA Length DW1: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_dma_len); printk("Entry[%d] Next Descriptor Address Low DW2: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_next_desc_addr_lo); printk("Entry[%d] Next Descriptor Address High DW3: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_next_desc_addr_hi); printk("Entry[%d] Source Address Low DW4: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_src_addr_lo); printk("Entry[%d] Source Address High DW5: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_src_addr_hi); printk("Entry[%d] Destination Address Low DW6: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_dest_addr_lo); printk("Entry[%d] Destination Address High DW7: 0x%x\n", num_sg_max, desc_entry[num_sg_max].h2f_dest_addr_hi); printk("==========================================================================\n"); } } /** * Generate descriptors for PCIe DMA H2F. * * Initializes and allocates memory for DMA descriptors based on the * scatter-gather (SG) tables prepared for user-space buffers. It handles the allocation * of coherent memory for descriptors, fills the SG elements information, and sets up * the DMA hardware registers with the descriptor addresses. * * @param pBrd: Pointer to the PCIe board structure. * * Iterates over the number of descriptors specified in the DMA channel * descriptor count. For each descriptor, it allocates memory and fills * the descriptor with SG elements info and configures the last entry. * Finally, it writes the descriptor addresses to the * DMA hardware registers and enables the interrupt mode. * @return 0 on success, negative error code on failure */ static int PcieDmaH2fDescGen(pcie_board_t *pBrd) { int i; uint32_t read_val = 0; struct sg_table *table; unsigned int num_sg, sum_sg = 0; dma_h2f_desc_hw_t *desc_entry; unsigned long desc_phy_addr; size_t desc_size; int desc_count; int cont_desc_count; int cont_desc_rem; if(dma_debug) { printk("Starting Descriptor Generation: \n"); printk("========== H2F Process Starts Here ==========\n"); } pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_total_desc_entry = 0; desc_count = pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt; for (i = 0; i < desc_count; i++) { table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; desc_size = (table->nents) * sizeof(dma_h2f_desc_hw_t); pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_total_desc_entry = pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_total_desc_entry + (table->nents); if (dma_debug) { printk("Number of Entries: %d", table->nents); printk("Desc Size: %zu \n", desc_size); } pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i] = dma_alloc_coherent(&pBrd->pdev->dev, desc_size, &pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[i], GFP_KERNEL | __GFP_ZERO); if (!pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i]) { PcieDmaH2fDescFree(pBrd); return -ENOMEM; } if(dma_debug) printk("Desc[%d]: Desc Virt Addr %px Desc Phy Addr: %llx, Desc Size = %zu\n", i, pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i], pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[i], desc_size); } printk("Total Number of H2F Descriptor Entries:%d\n",pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_total_desc_entry); for (i = 0; i < desc_count; i++) { desc_size = (table->nents) * sizeof(dma_h2f_desc_hw_t); table = pBrd->DMA.chan[DMA_CHANNEL_NUM].ubuf_sg_table[i]; desc_entry =pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc[i]; desc_phy_addr = pBrd->DMA.chan[DMA_CHANNEL_NUM].dma_h2f_desc_handle[i]; num_sg = table->nents; if (i == 0) { pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_num_sg_0 = num_sg; } sum_sg += num_sg; if(dma_debug) { printk("=== DESCRIPTOR[%d] has %d Number of Entries (nents)===\n", i, num_sg); } cont_desc_count = (num_sg)/CONT_DESC_MAX; cont_desc_rem = (num_sg)%CONT_DESC_MAX; PcieDmaH2fDescFill(pBrd, i); PcieDmaH2fLastEntry(pBrd, i, desc_count); /** * Write the Very First Descriptor Entry Address in to the DMA HW Registers * */ if (i == 0) { wrReg32(pBrd, H2F_DESC_ADDR_HIGH, (uint32_t) (desc_phy_addr >> 32)); mdelay(1); wrReg32(pBrd, H2F_DESC_ADDR_LOW, (uint32_t)(desc_phy_addr)); mdelay(1); if ((num_sg) >= CONT_DESC_MAX) { wrReg32(pBrd, H2F_CONT_REMAIN, CONT_DESC_64); } else { wrReg32(pBrd, H2F_CONT_REMAIN, num_sg); } } } if(dma_debug) { printk("===buf[%d] cur_desc %px desc_phy_addr 0x%lx ===\n", i, desc_entry, desc_phy_addr); read_val = rdReg32(pBrd, H2F_DESC_ADDR_HIGH); printk("H2F_DESC_ADDR_HIGH: 0x%x VAL: 0x%x \n",H2F_DESC_ADDR_HIGH, read_val); read_val = rdReg32(pBrd, H2F_DESC_ADDR_LOW); printk("H2F_DESC_ADDR_LOW: 0x%x VAL: 0x%x \n",H2F_DESC_ADDR_LOW,read_val); read_val = rdReg32(pBrd, H2F_CONT_REMAIN); printk("H2F_CONT_REMAIN: 0x%x VAL: 0x%x \n",H2F_CONT_REMAIN,read_val); printk("==========================================================================\n"); } return 0; } /** * Reads the H2F completed descriptor count. * * This function reads the H2F completed descriptor count from the specified * PCIe board. * * @param pBrd Pointer to the PCIe board structure. * @return The H2F completed descriptor count. */ static int PcieDmaH2fDescCount(pcie_board_t *pBrd) { uint32_t read_val; read_val = rdReg32(pBrd, H2F_CPLT_DESC_COUNT); return read_val; } /** * Reads the F2H completed descriptor count. * * This function reads the F2H completed descriptor count from the specified * PCIe board. * * @param pBrd Pointer to the PCIe board structure. * @return The F2H completed descriptor count. */ static int PcieDmaF2hDescCount(pcie_board_t *pBrd) { uint32_t read_val; read_val = rdReg32(pBrd, F2H_CPLT_DESC_COUNT); return read_val; } /** * initBoard - Initialize a PCIe board structure. * * @param PCI_Dev_Cfg: Pointer to the PCI device configuration structure. * @param devID: Pointer to the device ID. * This is called when probe() has found a matching PCI device (via the PCI subsystem * probing for boards on behalf of the driver) * This function initializes a PCIe board structure. It populates the board information, * sets up memory mappings, and retrieves PCIe link characteristics. If successful, it * returns a pointer to the initialized board structure; otherwise, it returns NULL. * * @return Pointer to the initialized pcie_board_t structure, or NULL on failure. */ static pcie_board_t* initBoard(struct pci_dev *PCI_Dev_Cfg, void *devID) { int i, msi_cnt, ret; uint8_t irq; pcie_board_t *pBrd; pci_dev_bar_t *pBAR, *p; uint16_t SubSystem; uint16_t VendorID; uint16_t DeviceID; uint16_t CommandReg; if (dma_debug) pr_info("enter initBoard \n"); if (global_pciedma.numBoards >= NUM_BOARDS) { pr_err("Too many boards! Increase NUM_BOARDS!\n"); return(NULL); } pBrd = &global_pciedma.Board[global_pciedma.numBoards]; memset(pBrd, 0, sizeof(pcie_board_t)); if (pci_read_config_word(PCI_Dev_Cfg, PCI_VENDOR_ID, &VendorID)) { pr_err("Board CFG access failed!\n"); return(NULL); } if (VendorID != 0x1204) { pr_err("Vendor ID (0x%x) of the board not Lattice!\n",VendorID); return(NULL); } if (pci_read_config_word(PCI_Dev_Cfg, PCI_DEVICE_ID, &DeviceID)) { pr_err("Board CFG access failed!\n"); return(NULL); } if (pci_read_config_word(PCI_Dev_Cfg, PCI_SUBSYSTEM_ID, &SubSystem)) { pr_err("Board CFG access failed!\n"); return(NULL); } pr_info("DeviceID= 0x%x, SubSystemID= 0x%x \n",DeviceID, SubSystem); if (pci_read_config_word(PCI_Dev_Cfg, PCI_CMD_REG, &CommandReg)) { pr_err("Board CFG access failed!\n"); return(NULL); } if ((CommandReg & 0x6) == 0) { pr_info("Bus Master and Memory Space Enable not set. Setting now"); pci_write_config_word(PCI_Dev_Cfg, PCI_CMD_REG, BME_MSE_ENABLE); } pBrd->ID = DeviceID; pBrd->demoID = SubSystem; pBrd->pdev = PCI_Dev_Cfg; pBrd->majorNum = MAJOR(global_pciedma.drvrDevNum); pBrd->minorNum = MINOR(global_pciedma.drvrDevNum) + global_pciedma.numBoards; atomic_set(&(pBrd->OpenToken), 1); // initialize "open" token to available spin_lock_init(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); // initialize spin-lock for register access protection pBrd->msi = false; ++global_pciedma.numPCIE_DMA; pBrd->instanceNum = global_pciedma.numPCIE_DMA; pBrd->boardType = LSC_BOARD; pBrd->demoType = DMA_DEMO; pBrd->ctrlBAR = 0; pBrd->start_f2h_transfer = PcieDmaF2hStartTransfer; pBrd->start_h2f_transfer = PcieDmaH2fStartTransfer; // BAR will be mmap'ed into user space for the driver interface to access. pBrd->mmapBAR = pBrd->ctrlBAR; // Legacy Interrupt if (pci_read_config_byte(PCI_Dev_Cfg, PCI_INTERRUPT_LINE, &irq)) pBrd->IRQ = -1; // no interrupt else pBrd->IRQ = irq; if (dma_debug) { pr_info("brdID: %x demoID: %x \n", DeviceID, SubSystem); pr_info("Board[] =%d, Legacy IRQ=%d\n", global_pciedma.numBoards, pBrd->IRQ); } //Interrupt handling msi_cnt = pci_alloc_irq_vectors(pBrd->pdev, 1, 32, PCI_IRQ_MSI); if(msi_cnt < 0) { pr_err("Failed to get MSI interrupts."); pBrd->DMA.chan[DMA_CHANNEL_NUM].num_irqs = 0; return NULL; } pr_info("msi_cnt = %d\n", msi_cnt); pBrd->DMA.chan[DMA_CHANNEL_NUM].num_irqs = msi_cnt; for (i = 0; i < pBrd->DMA.chan[DMA_CHANNEL_NUM].num_irqs; i++){ pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i] = pci_irq_vector(pBrd->pdev, i); //pr_info("Attach MSI interrupt: %d \n", pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i]); } if (dma_debug) { pr_info("brdID: %x demoID: %x\n", DeviceID, SubSystem); pr_info("Board[] =%d\n", global_pciedma.numBoards); } #ifdef MSI if (dma_debug) pr_info("Attach MSI interrupt\n"); for ( i = 0; i < MAX_NUM_MSI_VEC; i++) { ret = request_irq(pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i], // the IRQ assigned to us PcieDmaIrqHandler, // the ISR routine to invoke 0, // flags - none needed for MSI ch0_msi_name[i], // a name to show in /proc/interrupts pBrd); // arg to pass to our ISR, must pass exact same value to free_irq() if (ret) { pr_err("can't request MSI IRQ %d, name: %s\n", pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i], ch0_msi_name[i]); } else { pBrd->msi = true; if (dma_debug) pr_info("MSI IRQ=%d name=%s\n", pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i], ch0_msi_name[i]); } } #else // Use INTx legacy interrupts if (dma_debug) pr_info("Attach INTx interrupt\n"); ret = request_irq(pBrd->pdev->irq, PcieDmaIrqHandler, IRQF_SHARED, "pciedmaINTx", pBrd); if (ret) { if (dma_debug) pr_err("can't get INTx IRQ %d\n", pBrd->IRQ); pBrd->IRQ = -1; } else { pBrd->IRQ = pBrd->pdev->irq; if (dma_debug) pr_info("INTx IRQ=%d\n", pBrd->IRQ); } #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0) //It return zero if card can perofrm DMA properly on the machine given the address mask if (sizeof(dma_addr_t) == 8 && !pci_set_dma_mask(PCI_Dev_Cfg, DMA_BIT_MASK(64))) { pr_info("set DMA_BIT_MASK(64) ok \n"); pBrd->DMA.chan[DMA_CHANNEL_NUM].isDmaAddr64 = true; } else if(!pci_set_dma_mask(PCI_Dev_Cfg, DMA_BIT_MASK(32))) { pr_info("set DMA_BIT_MASK(32) ok \n"); } else { pr_err("DMA not supported on this platform!\n"); //pci_disable_pcie_error_reporting(PCI_Dev_Cfg); return NULL; } #else if (sizeof(dma_addr_t) == 8) { if (dma_set_mask_and_coherent(&PCI_Dev_Cfg->dev, DMA_BIT_MASK(64))) { pr_err("64-bit DMA not supported on this platform!\n"); //pci_disable_pcie_error_reporting(PCI_Dev_Cfg); return NULL; } pr_info("Set DMA_BIT_MASK(64) ok\n"); pBrd->DMA.chan[DMA_CHANNEL_NUM].isDmaAddr64 = true; } else { if (dma_set_mask_and_coherent(&PCI_Dev_Cfg->dev, DMA_BIT_MASK(32))) { pr_err("32-bit DMA not supported on this platform!\n"); //pci_disable_pcie_error_reporting(PCI_Dev_Cfg); return NULL; } pr_info("Set DMA_BIT_MASK(32) ok\n"); } #endif /* Get info on all the PCI BAR registers */ pBrd->numBars = 0; // initialize for (i = 0; i < NUM_BARS; i++) { p = &(pBrd->Dev_BARs[i]); p->pci_start = pci_resource_start(PCI_Dev_Cfg, i); p->pci_end = pci_resource_end(PCI_Dev_Cfg, i); p->len = pci_resource_len(PCI_Dev_Cfg, i); p->pci_flags = pci_resource_flags(PCI_Dev_Cfg, i); if ((p->pci_start > 0) && (p->pci_end > 0)) { ++(pBrd->numBars); p->bar = i; p->pci_addr = (void *)p->pci_start; p->memType = p->pci_flags; if (dma_debug) { pr_info("BAR=%d\n", i); pr_info("start=%lx\n", p->pci_start); pr_info("end=%lx\n", p->pci_end); pr_info("len=0x%lx\n", p->len); pr_info("flags=0x%lx\n", p->pci_flags); } } } // Map the BAR into kernel space so the driver can access registers. pBAR = &(pBrd->Dev_BARs[pBrd->ctrlBAR]); if (pBAR->pci_start) { pBrd->ctrlBARaddr = ioremap(pBAR->pci_start, pBAR->len); // pci_start:PCI bus start address, len:BAR size pBAR->kvm_addr = pBrd->ctrlBARaddr; if (pBrd->ctrlBARaddr) { pr_info("map the bar into kernel space success with ioremap\n"); } else { pr_err("error with ioremap\n"); return(NULL); } } else { pr_err("error ctrlBAR %d not avail!\n", pBrd->ctrlBAR); return(NULL); } // Get information about the PCIe link characteristics ParsePCIeLinkCap(pBrd); // Initialize the burst sizes for the DMA channels on this board pBrd->DMA.WriteBurstSize = pBrd->PCIeMaxPayloadSize; pBrd->DMA.ReadBurstSize = pBrd->PCIeMaxReadReqSize; // DMA initialize here pBrd->DMA.DmaOk = true; ++global_pciedma.numBoards; printk("InitBoard: Done! \n"); return pBrd; } /** * Open the PCIe DMA device and provide direct access to board resources. * * This function is responsible for opening the PCIe DMA device. It extracts the board * number and function number from the minor number, checks if the board is configured * correctly, and manages the open token to prevent multiple simultaneous accesses. * The private_data field of the file structure is set to point to the board structure * for subsequent operations. Additionally, it initializes the wait queue for DMA read * operations. * * @param inode: Pointer to the inode structure. * @param filp: Pointer to the file structure. * @return 0 on success, negative error code on failure */ static int PcieDmaOpen(struct inode *inode, struct file *filp) { uint32_t brdNum; uint32_t funcNum; pcie_board_t *pBrd; /* Extract the board number from the minor number */ brdNum = DMA_MINOR_TO_BOARD(iminor(inode)); funcNum = DMA_MINOR_TO_FUNCTION(iminor(inode)); if (dma_debug) pr_info("board#=%d func#=%d\n", brdNum, funcNum); if (brdNum >= global_pciedma.numBoards) { pr_err("brd# %d No such board!\n", brdNum); return(-ENODEV); } /* This is what the user wants to access */ pBrd = &global_pciedma.Board[brdNum]; if (pBrd->ID == 0) { pr_err("brd# %d not configured correctly!\n", brdNum); return(-ENODEV); // Board[] entry not configured correctly } if (!atomic_dec_and_test(&pBrd->OpenToken)) { pr_err("brd# %d already open!\n", brdNum); atomic_inc(&pBrd->OpenToken); // restore to prev value return(-EBUSY); } /* Provide direct access to the board's resources in future system calls */ filp->private_data = pBrd; pBrd->function = funcNum; init_waitqueue_head(&pBrd->DMA.ReadWaitQ); return 0; } /** * Release resources associated with a PCIe DMA board. * * @param inode: Pointer to the inode structure. * @param filp: Pointer to the file structure. * * This function is called when a file associated with a PCIe DMA board is closed. * It releases allocated resources, including descriptor memory, user buffer mappings, * and the open token. The board becomes available for other processes to open. * * @return 0 on success. */ static int PcieDmaRelease(struct inode *inode, struct file *filp) { struct PCIE_Board *pBrd = filp->private_data; uint32_t mnr = iminor(inode); if (dma_debug) pr_info("closing board=%d function=%x\n", DMA_MINOR_TO_BOARD(mnr), DMA_MINOR_TO_FUNCTION(mnr)); PcieDmaF2hDescFree(pBrd); PcieDmaH2fDescFree(pBrd); PcieDmaUbufUnmap(pBrd); pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt = 0; pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt = 0; // Give back the token and let someone else open the board atomic_inc(&(pBrd->OpenToken)); return 0; } /** * Custom IOCTL handler for PCIe DMA operations * * @param filp: Pointer to the file structure * @param cmd: IOCTL command * @param arg: Argument passed from user space * * This function handles various IOCTL commands related to PCIe DMA. * It interacts with the pcie_board_t structure and performs operations * such as getting version information, retrieving resources, reading/writing * data, and more. * * @return 0 on success, negative error code on failure */ static long PcieDmaIoctl(struct file *filp, unsigned int cmd, unsigned long arg) { //pcie_board_t *pBrd = NULL; pcie_board_t *pBrd; dma_f2h_ubuf_ioctl_t f2h_data; dma_h2f_ubuf_ioctl_t h2f_data; dma_rw_ioctl_t rw; void *uaddr[MAX_NUM_DESCRIPTORS]; //void *uaddr[pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt]; int i, ret, timeout; int cur_dma_sts = -1; uint32_t f2h_total_entry_ret = 0; uint32_t h2f_total_entry_ret = 0; PCIResourceInfo_t *pInfo; pBrd = filp->private_data; switch (cmd) { case IOCTL_PCIEDMA_GET_VERSION_INFO: //strncpy((void *)arg, version, MAX_DRIVER_VERSION_LEN - 1); if (copy_to_user((void *)arg, (void *)version, sizeof(version)) != 0) return -EFAULT; break; case IOCTL_PCIEDMA_GET_RESOURCES:; pInfo = kmalloc(sizeof(PCIResourceInfo_t), GFP_KERNEL); if (pInfo == NULL) { return -EFAULT; } if (pBrd->IRQ > 0) pInfo->hasInterrupt = true; else pInfo->hasInterrupt = false; pInfo->intrVector = pBrd->IRQ; pInfo->numBARs = pBrd->numBars; for (i = 0; i < MAX_PCI_BARS; i++) { pInfo->BAR[i].nBAR = pBrd->Dev_BARs[i].bar; pInfo->BAR[i].physStartAddr = (unsigned long)pBrd->Dev_BARs[i].pci_addr; pInfo->BAR[i].size = pBrd->Dev_BARs[i].len; pInfo->BAR[i].memMapped = (pBrd->Dev_BARs[i].kvm_addr) ? 1 : 0; pInfo->BAR[i].flags = (unsigned short)(pBrd->Dev_BARs[i].pci_flags); pInfo->BAR[i].type = (uint8_t)((pBrd->Dev_BARs[i].memType)>>8); } for (i = 0; i < 0x100; ++i) pci_read_config_byte(pBrd->pdev, i, &(pInfo->PCICfgReg[i])); if (copy_to_user((void *)arg, (void *)pInfo, sizeof(PCIResourceInfo_t)) != 0){ return -EFAULT; } kfree(pInfo); break; case IOCTL_PCIEDMA_READ_8BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) return -EACCES; rw.value = rdReg8(pBrd, rw.reg); if (copy_to_user((void*)arg, &rw, sizeof(dma_rw_ioctl_t))) { pr_err("copy_to_user failed.\n"); return -EACCES; } break; case IOCTL_PCIEDMA_READ_16BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) return -EACCES; rw.value = rdReg16(pBrd, rw.reg); if (copy_to_user((void*)arg, &rw, sizeof(dma_rw_ioctl_t))) { pr_err("copy_to_user failed.\n"); return -EACCES; } break; case IOCTL_PCIEDMA_READ_32BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) return -EACCES; //pr_info("reg:0x%x \n", rw.reg); rw.value = rdReg32(pBrd, rw.reg); if (copy_to_user((void*)arg, &rw, sizeof(dma_rw_ioctl_t))) { pr_err("copy_to_user failed.\n"); return -EACCES; } break; case IOCTL_PCIEDMA_WRITE_8BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) { pr_err("w8 copy_to_user failed.\n"); return -EACCES; } //pr_info("reg:0x%x val8: 0x%x\n", rw.reg, rw.value); wrReg8(pBrd, rw.reg, rw.value); break; case IOCTL_PCIEDMA_WRITE_16BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) { pr_err("w16 copy_to_user failed.\n"); return -EACCES; } //pr_info("reg:0x%x val16: 0x%x\n", rw.reg, rw.value); wrReg16(pBrd, rw.reg, rw.value); break; case IOCTL_PCIEDMA_WRITE_32BIT: if (copy_from_user(&rw, (dma_rw_ioctl_t *)arg, sizeof(dma_rw_ioctl_t))) { pr_err("w32 copy_to_user failed.\n"); return -EACCES; } //pr_info("reg:0x%x val32: 0x%x\n", rw.reg, rw.value); wrReg32(pBrd, rw.reg, rw.value); break; case IOCTL_PCIEDMA_F2H_DESC_GEN: pr_info("Start F2H IOCTL: \n"); if (pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt > 0){ return -EALREADY; } /* already set */ if (copy_from_user(&f2h_data, (void __user *)arg, _IOC_SIZE(cmd))){ return -EFAULT; } if (copy_from_user(uaddr, (void __user *)f2h_data.addr, f2h_data.desc_cnt * sizeof(void*))){ return -EFAULT; } if (PcieDmaUbufMap(pBrd, uaddr, f2h_data.len, f2h_data.desc_cnt)){ return -ENOMEM; } pr_info("desc_cnt:%d \n",f2h_data.desc_cnt ); pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt = f2h_data.desc_cnt; pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt = f2h_data.desc_cnt; pBrd->DMA.chan[DMA_CHANNEL_NUM].desc_mode = f2h_data.descriptor_mode; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_size = f2h_data.len; if (PcieDmaF2hDescGen(pBrd)) { PcieDmaUbufUnmap(pBrd); return -ENOMEM; } break; case IOCTL_PCIEDMA_H2F_DESC_GEN: pr_info("Start H2F IOCTL: \n"); if (pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt > 0){ return -EALREADY; } /* already set */ if (copy_from_user(&h2f_data, (void __user *)arg, _IOC_SIZE(cmd))){ return -EFAULT; } if (copy_from_user(uaddr, (void __user *)h2f_data.addr, h2f_data.desc_cnt * sizeof(void*))){ return -EFAULT; } if (PcieDmaUbufMap(pBrd, uaddr, h2f_data.len, h2f_data.desc_cnt)){ return -ENOMEM; } pr_info("desc_cnt:%d \n",h2f_data.desc_cnt ); pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt = h2f_data.desc_cnt; pBrd->DMA.chan[DMA_CHANNEL_NUM].max_desc_cnt = h2f_data.desc_cnt; pBrd->DMA.chan[DMA_CHANNEL_NUM].desc_mode = h2f_data.descriptor_mode; pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_size = h2f_data.len; if (PcieDmaH2fDescGen(pBrd)) { PcieDmaUbufUnmap(pBrd); return -ENOMEM; } break; case IOCTL_PCIEDMA_F2H_START: pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done = DMA_NOT_DONE; PcieDmaF2hStartTransfer(pBrd); break; case IOCTL_PCIEDMA_H2F_START: pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done = DMA_NOT_DONE; PcieDmaH2fStartTransfer(pBrd); break; case IOCTL_PCIEDMA_F2H_CHECK_DONE: //printk("Checking F2H DMA Done\n"); if (pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt == 0) /* not init */ return -ENOBUFS; if (copy_from_user(&timeout, (void __user *)arg, _IOC_SIZE(cmd))) return -EFAULT; spin_lock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); ret = wait_event_lock_timeout(pBrd->DMA.ReadWaitQ, pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done, pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess, msecs_to_jiffies(timeout)); if (0 == ret) { spin_unlock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); return -ETIME; } cur_dma_sts = pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done; spin_unlock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); if (copy_to_user((void __user *)arg, &cur_dma_sts, _IOC_SIZE(cmd))) return -EFAULT; pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_dma_done = DMA_NOT_DONE; //clear DMA DONE indicator after returning value to user break; /*We create 2 DMA Check Done IOCTLs for now in case future enhancements lead to creation of multiple MSI for each direction of transfer*/ case IOCTL_PCIEDMA_H2F_CHECK_DONE: //printk("Checking H2F DMA Done\n"); if (pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt == 0) /* not init */ return -ENOBUFS; if (copy_from_user(&timeout, (void __user *)arg, _IOC_SIZE(cmd))) return -EFAULT; spin_lock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); ret = wait_event_lock_timeout(pBrd->DMA.ReadWaitQ, pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done, pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess, msecs_to_jiffies(timeout)); if (0 == ret) { spin_unlock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); return -ETIME; } cur_dma_sts = pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done; spin_unlock_irq(&pBrd->DMA.chan[DMA_CHANNEL_NUM].hdwAccess); if (copy_to_user((void __user *)arg, &cur_dma_sts, _IOC_SIZE(cmd))) return -EFAULT; pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_dma_done = DMA_NOT_DONE; //clear DMA DONE indicator after returning value to user break; case IOCTL_PCIEDMA_F2H_STOP: if (pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt == 0) /* not init */ { return -ENOBUFS; } PcieDmaF2hStopTransfer(pBrd); PcieDmaF2hDescFree(pBrd); PcieDmaUbufUnmap(pBrd); pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_desc_cnt = 0; break; case IOCTL_PCIEDMA_H2F_STOP: if (pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt == 0) /* not init */ { return -ENOBUFS; } PcieDmaH2fStopTransfer(pBrd); PcieDmaH2fDescFree(pBrd); PcieDmaUbufUnmap(pBrd); pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_desc_cnt = 0; break; case IOCTL_PCIEDMA_CFG_REG_READ: if(copy_from_user(&rw, (dma_rw_ioctl_t*)arg, sizeof(dma_rw_ioctl_t))){ printk(KERN_INFO"pcie_dma_demo: unable to access user buffer \n"); return -EACCES; } rw.value = rdRegPciCfg(pBrd, rw.reg); if(copy_to_user((void*)arg, &rw, sizeof(dma_rw_ioctl_t))){ printk(KERN_INFO"pcie_dma_demo: unable to update buffer to user \n"); return -EACCES; } break; case IOCTL_PCIEDMA_CFG_REG_WRITE: if(copy_from_user(&rw, (dma_rw_ioctl_t*)arg, sizeof(dma_rw_ioctl_t))){ printk(KERN_INFO"pcie_dma_demo: unable to access user buffer \n"); return -EACCES; } wrRegPciCfg(pBrd, rw.reg, rw.value); break; case IOCTL_PCIEDMA_TOTAL_F2H_ENTRIES_CHECK: f2h_total_entry_ret = pBrd->DMA.chan[DMA_CHANNEL_NUM].f2h_total_desc_entry; printk("f2h_total_entry_ret: %d",f2h_total_entry_ret); if (copy_to_user((void*)arg, &f2h_total_entry_ret, sizeof(f2h_total_entry_ret))) return -EFAULT; break; case IOCTL_PCIEDMA_TOTAL_H2H_ENTRIES_CHECK: h2f_total_entry_ret = pBrd->DMA.chan[DMA_CHANNEL_NUM].h2f_total_desc_entry; printk("h2f_total_entry_ret: %d",h2f_total_entry_ret); //printk("h2f_total_entry_ret: %d",&h2f_total_entry_ret); if (copy_to_user((void*)arg, &h2f_total_entry_ret, sizeof(h2f_total_entry_ret))) { printk("is this happening? \n"); return -EFAULT; } break; default: return -EINVAL; } return 0; } /** * mmap. * This is the most important driver method. This maps the device's PCI * address space (based on the select mmap BAR number) into the user's * address space, allowing direct memory access with standard pointers. */ static int PcieDmaMmap(struct file *filp, struct vm_area_struct *vma) { return 0; } /** * read. * Read from system CommonBuffer DMA memory into users buffer. * User passes length (in bytes) like reading from a file. */ static ssize_t PcieDmaRead(struct file *filp, char __user *userBuf, size_t len, loff_t *offp) { return 0; } /** * write. * Write from users buffer into system CommonBuffer DMA memory. * User passes length (in bytes) like writing to a file. */ static ssize_t PcieDmaWrite(struct file *filp, const char __user *userBuf, size_t len, loff_t *offp) { return 0; } /** * @brief File operations structure for PCIe. * * This structure defines the file operations for the PCIe device, including * open, release, ioctl, mmap, read, and write operations. */ struct file_operations PCIeFileOpt = { owner: THIS_MODULE, open: PcieDmaOpen, release: PcieDmaRelease, unlocked_ioctl: PcieDmaIoctl, mmap: PcieDmaMmap, read: PcieDmaRead, write: PcieDmaWrite, }; #ifdef USE_PROC /** * @brief Show function for the PCIe DMA proc file. * * This function is called when the proc file is read. * * @param m Pointer to the seq_file structure. * @param v Pointer to the data. * @return Always returns 0. */ static int PcieDmaProcShow(struct seq_file *m, void *v) { return 0; } /** * @brief Open function for the PCIe DMA proc file. * * This function is called when the proc file is opened. * * @param inode Pointer to the inode structure. * @param pfile Pointer to the file structure. * @return Result of the single_open function. */ static int PcieDmaProcOpen(struct inode *inode, struct file *pfile) { pr_info("enter:"); return single_open(pfile, PcieDmaProcShow, NULL); } /** * @brief Write function for the PCIe DMA proc file. * * This function is called when data is written to the proc file. * * @param file Pointer to the file structure. * @param buffer Pointer to the user buffer. * @param count Number of bytes to write. * @param ppos Pointer to the position offset. * @return Number of bytes written, or an error code. */ static ssize_t PcieDmaProcWrite(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { //har buf[32] = {0}; char *buf = NULL; pcie_board_t *pBrd; pr_info("count:%zu\n",count); if(count > 31) count = 32; buf = memdup_user_nul(buffer, count); if (IS_ERR(buf)) return PTR_ERR(buf); //pr_info("input cmd:%c%c%c", buf[0],buf[1],buf[2]); if (buf[count - 1] == '\n') buf[count - 1] = '\0'; pBrd = &global_pciedma.Board[0]; kfree(buf); return count; } /** * @brief Proc operations structure for PCIe DMA. * * This structure defines the proc file operations for the PCIe DMA device, * including open, read, lseek, release, and write operations. */ static const struct proc_ops PcieDmaProcOps = { .proc_open = PcieDmaProcOpen, .proc_read = seq_read, .proc_lseek = seq_lseek, .proc_release = single_release, .proc_write = PcieDmaProcWrite, }; #endif /** * PcieDmaProbe - Probe function for PCI Express DMA device * * @param pdev: Pointer to the PCI device structure * @param ent: Pointer to the PCI device ID structure * * This function is called during PCI device probing. It initializes the PCI * Express DMA device, requests regions, enables the device, and performs * board-specific initialization. A new device is added to the * /sys/class/lscvdma/ tree with the name created by the: * information. * @return 0 on success or an error code otherwise. */ static int PcieDmaProbe(struct pci_dev *pdev, const struct pci_device_id *ent) { static char devNameStr[12] = "pciedma__"; pcie_board_t *pBrd; int err; devNameStr[8] = '0'+ global_pciedma.numBoards; if (dma_debug) pr_info("pci probe for: %s pdev=%p ent=%p\n", devNameStr, pdev, ent); err = pci_request_regions(pdev, devNameStr); if (err) return err; pci_set_master(pdev); err = pci_enable_device(pdev); if (err) return err; /* * Call to perform board specific initialization and figure out * which BARs are active, interrupt vectors, register ISR, what board * it is, what demo (SGMDA) and what instance number (is it the 2nd time we've seen a SC DMA?) * Returns pointer to the Board structure after all info filled in. */ pBrd = initBoard(pdev, (void *)ent); if (pBrd == NULL) { pr_err("error initializing board\n"); // Clean up any resources we acquired along the way pci_release_regions(pdev); pci_disable_device(pdev); return(-1); } // Initialize the CharDev entry for this new found hw board device pBrd->cdev.owner = THIS_MODULE; kobject_set_name(&(pBrd->cdev.kobj), gDrvrName); cdev_init(&(pBrd->cdev), &PCIeFileOpt); if (cdev_add(&(pBrd->cdev), MKDEV(pBrd->majorNum,pBrd->minorNum), MINORS_PER_BOARD)) { pr_err("error adding char device\n"); kobject_put(&(pBrd->cdev.kobj)); return(-1); } device_create(global_pciedma.sysfs, NULL, MKDEV(pBrd->majorNum,pBrd->minorNum), &(pdev->dev), "%s_%s_%d", BoardName[pBrd->boardType], DemoName[pBrd->demoType], pBrd->instanceNum); pr_info("added device: %s_%s_%d\n", BoardName[pBrd->boardType], DemoName[pBrd->demoType], pBrd->instanceNum); pci_set_drvdata(pdev, pBrd); return 0; } /** * Cleanup function for PCI Express DMA device removal * * @param pdev: Pointer to the PCI device structure * * This function is called during PCI device removal. It performs cleanup tasks * such as freeing IRQs, unmapping memory regions, releasing device regions, * unbinding minor numbers, and removing the device entry from the sysfs tree. */ static void PcieDmaRemove(struct pci_dev *pdev) { int i; pcie_board_t *pBrd = pci_get_drvdata(pdev); if (dma_debug) pr_info("pci remove for device: pdev=%p board=%p\n", pdev, pBrd); // Disable PCIe error reporting //pci_disable_pcie_error_reporting(pdev); #ifdef MSI for (i = 0; i < 3; i++) { free_irq(pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i], pBrd); } for (i = 24; i <= 28; i++) { free_irq(pBrd->DMA.chan[DMA_CHANNEL_NUM].irq_vec[i], pBrd); } if (pBrd->msi) pci_free_irq_vectors(pBrd->pdev); pBrd->msi = false; pBrd->DMA.chan[DMA_CHANNEL_NUM].num_irqs = 0; #else if (pBrd->IRQ != -1) { if (dma_debug) pr_info(" close - free_irq %d\n", pBrd->IRQ); free_irq(pBrd->IRQ, pBrd); } pBrd->IRQ = -1; #endif // Free our internal access to the control BAR address space if (pBrd->ctrlBARaddr) iounmap(pBrd->ctrlBARaddr); // No more access after this call pci_release_regions(pdev); // Unbind the minor numbers of this device cdev_del(&(pBrd->cdev)); unregister_chrdev_region(MKDEV(pBrd->majorNum, pBrd->minorNum), MINORS_PER_BOARD); // Remove the device entry in the /sys/class/lscvdma/ tree device_destroy(global_pciedma.sysfs, MKDEV(pBrd->majorNum, pBrd->minorNum)); #ifdef USE_PROC remove_proc_entry("driver/pciedma", NULL); #endif return; } /** * Placeholder function for handling device suspension during system sleep. * * @param pdev: Pointer to the PCI device structure * @param state: Power management message indicating the sleep state (e.g., suspend, hibernate) * * This function is called when the system enters a sleep state. It logs a message indicating * that the device is being suspended. The return value of 0 indicates success (no error). */ static int PcieDmaSuspend(struct pci_dev *pdev, pm_message_t state) { pr_info("suspend\n"); return 0; } /** * Placeholder function for handling device resume after system sleep. * * @param pdev: Pointer to the PCI device structure * * This function is called when the system resumes from a sleep state. It logs a message * indicating that the device is being resumed. * @return value of 0 indicates success (no error). */ static int PcieDmaResume(struct pci_dev *pdev) { pr_info("resume \n"); return 0; } /** * Placeholder function for handling device shutdown. * * @param pdev: Pointer to the PCI device structure * * This function is called during system shutdown or device removal. It logs a message * indicating that the device is being shut down. */ static void PcieDmaShutdown(struct pci_dev *pdev) { pr_info("shutdown \n"); } /** * Error handling function for PCI Express DMA device. * * @param pdev: Pointer to the PCI device structure * @param state: PCI channel state indicating the error condition * * This function is called when an error is detected on the PCI channel. It handles * different error states and returns the appropriate result for error recovery. * * @return pci_ers_result_t: Result indicating the recommended action (e.g., recover, reset, disconnect) */ static pci_ers_result_t PcieDmaErrorDetected (struct pci_dev *pdev, pci_channel_state_t state) { pci_channel_state_t s; s = state; switch (s) { case pci_channel_io_normal: return PCI_ERS_RESULT_CAN_RECOVER; case pci_channel_io_frozen: pr_warn(" dev 0x%p, frozen state error, reset controller \n", pdev); pci_disable_device(pdev); return PCI_ERS_RESULT_NEED_RESET; case pci_channel_io_perm_failure: pr_warn("dev 0x%p, failure state error, request disconnect \n", pdev); return PCI_ERS_RESULT_DISCONNECT; } return PCI_ERS_RESULT_NEED_RESET; } /** * Slot reset handler for PCI Express DMA device. * * @param pdev: Pointer to the PCI device structure * * This function is called when a slot reset occurs for the PCI device. It attempts * to re-enable the device, restore its state, and save the state. * @return PCI_ERS_RESULT_RECOVERED; otherwise, it returns PCI_ERS_RESULT_DISCONNECT. */ static pci_ers_result_t PcieDmaSlotReset(struct pci_dev *pdev) { pr_info("0x%p restart after slot reset \n", pdev); if(pci_enable_device_mem(pdev)) { pr_err(" failed to reenable after slot reset \n"); return PCI_ERS_RESULT_DISCONNECT; } pci_set_master(pdev); pci_restore_state(pdev); pci_save_state(pdev); return PCI_ERS_RESULT_RECOVERED; } /** * PCI error handlers for PCI Express DMA device. * * This structure defines the error handling functions associated with the PCI device. * - `error_detected`: Called when an error is detected on the PCI channel. * - `slot_reset`: Called when a slot reset occurs for the PCI device. * - `resume`: Placeholder function for handling device resume after system sleep. * - `reset_prepare`: Placeholder function for preparing for a device reset. * - `reset_done`: Placeholder function for handling device reset completion. */ static const struct pci_error_handlers PcieDmaErrHandler = { .error_detected = PcieDmaErrorDetected, .slot_reset = PcieDmaSlotReset, //.resume, //.reset_prepare, //.reset_done, }; /** * Main structure required for registering a driver with the PCI core. * name must be unique across all registered PCI drivers, and shows up in * /sys/bus/pci/drivers/ * id_table points to the table of Vendor,Device,SubSystem matches * probe is the function to call when enumerating PCI buses to match driver to device * remove is the function called when PCI is shutting down and devices/drivers are * being removed. */ static struct pci_driver PcieDmaDriver = { .name = gDrvrName, .id_table = lattice_pci_id_tbl, .probe = PcieDmaProbe, .remove = PcieDmaRemove, .suspend = PcieDmaSuspend, .resume = PcieDmaResume, .shutdown = PcieDmaShutdown, .err_handler = &PcieDmaErrHandler, }; /** * Initialization function for the PCI Express DMA driver. * * This function performs the following tasks during driver initialization: * 1. Allocates a dynamic major number for the character device. * 2. Creates a sysfs class entry to hold the tree of detected PCIe board devices. * 3. Registers the PCI driver components and functions with the Kernel PCI core. * 4. Optionally creates a proc entry for additional information (if USE_PROC is defined). * * @return 0 on success or an error code otherwise. */ static int __init PcieDmaInit(void) { int n,i; int result; int err; pr_info("dma_debug=%d\n", dma_debug); pr_info("%s\n", version); /* Initialize the driver database to nothing found, no BARs, no devices */ memset(&global_pciedma, 0, sizeof(global_pciedma)); for (n = 0; n < NUM_BOARDS; n++) for (i = 0; i < NUM_BARS; i++) global_pciedma.Board[n].Dev_BARs[i].bar = -1; /* * Register device driver as a character device and get a dynamic Major number */ result = alloc_chrdev_region(&global_pciedma.drvrDevNum, // return allocated Device Num here 0, // first minor number MAX_MINORS, gDrvrName); if (result < 0) { pr_err("can't get major/minor numbers!\n"); return(result); } if (dma_debug) pr_info("Major=%d num boards=%d\n", MAJOR(global_pciedma.drvrDevNum), global_pciedma.numBoards); if (dma_debug) pr_info("cdev_init()\n"); /* Create the new sysfs Class entry that will hold the tree of detected PCIe board devices. */ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)) global_pciedma.sysfs = class_create(gDrvrName); #else global_pciedma.sysfs = class_create(THIS_MODULE, gDrvrName); #endif if (IS_ERR(global_pciedma.sysfs)) { pr_err("Error creating simple class interface\n"); return(-1); } if (dma_debug) pr_info("registering driver with PCI\n"); /* Register our PCI components and functions with the Kernel PCI core.*/ err = pci_register_driver(&PcieDmaDriver); if (dma_debug) pr_info("pci_register_driver()=%d\n", err); if (err < 0) return(err); #ifdef USE_PROC /* only when available */ if(!proc_create("driver/pciedma", 0, NULL, &PcieDmaProcOps)) return -ENOMEM; #endif return 0; } module_init(PcieDmaInit); /** * Cleanup function for the PCI Express DMA driver during module exit. * * This function performs the following tasks during driver cleanup: * 1. Unregisters the PCI driver. * 2. Cleans up each active board (if any). * 3. Destroys the sysfs class entry. * 4. Frees the allocated major and minor numbers. */ static void __exit PcieDmaExit(void) { uint8_t i; pr_info("%s.\n", __func__); pci_unregister_driver(&PcieDmaDriver); for (i = 0; i < NUM_BOARDS; i++) { if (global_pciedma.Board[i].ID != 0) { /* Do the cleanup for each active board */ pr_info("Cleaning up board: %d\n", i); // Disable and release IRQ if still active } } class_destroy(global_pciedma.sysfs); // Free every minor number and major number we reserved in init unregister_chrdev_region(global_pciedma.drvrDevNum, MAX_MINORS); return; } module_exit(PcieDmaExit); MODULE_AUTHOR("Lattice Semiconductor"); MODULE_DESCRIPTION("LSC PCIe DMA Device Driver"); MODULE_VERSION(DRV_MODULE_VERSION); /* License this so no annoying messages when loading module */ MODULE_LICENSE("Dual BSD/GPL"); MODULE_ALIAS("platform:pciedma");