flxnet_pcie.c

/*

	This is an minimal example driver for host side
	to demonstrate serial communication over 
	PCIe using only a few registers from a separate BAR.

	When module is loaded:
	1. Find vmk180 PCIe card (vendor=0x10ee, device=0xb03f)
	2. Enable the PCIe device, reserve and map the BAR corresponding
		to the CPM AXI bridge (set by bar_idx parameter)
	3. Register the peer with flxnet network device


	Elena Zhivun <elena.zhivun@cern.ch>
*/

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/stat.h>
#include <linux/mm.h>
#include <linux/pci.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/vmalloc.h>
#include <linux/module.h>

#include "flxnet/flxnet.h"

extern struct flxnet_peer * flxnet_add_peer(void* __iomem base_address,
	int index, struct module *owner);

#define DRV_MODULE_NAME	"flxnet_pcie"

#define FLX_PCIE_MAGIC	0xDED133D0UL

struct pcie_flx_serial_device {
	unsigned long magic;		/* structure ID for sanity checks */
	struct list_head list_head;	
	struct pci_dev *pdev;		/* pci device struct from probe() */
	void __iomem *bar;			/* addresses for the relevant BAR */
	int bar_idx;

	spinlock_t lock;			/* protects concurrent access */
	unsigned int flags;

	struct flxnet_peer * peer;
};


// which bar number the AXI serial device is mapped to
static int bar_idx = 3;

// which offset the AXI serial device is found at
static uint dev_base_addr = 0x0;

// list of FELIX devices
static LIST_HEAD(flx_devices);

static int configure_pcie_device(struct pci_dev *pdev, int index);
static void deconfigure_device(struct pcie_flx_serial_device *fdev);
static int map_single_bar(struct pcie_flx_serial_device *fdev, int idx);

static int flxnet_pcie_init(void) {

	int i;
	int index = 0;
	int rv = -ENODEV;
	struct pci_dev *pdev; //previous device
	pdev = NULL; 

	pr_info("flxnet_pcie: loading %s module", DRV_MODULE_NAME);	
	for(i=0; i<MAXCARDS; i++) {
		if ( (pdev = pci_get_device(0x10dc, 0x0427, pdev)) ) {
			pr_info("flxnet_pcie: found %s", pci_name(pdev));
			rv = configure_pcie_device(pdev, index);
			index++;
			if (rv < 0) {			
				pr_warn("flxnet_pcie: failed to configure %s", pci_name(pdev));
			}			
		}
	}

	return rv;	
}


static void flxnet_pcie_exit(void) {

	struct pcie_flx_serial_device *fdev, *tmp;

	pr_info("flxnet_pcie: removing %s module", DRV_MODULE_NAME);

	if (!list_empty(&flx_devices)) {
		pr_info("flxnet_pcie: reconfiguring PCIe devices");
		list_for_each_entry_safe(fdev, tmp, &flx_devices, list_head) {
			deconfigure_device(fdev);
		}	
	}
}


static int configure_pcie_device(struct pci_dev *pdev, int index) {

	int rv;
	struct pcie_flx_serial_device *fdev;

	if (!pdev) {
		pr_err("flxnet_pcie: invalid PCIe device structure");
		return -EFAULT;
	}

	pr_info("flxnet_pcie: initializing the PCIe device");

	fdev = kzalloc(sizeof(struct pcie_flx_serial_device), GFP_KERNEL);
	if (!fdev)
		return -ENOMEM;

	spin_lock_init(&fdev->lock);
	fdev->magic = FLX_PCIE_MAGIC;
	fdev->pdev = pdev;
	fdev->bar_idx = bar_idx;
	INIT_LIST_HEAD(&fdev->list_head);

	pr_info("flxnet_pcie: enabling PCIe device");
	rv = pci_enable_device(pdev);
	if (rv) {
		pr_err("flxnet_pcie: pci_enable_device() failed, %d", rv);
		goto err_enable;
	}

	// reserve the selected bar for this driver
	pr_info("flxnet_pcie: reserving BAR memory");
	rv = pci_request_region(pdev, bar_idx, DRV_MODULE_NAME);
	if (rv) {
		pr_err("flxnet_pcie: pci_request_region() failed (%d)", rv);
		goto err_region;
	}		


	pr_info("flxnet_pcie: mapping BAR memory");
	rv = map_single_bar(fdev, bar_idx);
	if (rv < 0) {
		pr_info("flxnet_pcie: disabling PCIe device, no network registred");
		pci_disable_device(pdev);
		pci_release_region(pdev, bar_idx);
		return 0;
	} else if (rv < MIN_MMAP_SIZE) {
		pr_err("flxnet_pcie: mapped region is too small (%d bytes < %d)",
			rv, MIN_MMAP_SIZE);
		goto err_peer;
	}

	// add this device to the network
	pr_info("flxnet_pcie: inserting peer into flxnet");
	fdev->peer = flxnet_add_peer(fdev->bar + dev_base_addr,
		index, THIS_MODULE);

	if (IS_ERR(fdev->peer)) {
		pr_err("flxnet_pcie: error inserting peer into flxnet");
		goto err_peer;
	}

	// device structure
	list_add(&fdev->list_head, &flx_devices);

	return 0;


err_peer:
	pr_info("flxnet_pcie: unmapping BAR memory");
	pci_iounmap(pdev, fdev->bar);

err_map:
	pr_info("flxnet_pcie: disabling PCIe device");
	pci_disable_device(pdev);
	pci_release_region(pdev, bar_idx);

err_enable:
	pr_err("flxnet_pcie: pdev 0x%p, err %d", pdev, rv);
	kfree(fdev);
	return rv;

err_region:
	pci_disable_device(pdev);
	goto err_enable;
}


// Unregister and deconfigure a felix serial device
static void deconfigure_device(struct pcie_flx_serial_device *fdev) {
	struct pci_dev * pdev;

	pr_info("flxnet_pcie: deconfiguring device %p", fdev);

	if (!fdev) {
		pr_err("flxnet_pcie: felix_flx_serial_device ptr is NULL");
		return;
	}

	if (fdev->magic != FLX_PCIE_MAGIC) {
		pr_err("flxnet_pcie: incorrect MAGIC value in deconfigure_device()");
		return;
	}

	pdev = fdev->pdev;

	if (!pdev) {
		pr_err("flxnet_pcie: pci_dev ptr is NULL");
		goto err_exit;
	}

	if (fdev->bar) {
		pci_iounmap(pdev, fdev->bar);		
	} else {
		pr_err("flxnet_pcie: BAR memory is not mapped");
	}

	pci_disable_device(pdev);
	pci_release_region(pdev, bar_idx);
	
err_exit:
	list_del(&fdev->list_head);
	kfree(fdev);
}


// map BAR containing the serial communication registers
static int map_single_bar(struct pcie_flx_serial_device *fdev, int idx) {

	struct pci_dev *pdev = fdev->pdev;
	resource_size_t bar_start = pci_resource_start(pdev, idx);
	resource_size_t bar_len = pci_resource_len(pdev, idx);
	resource_size_t map_len = bar_len;

	fdev->bar = NULL;

	/* do not map BARs with length 0. Note that start MAY be 0! */
	if (!bar_len) {
		pr_info("flxnet_pcie: BAR #%d is not present", idx);
		return -1;
	}

	/* BAR size exceeds maximum desired mapping? */
	if (bar_len > INT_MAX) {
		pr_info("flxnet_pcie: limit BAR %d mapping from %llu to %d bytes", idx,
			(u64)bar_len, INT_MAX);
		map_len = (resource_size_t)INT_MAX;
	}

	/*
	 * map the full device memory or IO region into kernel virtual
	 * address space
	 */
	fdev->bar = pci_iomap(pdev, idx, map_len);

	if (!fdev->bar) {
		pr_info("flxnet_pcie: could not map BAR %d", idx);
		return -1;
	}

	pr_info("flxnet_pcie: BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)", idx,
		(u64)bar_start, fdev->bar, (u64)map_len, (u64)bar_len);

	return (int)map_len;
}


module_init(flxnet_pcie_init);
module_exit(flxnet_pcie_exit);
module_param(bar_idx, int, S_IRUSR);
MODULE_PARM_DESC(bar_idx, "BAR of the FIFO AXI device");
module_param(dev_base_addr, uint, S_IRUSR);
MODULE_PARM_DESC(dev_base_addr, "Base offset of the FIFO AXI device");

MODULE_AUTHOR("Elena Zhivun <elena.zhivun@cern.ch>");
MODULE_DESCRIPTION("Example driver for establishing network with VMK180 over PCIe");
MODULE_LICENSE("GPL");