Implement DMA access synchronization in the driver

5 files changed, 142 insertions, 31 deletions

diff --git a/driver/base.c b/driver/base.c
index 37102d8..d88dcf2 100644
--- a/driver/base.c
+++ b/driver/base.c
@@ -353,8 +353,6 @@ static int __devinit pcidriver_probe(struct pci_dev *pdev, const struct pci_devi
 	spin_lock_init(&(privdata->kmemlist_lock));
 	atomic_set(&privdata->kmem_count, 0);
 	
-	privdata->kmem_cur = NULL;
-
 	INIT_LIST_HEAD(&(privdata->umem_list));
 	spin_lock_init(&(privdata->umemlist_lock));
 	atomic_set(&privdata->umem_count, 0);
diff --git a/driver/common.h b/driver/common.h
index e45ae25..b79d5ec 100644
--- a/driver/common.h
+++ b/driver/common.h
@@ -6,6 +6,14 @@
 /*************************************************************************/
 /* Private data types and structures */
 
+#define KMEM_REF_HW 		0x80000000	/**< Special reference to indicate hardware access */
+#define KMEM_REF_COUNT		0x0FFFFFFF	/**< Mask of reference counter (mmap/munmap) */
+
+#define KMEM_MODE_REUSABLE	0x80000000	/**< Indicates reusable buffer */
+#define KMEM_MODE_EXCLUSIVE	0x40000000	/**< Only a single process is allowed to mmap the buffer */
+#define KMEM_MODE_PERSISTENT	0x20000000	/**< Persistent mode instructs kmem_free to preserve buffer in memory */
+#define KMEM_MODE_COUNT		0x0FFFFFFF	/**< Mask of reuse counter (alloc/free) */
+
 /* Define an entry in the kmem list (this list is per device) */
 /* This list keeps references to the allocated kernel buffers */
 typedef struct {
@@ -15,8 +23,14 @@ typedef struct {
 	unsigned long cpua;
 	unsigned long size;
 	unsigned long type;
+
 	unsigned long use;
 	unsigned long item;
+
+	spinlock_t lock;
+	unsigned long mode;
+	unsigned long refs;
+
 	struct class_device_attribute sysfs_attr;	/* initialized when adding the entry */
 } pcidriver_kmem_entry_t;
 
@@ -57,7 +71,7 @@ typedef struct  {
 	struct list_head kmem_list;			/* List of 'kmem_list_entry's associated with this device */
 	atomic_t kmem_count;				/* id for next kmem entry */
 
-	pcidriver_kmem_entry_t *kmem_cur;		/* Currently selected kmem buffer, for mmap */
+	int kmem_cur_id;				/* Currently selected kmem buffer, for mmap */
 	
 	spinlock_t umemlist_lock;			/* Spinlock to lock umem list operations */
 	struct list_head umem_list;			/* List of 'umem_list_entry's associated with this device */
diff --git a/driver/kmem.c b/driver/kmem.c
index ed2a2af..a720095 100644
--- a/driver/kmem.c
+++ b/driver/kmem.c
@@ -35,24 +35,48 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 	pcidriver_kmem_entry_t *kmem_entry;
 	void *retptr;
 
-	privdata->kmem_cur = NULL;
-	
-	if (kmem_handle->reuse) {
+	if (kmem_handle->flags&KMEM_FLAG_REUSE) {
 /*	    kmem_entry = pcidriver_kmem_find_entry_use(privdata, kmem_handle->use, kmem_handle->item);
 	    if (kmem_entry) {
-		if (kmem_handle->type != kmem_entry->type) return EINVAL;
+		unsigned long flags = kmem_handle->flags;
+		
+		if (kmem_handle->type != kmem_entry->type) {
+			mod_info("Invalid type of reusable kmem_entry\n");
+			return -EINVAL;
+		}
 
-		if (kmem_handle->type == PCILIB_KMEM_TYPE_PAGE) kmem_handle->size = kmem_entry->size;
-		else if (kmem_handle->size != kmem_entry->size) return EINVAL;
+		if (kmem_handle->type == PCILIB_KMEM_TYPE_PAGE) {
+			kmem_handle->size = kmem_entry->size;
+		} else if (kmem_handle->size != kmem_entry->size) {
+			mod_info("Invalid size of reusable kmem_entry\n");
+			return -EINVAL;
+		}
+		
+		if (((kmem_entry->mode&KMEM_MODE_EXCLUSIVE)?1:0) != ((flags&KMEM_FLAG_EXCLUSIVE)?1:0)) {
+			mod_info("Invalid mode of reusable kmem_entry\n");
+			return -EINVAL;
+		}
 
+		if ((kmem_entry->mode&KMEM_MODE_COUNT)==KMEM_MODE_COUNT) {
+			mod_info("Reuse counter of kmem_entry is overflown");
+			return -EBUSY;
+		}
+		
 		kmem_handle->handle_id = kmem_entry->id;
 		kmem_handle->pa = (unsigned long)(kmem_entry->dma_handle);
-		
-		privdata->kmem_cur = kmem_entry;
+
+		kmem_handle->flags = KMEM_FLAG_REUSED;
+		if (kmem_entry->refs&KMEM_REF_HW) kmem_handle->flags |= KMEM_FLAG_REUSED_HW;
+		if (kmem_entry->mode&KMEM_MODE_PERSISTENT) kmem_handle->flags |= KMEM_FLAG_REUSED_PERSISTENT;
+
+		kmem_entry->mode += 1;
+		if (flags&KMEM_FLAG_HW) kmem_entry->refs |= KMEM_REF_HW;
+		if (flags&KMEM_FLAG_PERSISTENT) kmem_entry->mode |= KMEM_MODE_PERSISTENT;
+
+		privdata->kmem_cur_id = kmem_entry->id;
 
 		return 0;
 	    }*/
-	    kmem_handle->reuse = 0;
 	}
 
 	/* First, allocate zeroed memory for the kmem_entry */
@@ -61,10 +85,12 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 
 	/* Initialize the kmem_entry */
 	kmem_entry->id = atomic_inc_return(&privdata->kmem_count) - 1;
+	privdata->kmem_cur_id = kmem_entry->id;
+	kmem_handle->handle_id = kmem_entry->id;
+
 	kmem_entry->use = kmem_handle->use;
 	kmem_entry->item = kmem_handle->item;
 	kmem_entry->type = kmem_handle->type;
-	kmem_handle->handle_id = kmem_entry->id;
 
 	/* Initialize sysfs if possible */
 	if (pcidriver_sysfs_initialize_kmem(privdata, kmem_entry->id, &(kmem_entry->sysfs_attr)) != 0)
@@ -94,11 +120,23 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 	}
 	if (retptr == NULL)
 		goto kmem_alloc_mem_fail;
-	
+
 	kmem_entry->size = kmem_handle->size;
 	kmem_entry->cpua = (unsigned long)retptr;
 	kmem_handle->pa = (unsigned long)(kmem_entry->dma_handle);
 
+	kmem_entry->mode = 1;
+	if (kmem_handle->flags&KMEM_FLAG_REUSE) {
+	    kmem_entry->mode |= KMEM_MODE_REUSABLE;
+	    if (kmem_handle->flags&KMEM_FLAG_EXCLUSIVE) kmem_entry->mode |= KMEM_MODE_EXCLUSIVE;
+	    if (kmem_handle->flags&KMEM_FLAG_PERSISTENT) kmem_entry->mode |= KMEM_MODE_PERSISTENT;
+	}
+	
+	kmem_entry->refs = 0;
+	if (kmem_handle->flags&KMEM_FLAG_HW) kmem_entry->refs |= KMEM_REF_HW;
+
+        kmem_handle->flags = 0;
+	
 	set_pages_reserved_compat(kmem_entry->cpua, kmem_entry->size);
 
 	/* Add the kmem_entry to the list of the device */
@@ -123,15 +161,37 @@ int pcidriver_kmem_free( pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 {
 	pcidriver_kmem_entry_t *kmem_entry;
 
-	if (kmem_handle->reuse) {
-	    // just mark free
-	    return 0;
-	}
-
 	/* Find the associated kmem_entry for this buffer */
 	if ((kmem_entry = pcidriver_kmem_find_entry(privdata, kmem_handle)) == NULL)
 		return -EINVAL;					/* kmem_handle is not valid */
 
+
+	if (kmem_entry->mode&KMEM_MODE_COUNT)
+		kmem_entry->mode -= 1;
+	
+	if (kmem_handle->flags&KMEM_FLAG_HW)
+		kmem_entry->refs &= ~KMEM_REF_HW;
+	
+	if (kmem_handle->flags&KMEM_FLAG_PERSISTENT)
+		kmem_entry->mode &= ~KMEM_MODE_PERSISTENT;
+	
+	if (kmem_handle->flags&KMEM_FLAG_REUSE) 
+		return 0;
+
+	if (kmem_entry->refs) {
+		mod_info("can't free referenced kmem_entry\n");
+		kmem_entry->mode += 1;
+		return -EBUSY;
+	}
+	
+	if (kmem_entry->mode & KMEM_MODE_PERSISTENT) {
+		mod_info("can't free persistent kmem_entry\n");
+		return -EBUSY;
+	}
+
+	if (((kmem_entry->mode&KMEM_MODE_EXCLUSIVE)==0)&&(kmem_entry->mode&KMEM_MODE_COUNT)) 
+		return 0;
+	
 	return pcidriver_kmem_free_entry(privdata, kmem_entry);
 }
 
@@ -142,14 +202,23 @@ int pcidriver_kmem_free( pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
  */
 int pcidriver_kmem_free_all(pcidriver_privdata_t *privdata)
 {
+	int failed = 0;
 	struct list_head *ptr, *next;
 	pcidriver_kmem_entry_t *kmem_entry;
 
 	/* iterate safely over the entries and delete them */
 	list_for_each_safe(ptr, next, &(privdata->kmem_list)) {
 		kmem_entry = list_entry(ptr, pcidriver_kmem_entry_t, list);
-		pcidriver_kmem_free_entry(privdata, kmem_entry); 		/* spin lock inside! */
+/*		if (kmem_entry->refs)
+			failed = 1;
+		else*/
+			pcidriver_kmem_free_entry(privdata, kmem_entry); 		/* spin lock inside! */
 	}
+	
+	if (failed) {
+		mod_info("Some kmem_entries are still referenced\n");
+		return -EBUSY;
+	}	
 
 	return 0;
 }
@@ -219,8 +288,6 @@ int pcidriver_kmem_sync( pcidriver_privdata_t *privdata, kmem_sync_t *kmem_sync
  */
 int pcidriver_kmem_free_entry(pcidriver_privdata_t *privdata, pcidriver_kmem_entry_t *kmem_entry)
 {
-	privdata->kmem_cur = NULL;
-
 	pcidriver_sysfs_remove(privdata, &(kmem_entry->sysfs_attr));
 
 	/* Go over the pages of the kmem buffer, and mark them as not reserved */
@@ -339,7 +406,7 @@ pcidriver_kmem_entry_t *pcidriver_kmem_find_entry_use(pcidriver_privdata_t *priv
 	list_for_each(ptr, &(privdata->kmem_list)) {
 		entry = list_entry(ptr, pcidriver_kmem_entry_t, list);
 
-		if ((entry->use == use)&&(entry->item == item)) {
+		if ((entry->use == use)&&(entry->item == item)&&(entry->mode&KMEM_MODE_REUSABLE)) {
 			result = entry;
 			break;
 		}
@@ -350,6 +417,15 @@ pcidriver_kmem_entry_t *pcidriver_kmem_find_entry_use(pcidriver_privdata_t *priv
 }
 
 
+void pcidriver_kmem_mmap_close(struct vm_area_struct *vma) {
+    pcidriver_kmem_entry_t *kmem_entry = (pcidriver_kmem_entry_t*)vma->vm_private_data;
+    if (kmem_entry) kmem_entry->refs -= 1;
+}
+
+static struct vm_operations_struct pcidriver_kmem_mmap_ops = {
+    .close = pcidriver_kmem_mmap_close
+};
+
 /**
  *
  * mmap() kernel memory to userspace.
@@ -365,15 +441,11 @@ int pcidriver_mmap_kmem(pcidriver_privdata_t *privdata, struct vm_area_struct *v
 
 	/* FIXME: Is this really right? Always just the latest one? Can't we identify one? */
 	/* Get latest entry on the kmem_list */
-	spin_lock(&(privdata->kmemlist_lock));
-	if (list_empty(&(privdata->kmem_list))) {
-		spin_unlock(&(privdata->kmemlist_lock));
+	kmem_entry = pcidriver_kmem_find_entry_id(privdata, privdata->kmem_cur_id);
+	if (!kmem_entry) {
 		mod_info("Trying to mmap a kernel memory buffer without creating it first!\n");
 		return -EFAULT;
 	}
-	if (privdata->kmem_cur) kmem_entry = privdata->kmem_cur;
-	else kmem_entry = list_entry(privdata->kmem_list.prev, pcidriver_kmem_entry_t, list);
-	spin_unlock(&(privdata->kmemlist_lock));
 
 	mod_info_dbg("Got kmem_entry with id: %d\n", kmem_entry->id);
 
@@ -386,6 +458,17 @@ int pcidriver_mmap_kmem(pcidriver_privdata_t *privdata, struct vm_area_struct *v
 		return -EINVAL;
 	}
 
+	/* reference counting */
+	if ((kmem_entry->mode&KMEM_MODE_EXCLUSIVE)&&(kmem_entry->refs&KMEM_REF_COUNT)) {
+		mod_info("can't make second mmaping for exclusive kmem_entry\n");
+		return -EBUSY;
+	}
+	if ((kmem_entry->refs&KMEM_REF_COUNT)==KMEM_REF_COUNT) {
+		mod_info("maximal amount of references is reached by kmem_entry\n");
+		return -EBUSY;
+	}
+	kmem_entry->refs += 1;
+
 	vma->vm_flags |= (VM_RESERVED);
 
 #ifdef pgprot_noncached
@@ -406,8 +489,12 @@ int pcidriver_mmap_kmem(pcidriver_privdata_t *privdata, struct vm_area_struct *v
 
 	if (ret) {
 		mod_info("kmem remap failed: %d (%lx)\n", ret,kmem_entry->cpua);
+		kmem_entry->refs -= 1;
 		return -EAGAIN;
 	}
+
+	vma->vm_ops = &pcidriver_kmem_mmap_ops;
+	vma->vm_private_data = (void*)kmem_entry;
 	
 	return ret;
 }
diff --git a/driver/pciDriver.h b/driver/pciDriver.h
index 3ff3357..9aa580f 100644
--- a/driver/pciDriver.h
+++ b/driver/pciDriver.h
@@ -99,6 +99,18 @@
 /* Maximum number of interrupt sources */
 #define PCIDRIVER_INT_MAXSOURCES 16
 
+
+#define KMEM_FLAG_REUSE 1	/**< Try to reuse existing buffer with the same use & item */
+#define KMEM_FLAG_EXCLUSIVE 2	/**< Allow only a single application accessing a specified use & item */
+#define KMEM_FLAG_PERSISTENT 4	/**< Sets persistent mode */
+#define KMEM_FLAG_HW 8		/**< The buffer may be accessed by hardware, the hardware access will not occur any more if passed to _free function */
+
+#define KMEM_FLAG_REUSED 1		/**< Indicates if buffer with specified use & item was already allocated and reused */
+#define KMEM_FLAG_REUSED_PERSISTENT 4	/**< Indicates that reused buffer was persistent before the call */
+#define KMEM_FLAG_REUSED_HW 8		/**< Indicates that reused buffer had a HW reference before the call */
+
+
+
 /* Types */
 typedef struct {
 	unsigned long type;
@@ -107,7 +119,7 @@ typedef struct {
 	unsigned long align;
 	unsigned long use;
 	unsigned long item;
-	int reuse;
+	int flags;
 	int handle_id;
 } kmem_handle_t;
 
diff --git a/driver/sysfs.c b/driver/sysfs.c
index 0653aef..ab0925c 100644
--- a/driver/sysfs.c
+++ b/driver/sysfs.c
@@ -98,7 +98,7 @@ static SYSFS_GET_FUNCTION(pcidriver_show_kmem_entry)
         int id = simple_strtol(attr->attr.name + strlen("kbuf"), NULL, 10);
 	pcidriver_kmem_entry_t *entry = pcidriver_kmem_find_entry_id(privdata, id);
 	if (entry)
-	    return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lu\nuse: 0x%lx\nitem: %lu\nsize: %lu\n", id, entry->type, entry->use, entry->item, entry->size);
+	    return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lu\nuse: 0x%lx\nitem: %lu\nsize: %lu\nrefs: %lu\nhw ref: %i\nmode: %lx\n", id, entry->type, entry->use, entry->item, entry->size, entry->refs&KMEM_REF_COUNT, (entry->refs&KMEM_REF_HW)?1:0, entry->mode);
 	else
 	    return snprintf(buf, PAGE_SIZE, "I am in the kmem_entry show function for buffer %d\n", id);
 #else