[IB] uverbs: Fix device lifetime problems

Move ib_uverbs module to using cdev_alloc() and class_device_create()
so that we can handle device lifetime properly.  Now we can make sure
we keep all of our data structures around until the last way to reach
them is gone.

Signed-off-by: Roland Dreier <rolandd@cisco.com>

drivers/infiniband/core/uverbs.h

@@ -39,20 +39,38 @@
 #ifndef UVERBS_H
 #define UVERBS_H
 
-/* Include device.h and fs.h until cdev.h is self-sufficient */
-#include <linux/fs.h>
-#include <linux/device.h>
-#include <linux/cdev.h>
 #include <linux/kref.h>
 #include <linux/idr.h>
 
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_user_verbs.h>
 
+/*
+ * Our lifetime rules for these structs are the following:
+ *
+ * struct ib_uverbs_device: One reference is held by the module and
+ * released in ib_uverbs_remove_one().  Another reference is taken by
+ * ib_uverbs_open() each time the character special file is opened,
+ * and released in ib_uverbs_release_file() when the file is released.
+ *
+ * struct ib_uverbs_file: One reference is held by the VFS and
+ * released when the file is closed.  Another reference is taken when
+ * an asynchronous event queue file is created and released when the
+ * event file is closed.
+ *
+ * struct ib_uverbs_event_file: One reference is held by the VFS and
+ * released when the file is closed.  For asynchronous event files,
+ * another reference is held by the corresponding main context file
+ * and released when that file is closed.  For completion event files,
+ * a reference is taken when a CQ is created that uses the file, and
+ * released when the CQ is destroyed.
+ */
+
 struct ib_uverbs_device {
+	struct kref				ref;
 	int					devnum;
-	struct cdev				dev;
-	struct class_device			class_dev;
+	struct cdev			       *dev;
+	struct class_device		       *class_dev;
 	struct ib_device		       *ib_dev;
 	int					num_comp_vectors;
 };
@@ -115,6 +133,12 @@ struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
 void ib_uverbs_release_event_file(struct kref *ref);
 struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd);
 
+void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
+			   struct ib_uverbs_event_file *ev_file,
+			   struct ib_ucq_object *uobj);
+void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
+			      struct ib_uevent_object *uobj);
+
 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);
 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);

drivers/infiniband/core/uverbs_cmd.c

@@ -35,6 +35,7 @@
  */
 
 #include <linux/file.h>
+#include <linux/fs.h>
 
 #include <asm/uaccess.h>
 
@@ -114,7 +115,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 
 	kref_get(&file->async_file->ref);
 	kref_get(&file->ref);
-	file->ucontext   = ucontext;
+	file->ucontext = ucontext;
 
 	fd_install(resp.async_fd, filp);
 
@@ -761,7 +762,6 @@ ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
 	struct ib_cq               	*cq;
 	struct ib_ucq_object        	*uobj;
 	struct ib_uverbs_event_file	*ev_file;
-	struct ib_uverbs_event		*evt, *tmp;
 	u64				 user_handle;
 	int                        	 ret = -EINVAL;
 
@@ -790,23 +790,7 @@ ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
 	list_del(&uobj->uobject.list);
 	up(&file->mutex);
 
-	if (ev_file) {
-		spin_lock_irq(&ev_file->lock);
-		list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
-			list_del(&evt->list);
-			kfree(evt);
-		}
-		spin_unlock_irq(&ev_file->lock);
-
-		kref_put(&ev_file->ref, ib_uverbs_release_event_file);
-	}
-
-	spin_lock_irq(&file->async_file->lock);
-	list_for_each_entry_safe(evt, tmp, &uobj->async_list, obj_list) {
-		list_del(&evt->list);
-		kfree(evt);
-	}
-	spin_unlock_irq(&file->async_file->lock);
+	ib_uverbs_release_ucq(file, ev_file, uobj);
 
 	resp.comp_events_reported  = uobj->comp_events_reported;
 	resp.async_events_reported = uobj->async_events_reported;
@@ -1043,7 +1027,6 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
 	struct ib_uverbs_destroy_qp_resp resp;
 	struct ib_qp               	*qp;
 	struct ib_uevent_object        	*uobj;
-	struct ib_uverbs_event		*evt, *tmp;
 	int                        	 ret = -EINVAL;
 
 	if (copy_from_user(&cmd, buf, sizeof cmd))
@@ -1069,12 +1052,7 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
 	list_del(&uobj->uobject.list);
 	up(&file->mutex);
 
-	spin_lock_irq(&file->async_file->lock);
-	list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
-		list_del(&evt->list);
-		kfree(evt);
-	}
-	spin_unlock_irq(&file->async_file->lock);
+	ib_uverbs_release_uevent(file, uobj);
 
 	resp.events_reported = uobj->events_reported;
 
@@ -1741,7 +1719,6 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
 	struct ib_uverbs_destroy_srq_resp resp;
 	struct ib_srq               	 *srq;
 	struct ib_uevent_object        	 *uobj;
-	struct ib_uverbs_event		 *evt, *tmp;
 	int                         	  ret = -EINVAL;
 
 	if (copy_from_user(&cmd, buf, sizeof cmd))
@@ -1767,12 +1744,7 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
 	list_del(&uobj->uobject.list);
 	up(&file->mutex);
 
-	spin_lock_irq(&file->async_file->lock);
-	list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
-		list_del(&evt->list);
-		kfree(evt);
-	}
-	spin_unlock_irq(&file->async_file->lock);
+	ib_uverbs_release_uevent(file, uobj);
 
 	resp.events_reported = uobj->events_reported;