libusb: change callback register handler to int

libusb upstream uses int for register handler. This causes some library
user (like pyusb) to assume that we have int in all implementations and
therefore provides a 4 byte storage only. This causes Segmentation
fault as we will right the pointer.

Reviewed by:    adrian
Sponsored by:   The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D54211

lib/libusb/libusb.h

@@ -293,7 +293,6 @@ struct libusb_context;
 struct libusb_device;
 struct libusb_transfer;
 struct libusb_device_handle;
-struct libusb_hotplug_callback_handle_struct;
 
 struct libusb_pollfd {
 	int	fd;
@@ -322,7 +321,7 @@ typedef struct libusb_device_handle libusb_device_handle;
 typedef struct libusb_pollfd libusb_pollfd;
 typedef void (*libusb_pollfd_added_cb) (int fd, short events, void *user_data);
 typedef void (*libusb_pollfd_removed_cb) (int fd, void *user_data);
-typedef struct libusb_hotplug_callback_handle_struct *libusb_hotplug_callback_handle;
+typedef int libusb_hotplug_callback_handle;
 
 typedef struct libusb_device_descriptor {
 	uint8_t	bLength;
@@ -632,7 +631,7 @@ typedef int (*libusb_hotplug_callback_fn)(libusb_context *ctx,
     libusb_device *device, libusb_hotplug_event event, void *user_data);
 
 int	libusb_hotplug_register_callback(libusb_context *ctx, libusb_hotplug_event events, libusb_hotplug_flag flags, int vendor_id, int product_id, int dev_class, libusb_hotplug_callback_fn cb_fn, void *user_data, libusb_hotplug_callback_handle *handle);
-void	libusb_hotplug_deregister_callback(libusb_context *ctx, libusb_hotplug_callback_handle handle);
+void	libusb_hotplug_deregister_callback(libusb_context *ctx, libusb_hotplug_callback_handle callback_handle);
 void   *libusb_hotplug_get_user_data(struct libusb_context *ctx,
     libusb_hotplug_callback_handle callback_handle);
 

lib/libusb/libusb10.c

@@ -205,6 +205,7 @@ libusb_init_context(libusb_context **context,
 
 	memset(ctx, 0, sizeof(*ctx));
 	ctx->devd_pipe = -1;
+	ctx->next_callback_id = 1;
 
 	debug = getenv("LIBUSB_DEBUG");
 	if (debug != NULL) {

lib/libusb/libusb10.h

@@ -80,6 +80,7 @@ struct libusb_hotplug_callback_handle_struct {
 	int devclass;
 	libusb_hotplug_callback_fn fn;
 	void *user_data;
+	int id;
 };
 
 TAILQ_HEAD(libusb_device_head, libusb_device);
@@ -118,6 +119,7 @@ struct libusb_context {
 	libusb_pollfd_added_cb fd_added_cb;
 	libusb_pollfd_removed_cb fd_removed_cb;
 	void   *fd_cb_user_data;
+	int next_callback_id;
 };
 
 struct libusb_device {

lib/libusb/libusb10_hotplug.c

@@ -176,8 +176,9 @@ libusb_hotplug_equal(libusb_device *_adev, libusb_device *_bdev)
 }
 
 static int
-libusb_hotplug_filter(libusb_context *ctx, libusb_hotplug_callback_handle pcbh,
-    libusb_device *dev, libusb_hotplug_event event)
+libusb_hotplug_filter(libusb_context *ctx,
+    struct libusb_hotplug_callback_handle_struct *pcbh, libusb_device *dev,
+    libusb_hotplug_event event)
 {
 	if (!(pcbh->events & event))
 		return (0);
@@ -216,8 +217,7 @@ libusb_hotplug_scan(void *arg)
 {
 	struct pollfd pfd;
 	struct libusb_device_head hotplug_devs;
-	libusb_hotplug_callback_handle acbh;
-	libusb_hotplug_callback_handle bcbh;
+	struct libusb_hotplug_callback_handle_struct *acbh, *bcbh;
 	libusb_context *ctx = arg;
 	libusb_device *temp;
 	libusb_device *adev;
@@ -343,7 +343,7 @@ int libusb_hotplug_register_callback(libusb_context *ctx,
     libusb_hotplug_callback_fn cb_fn, void *user_data,
     libusb_hotplug_callback_handle *phandle)
 {
-	libusb_hotplug_callback_handle handle;
+	struct libusb_hotplug_callback_handle_struct *handle;
 	struct libusb_device *adev;
 
 	ctx = GET_CONTEXT(ctx);
@@ -365,6 +365,17 @@ int libusb_hotplug_register_callback(libusb_context *ctx,
 	if (handle == NULL)
 		return (LIBUSB_ERROR_NO_MEM);
 
+	handle->events = events;
+	handle->vendor = vendor_id;
+	handle->product = product_id;
+	handle->devclass = dev_class;
+	handle->fn = cb_fn;
+	handle->user_data = user_data;
+	CTX_LOCK(ctx);
+	if ((handle->id = ctx->next_callback_id++) < 0)
+		handle->id = ctx->next_callback_id = 1;
+	CTX_UNLOCK(ctx);
+
 	HOTPLUG_LOCK(ctx);
 	if (ctx->hotplug_handler == NO_THREAD) {
 	  	libusb_hotplug_enumerate(ctx, &ctx->hotplug_devs);
@@ -373,12 +384,6 @@ int libusb_hotplug_register_callback(libusb_context *ctx,
 		    &libusb_hotplug_scan, ctx) != 0)
 			ctx->hotplug_handler = NO_THREAD;
 	}
-	handle->events = events;
-	handle->vendor = vendor_id;
-	handle->product = product_id;
-	handle->devclass = dev_class;
-	handle->fn = cb_fn;
-	handle->user_data = user_data;
 
 	if (flags & LIBUSB_HOTPLUG_ENUMERATE) {
 		TAILQ_FOREACH(adev, &ctx->hotplug_devs, hotplug_entry) {
@@ -395,23 +400,32 @@ int libusb_hotplug_register_callback(libusb_context *ctx,
 	HOTPLUG_UNLOCK(ctx);
 
 	if (phandle != NULL)
-		*phandle = handle;
+		*phandle = handle->id;
 	return (LIBUSB_SUCCESS);
 }
 
 void libusb_hotplug_deregister_callback(libusb_context *ctx,
-    libusb_hotplug_callback_handle handle)
+    libusb_hotplug_callback_handle callback_handle)
 {
-  	ctx = GET_CONTEXT(ctx);
+	struct libusb_hotplug_callback_handle_struct *handle;
 
-	if (ctx == NULL || handle == NULL)
+	ctx = GET_CONTEXT(ctx);
+
+	if (ctx == NULL || callback_handle == 0)
 		return;
 
 	HOTPLUG_LOCK(ctx);
+	TAILQ_FOREACH(handle, &ctx->hotplug_cbh, entry) {
+		if (handle->id == callback_handle)
+			break;
+	}
+	if (handle == NULL)
+		goto clean;
 	TAILQ_REMOVE(&ctx->hotplug_cbh, handle, entry);
 	libusb_interrupt_event_handler(ctx);
-	HOTPLUG_UNLOCK(ctx);
 
+clean:
+	HOTPLUG_UNLOCK(ctx);
 	free(handle);
 }
 
@@ -419,13 +433,13 @@ void *
 libusb_hotplug_get_user_data(struct libusb_context *ctx,
     libusb_hotplug_callback_handle callback_handle)
 {
-	libusb_hotplug_callback_handle handle;
+	struct libusb_hotplug_callback_handle_struct *handle;
 
 	ctx = GET_CONTEXT(ctx);
 
 	HOTPLUG_LOCK(ctx);
 	TAILQ_FOREACH(handle, &ctx->hotplug_cbh, entry) {
-		if (handle == callback_handle)
+		if (handle->id == callback_handle)
 			break;
 	}
 	HOTPLUG_UNLOCK(ctx);