Introduce fine-grained CTL locking to improve SMP scalability.

Split global ctl_lock, historically protecting most of CTL context:
 - remaining ctl_lock now protects lists of fronends and backends;
 - per-LUN lun_lock(s) protect LUN-specific information;
 - per-thread queue_lock(s) protect request queues.
This allows to radically reduce congestion on ctl_lock.

Create multiple worker threads, depending on number of CPUs, and assign
each LUN to one of them.  This allows to spread load between multiple CPUs,
still avoiging congestion on queues and LUNs locks.

On 40-core server, exporting 5 LUNs, each backed by gstripe of SATA SSDs,
accessed via 6 iSCSI connections, this change improves peak request rate
from 250K to 680K IOPS.

MFC after:	2 weeks
Sponsored by:	iXsystems, Inc.

sys/cam/ctl/ctl.h

@@ -130,21 +130,6 @@ typedef enum {
 
 MALLOC_DECLARE(M_CTL);
 
-typedef enum {
-	CTL_THREAD_NONE		= 0x00,
-	CTL_THREAD_WAKEUP	= 0x01
-} ctl_thread_flags;
-
-struct ctl_thread {
-	void			(*thread_func)(void *arg);
-	void			*arg;
-	struct cv		wait_queue;
-	const char		*thread_name;
-	ctl_thread_flags	thread_flags;
-	struct completion	*thread_event;
-	struct task_struct	*task;
-};
-
 struct ctl_page_index;
 
 #ifdef SYSCTL_DECL	/* from sysctl.h */
@@ -198,16 +183,6 @@ void ctl_datamove(union ctl_io *io);
 void ctl_done(union ctl_io *io);
 void ctl_data_submit_done(union ctl_io *io);
 void ctl_config_write_done(union ctl_io *io);
-#if 0
-int ctl_thread(void *arg);
-#endif
-void ctl_wakeup_thread(void);
-#if 0
-struct ctl_thread *ctl_create_thread(void (*thread_func)
-	(void *thread_arg), void *thread_arg, const char *thread_name);
-void ctl_signal_thread(struct ctl_thread *thread);
-void ctl_shutdown_thread(struct ctl_thread *thread);
-#endif
 void ctl_portDB_changed(int portnum);
 void ctl_init_isc_msg(void);
 

sys/cam/ctl/ctl_io.h

@@ -220,6 +220,7 @@ struct ctl_nexus {
 	uint32_t targ_port;		/* Target port, filled in by PORT */
 	struct ctl_id targ_target;	/* Destination target */
 	uint32_t targ_lun;		/* Destination lun */
+	uint32_t targ_mapped_lun;	/* Destination lun CTL-wide */
 	uint32_t (*lun_map_fn)(void *arg, uint32_t lun);
 	void *lun_map_arg;
 };

sys/cam/ctl/ctl_private.h

@@ -412,6 +412,18 @@ struct ctl_wwpn_iid {
 	int32_t port;
 };
 
+#define CTL_MAX_THREADS		16
+
+struct ctl_thread {
+	struct mtx_padalign queue_lock;
+	struct ctl_softc	*ctl_softc;
+	struct thread		*thread;
+	STAILQ_HEAD(, ctl_io_hdr) incoming_queue;
+	STAILQ_HEAD(, ctl_io_hdr) rtr_queue;
+	STAILQ_HEAD(, ctl_io_hdr) done_queue;
+	STAILQ_HEAD(, ctl_io_hdr) isc_queue;
+};
+
 struct ctl_softc {
 	struct mtx ctl_lock;
 	struct cdev *dev;
@@ -428,7 +440,7 @@ struct ctl_softc {
 	struct ctl_io_pool *internal_pool;
 	struct ctl_io_pool *emergency_pool;
 	struct ctl_io_pool *othersc_pool;
-	struct proc *work_thread;
+	struct proc *ctl_proc;
 	int targ_online;
 	uint32_t ctl_lun_mask[CTL_MAX_LUNS >> 5];
 	struct ctl_lun *ctl_luns[CTL_MAX_LUNS];
@@ -437,10 +449,6 @@ struct ctl_softc {
 	uint64_t aps_locked_lun;
 	STAILQ_HEAD(, ctl_lun) lun_list;
 	STAILQ_HEAD(, ctl_be_lun) pending_lun_queue;
-	STAILQ_HEAD(, ctl_io_hdr) incoming_queue;
-	STAILQ_HEAD(, ctl_io_hdr) rtr_queue;
-	STAILQ_HEAD(, ctl_io_hdr) done_queue;
-	STAILQ_HEAD(, ctl_io_hdr) isc_queue;
 	uint32_t num_frontends;
 	STAILQ_HEAD(, ctl_frontend) fe_list;
 	struct ctl_frontend *ctl_ports[CTL_MAX_PORTS];
@@ -452,6 +460,7 @@ struct ctl_softc {
 	STAILQ_HEAD(, ctl_io_pool) io_pools;
 	time_t last_print_jiffies;
 	uint32_t skipped_prints;
+	struct ctl_thread threads[CTL_MAX_THREADS];
 };
 
 #ifdef _KERNEL
@@ -484,7 +493,6 @@ int ctl_inquiry(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_in(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_out(struct ctl_scsiio *ctsio);
 int ctl_maintenance_in(struct ctl_scsiio *ctsio);
-void ctl_done_lock(union ctl_io *io, int have_lock);
 int ctl_isc(struct ctl_scsiio *ctsio);
 
 #endif	/* _KERNEL */