Fix several problems with mapping code.

Reviewed by:    ken, scottl, asomers, ambrisko, mav
Approved by:	ken, mav
MFC after:      1 week
Differential Revision: https://reviews.freebsd.org/D10861

share/man/man4/mpr.4

@@ -1,8 +1,8 @@
 .\"
 .\" Copyright (c) 2010 Spectra Logic Corporation
 .\" Copyright (c) 2014 LSI Corp
-.\" Copyright (c) 2017 Avago Technologies
+.\" Copyright (c) 2015-2017 Avago Technologies
-.\" Copyright (c) 2017 Broadcom Ltd.
+.\" Copyright (c) 2015-2017 Broadcom Ltd.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -38,7 +38,7 @@
 .\" $Id$
 .\" $FreeBSD$
 .\"
-.Dd May 17, 2017
+.Dd May 25, 2017
 .Dt MPR 4
 .Os
 .Sh NAME

sys/dev/mpr/mpr.c

@@ -520,7 +520,8 @@ mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching)
 	 */
 	if (reallocating) {
 		mpr_iocfacts_free(sc);
-		mprsas_realloc_targets(sc, saved_facts.MaxTargets);
+		mprsas_realloc_targets(sc, saved_facts.MaxTargets +
+		    saved_facts.MaxVolumes);
 	}
 
 	/*
@@ -1663,6 +1664,7 @@ mpr_attach(struct mpr_softc *sc)
 
 	mtx_init(&sc->mpr_mtx, "MPR lock", NULL, MTX_DEF);
 	callout_init_mtx(&sc->periodic, &sc->mpr_mtx, 0);
+	callout_init_mtx(&sc->device_check_callout, &sc->mpr_mtx, 0);
 	TAILQ_INIT(&sc->event_list);
 	timevalclear(&sc->lastfail);
 
@@ -1832,6 +1834,7 @@ mpr_free(struct mpr_softc *sc)
 	mpr_unlock(sc);
 	/* Lock must not be held for this */
 	callout_drain(&sc->periodic);
+	callout_drain(&sc->device_check_callout);
 
 	if (((error = mpr_detach_log(sc)) != 0) ||
 	    ((error = mpr_detach_sas(sc)) != 0))

sys/dev/mpr/mpr_sas.c

@@ -745,7 +745,7 @@ mpr_attach_sas(struct mpr_softc *sc)
 	 * of MaxTargets here so that we don't get into trouble later.  This
 	 * should move into the reinit logic.
 	 */
-	sassc->maxtargets = sc->facts->MaxTargets;
+	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
 	sassc->targets = malloc(sizeof(struct mprsas_target) *
 	    sassc->maxtargets, M_MPR, M_WAITOK|M_ZERO);
 	if (!sassc->targets) {
@@ -963,6 +963,25 @@ mprsas_discovery_end(struct mprsas_softc *sassc)
 	if (sassc->flags & MPRSAS_DISCOVERY_TIMEOUT_PENDING)
 		callout_stop(&sassc->discovery_callout);
 
+	/*
+	 * After discovery has completed, check the mapping table for any
+	 * missing devices and update their missing counts. Only do this once
+	 * whenever the driver is initialized so that missing counts aren't
+	 * updated unnecessarily. Note that just because discovery has
+	 * completed doesn't mean that events have been processed yet. The
+	 * check_devices function is a callout timer that checks if ALL devices
+	 * are missing. If so, it will wait a little longer for events to
+	 * complete and keep resetting itself until some device in the mapping
+	 * table is not missing, meaning that event processing has started.
+	 */
+	if (sc->track_mapping_events) {
+		mpr_dprint(sc, MPR_XINFO | MPR_MAPPING, "Discovery has "
+		    "completed. Check for missing devices in the mapping "
+		    "table.\n");
+		callout_reset(&sc->device_check_callout,
+		    MPR_MISSING_CHECK_DELAY * hz, mpr_mapping_check_devices,
+		    sc);
+	}
 }
 
 static void
@@ -996,7 +1015,12 @@ mprsas_action(struct cam_sim *sim, union ccb *ccb)
 		cpi->hba_eng_cnt = 0;
 		cpi->max_target = sassc->maxtargets - 1;
 		cpi->max_lun = 255;
-		cpi->initiator_id = sassc->maxtargets - 1;
+
+		/*
+		 * initiator_id is set here to an ID outside the set of valid
+		 * target IDs (including volumes).
+		 */
+		cpi->initiator_id = sassc->maxtargets;
 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);

sys/dev/mpr/mpr_sas_lsi.c

@@ -223,8 +223,9 @@ mprsas_fw_work(struct mpr_softc *sc, struct mpr_fw_event_work *fw_event)
 				if (mprsas_add_device(sc,
 				    le16toh(phy->AttachedDevHandle),
 				    phy->LinkRate)) {
-					printf("%s: failed to add device with "
+					mpr_dprint(sc, MPR_ERROR, "%s: "
-					    "handle 0x%x\n", __func__,
+					    "failed to add device with handle "
+					    "0x%x\n", __func__,
 					    le16toh(phy->AttachedDevHandle));
 					mprsas_prepare_remove(sassc, le16toh(
 					    phy->AttachedDevHandle));
@@ -289,7 +290,7 @@ mprsas_fw_work(struct mpr_softc *sc, struct mpr_fw_event_work *fw_event)
 
 		element =
 		    (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
-		id = mpr_mapping_get_raid_id_from_handle(sc,
+		id = mpr_mapping_get_raid_tid_from_handle(sc,
 		    element->VolDevHandle);
 
 		mpr_mapping_ir_config_change_event(sc, event_data);
@@ -833,10 +834,17 @@ mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate)
 	 *  1 - use the PhyNum field as a fallback to the mapping logic
 	 *  0 - never use the PhyNum field
 	 * -1 - only use the PhyNum field
+	 *
+	 * Note that using the Phy number to map a device can cause device adds
+	 * to fail if multiple enclosures/expanders are in the topology. For
+	 * example, if two devices are in the same slot number in two different
+	 * enclosures within the topology, only one of those devices will be
+	 * added. PhyNum mapping should not be used if multiple enclosures are
+	 * in the topology.
 	 */
 	id = MPR_MAP_BAD_ID;
 	if (sc->use_phynum != -1) 
-		id = mpr_mapping_get_sas_id(sc, sas_address, handle);
+		id = mpr_mapping_get_tid(sc, sas_address, handle);
 	if (id == MPR_MAP_BAD_ID) {
 		if ((sc->use_phynum == 0) ||
 		    ((id = config_page.PhyNum) > sassc->maxtargets)) {
@@ -847,20 +855,32 @@ mprsas_add_device(struct mpr_softc *sc, u16 handle, u8 linkrate)
 			goto out;
 		}
 	}
+	mpr_dprint(sc, MPR_MAPPING, "%s: Target ID for added device is %d.\n",
+	    __func__, id);
 
+	/*
+	 * Only do the ID check and reuse check if the target is not from a
+	 * RAID Component. For Physical Disks of a Volume, the ID will be reused
+	 * when a volume is deleted because the mapping entry for the PD will
+	 * still be in the mapping table. The ID check should not be done here
+	 * either since this PD is already being used.
+	 */
+	targ = &sassc->targets[id];
+	if (!(targ->flags & MPR_TARGET_FLAGS_RAID_COMPONENT)) {
 		if (mprsas_check_id(sassc, id) != 0) {
-		device_printf(sc->mpr_dev, "Excluding target id %d\n", id);
+			device_printf(sc->mpr_dev, "Excluding target id %d\n",
+			    id);
 			error = ENXIO;
 			goto out;
 		}
 
-	targ = &sassc->targets[id];
 		if (targ->handle != 0x0) {
-		mpr_dprint(sc, MPR_MAPPING, "Attempting to reuse target id "
+			mpr_dprint(sc, MPR_MAPPING, "Attempting to reuse "
-		    "%d handle 0x%04x\n", id, targ->handle);
+			    "target id %d handle 0x%04x\n", id, targ->handle);
 			error = ENXIO;
 			goto out;
 		}
+	}
 
 	mpr_dprint(sc, MPR_MAPPING, "SAS Address from SAS device page0 = %jx\n",
 	    sas_address);
@@ -1256,14 +1276,16 @@ mprsas_add_pcie_device(struct mpr_softc *sc, u16 handle, u8 linkrate)
 		goto out;
 	}
 
-	id = mpr_mapping_get_sas_id(sc, pcie_wwid, handle);
+	id = mpr_mapping_get_tid(sc, pcie_wwid, handle);
 	if (id == MPR_MAP_BAD_ID) {
-		printf("failure at %s:%d/%s()! Could not get ID for device "
+		mpr_dprint(sc, MPR_ERROR | MPR_INFO, "failure at %s:%d/%s()! "
-		    "with handle 0x%04x\n", __FILE__, __LINE__, __func__,
+		    "Could not get ID for device with handle 0x%04x\n",
-		    handle);
+		    __FILE__, __LINE__, __func__, handle);
 		error = ENXIO;
 		goto out;
 	}
+	mpr_dprint(sc, MPR_MAPPING, "%s: Target ID for added device is %d.\n",
+	    __func__, id);
 
 	if (mprsas_check_id(sassc, id) != 0) {
 		device_printf(sc->mpr_dev, "Excluding target id %d\n", id);
@@ -1356,7 +1378,7 @@ mprsas_volume_add(struct mpr_softc *sc, u16 handle)
 		goto out;
 	}
 
-	id = mpr_mapping_get_raid_id(sc, wwid, handle);
+	id = mpr_mapping_get_raid_tid(sc, wwid, handle);
 	if (id == MPR_MAP_BAD_ID) {
 		printf("%s: could not get ID for volume with handle 0x%04x and "
 		    "WWID 0x%016llx\n", __func__, handle,
@@ -1418,7 +1440,7 @@ mprsas_SSU_to_SATA_devices(struct mpr_softc *sc)
 	 */
 	sc->SSU_started = TRUE;
 	sc->SSU_refcount = 0;
-	for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) {
+	for (targetid = 0; targetid < sc->max_devices; targetid++) {
 		target = &sassc->targets[targetid];
 		if (target->handle == 0x0) {
 			continue;
@@ -1602,7 +1624,7 @@ mprsas_ir_shutdown(struct mpr_softc *sc)
 	 * 3: enable to SSD and HDD
 	 * anything else will default to 1.
 	 */
-	for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) {
+	for (targetid = 0; targetid < sc->max_devices; targetid++) {
 		target = &sc->sassc->targets[targetid];
 		if (target->handle == 0x0) {
 			continue;

sys/dev/mpr/mpr_user.c

@@ -2145,7 +2145,7 @@ mpr_user_btdh(struct mpr_softc *sc, mpr_btdh_mapping_t *data)
 			data->DevHandle = dev_handle;
 	} else {
 		bus = 0;
-		target = mpr_mapping_get_sas_id_from_handle(sc, dev_handle);
+		target = mpr_mapping_get_tid_from_handle(sc, dev_handle);
 		data->Bus = bus;
 		data->TargetID = target;
 	}

sys/dev/mpr/mprvar.h

@@ -33,7 +33,7 @@
 #ifndef _MPRVAR_H
 #define _MPRVAR_H
 
-#define MPR_DRIVER_VERSION	"15.02.00.00-fbsd"
+#define MPR_DRIVER_VERSION	"15.03.00.00-fbsd"
 
 #define MPR_DB_MAX_WAIT		2500
 
@@ -69,6 +69,7 @@
 
 #define MPR_PERIODIC_DELAY	1	/* 1 second heartbeat/watchdog check */
 #define MPR_ATA_ID_TIMEOUT	5	/* 5 second timeout for SATA ID cmd */
+#define MPR_MISSING_CHECK_DELAY	10	/* 10 seconds between missing check */
 
 #define	IFAULT_IOP_OVER_TEMP_THRESHOLD_EXCEEDED	0x2810
 
@@ -88,7 +89,6 @@
 #define MPR_MAX_MISSING_COUNT	0x0F
 #define MPR_DEV_RESERVED	0x20000000
 #define MPR_MAP_IN_USE		0x10000000
-#define MPR_RAID_CHANNEL	1
 #define MPR_MAP_BAD_ID		0xFFFFFFFF
 
 typedef uint8_t u8;
@@ -103,7 +103,6 @@ typedef uint64_t u64;
  * @phy_bits: bitfields indicating controller phys
  * @dpm_entry_num: index of this device in device persistent map table
  * @dev_handle: device handle for the device pointed by this entry
- * @channel: target channel
  * @id: target id
  * @missing_count: number of times the device not detected by driver
  * @hide_flag: Hide this physical disk/not (foreign configuration)
@@ -116,8 +115,7 @@ struct dev_mapping_table {
 	u32	phy_bits;
 	u16	dpm_entry_num;
 	u16	dev_handle;
-	u8	reserved1;
+	u16	reserved1;
-	u8	channel;
 	u16	id;
 	u8	missing_count;
 	u8	init_complete;
@@ -306,6 +304,7 @@ struct mpr_softc {
 	struct mpr_chain		*chains;
 	struct mpr_prp_page		*prps;
 	struct callout			periodic;
+	struct callout			device_check_callout;
 
 	struct mprsas_softc		*sassc;
 	char            tmp_string[MPR_STRING_LENGTH];
@@ -397,13 +396,10 @@ struct mpr_softc {
 	uint8_t				max_volumes;
 	uint8_t				num_enc_table_entries;
 	uint8_t				num_rsvd_entries;
-	uint8_t				num_channels;
 	uint16_t			max_dpm_entries;
 	uint8_t				is_dpm_enable;
 	uint8_t				track_mapping_events;
 	uint32_t			pending_map_events;
-	uint8_t				mt_full_retry;
-	uint8_t				mt_add_device_failed;
 
 	/* FW diag Buffer List */
 	mpr_fw_diagnostic_buffer_t
@@ -774,19 +770,18 @@ void mpr_mapping_topology_change_event(struct mpr_softc *,
     Mpi2EventDataSasTopologyChangeList_t *);
 void mpr_mapping_pcie_topology_change_event(struct mpr_softc *sc,
     Mpi26EventDataPCIeTopologyChangeList_t *event_data);
-int mpr_mapping_is_reinit_required(struct mpr_softc *);
 void mpr_mapping_free_memory(struct mpr_softc *sc);
 int mpr_config_set_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
     Mpi2DriverMappingPage0_t *, u16 );
 void mpr_mapping_exit(struct mpr_softc *);
-void mpr_mapping_check_devices(struct mpr_softc *, int);
+void mpr_mapping_check_devices(void *);
 int mpr_mapping_allocate_memory(struct mpr_softc *sc);
-unsigned int mpr_mapping_get_sas_id(struct mpr_softc *, uint64_t , u16);
+unsigned int mpr_mapping_get_tid(struct mpr_softc *, uint64_t , u16);
-unsigned int mpr_mapping_get_sas_id_from_handle(struct mpr_softc *sc,
+unsigned int mpr_mapping_get_tid_from_handle(struct mpr_softc *sc,
     u16 handle);
-unsigned int mpr_mapping_get_raid_id(struct mpr_softc *sc, u64 wwid,
+unsigned int mpr_mapping_get_raid_tid(struct mpr_softc *sc, u64 wwid,
-    u16 handle);
+    u16 volHandle);
-unsigned int mpr_mapping_get_raid_id_from_handle(struct mpr_softc *sc,
+unsigned int mpr_mapping_get_raid_tid_from_handle(struct mpr_softc *sc,
     u16 volHandle);
 void mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *,
     Mpi2EventDataSasEnclDevStatusChange_t *event_data);