ctlstat: add prometheus output

When invoked by inetd, ctlstat -P will now produce output suitable for
ingestion into Prometheus.

It's a drop-in replacement for https://github.com/Gandi/ctld_exporter,
except that it doesn't report the number of initiators per target, and
it does report time and dma_time.

MFC after:	2 weeks
Sponsored by:	Axcient
Relnotes:	yes
Reviewed by: 	bapt, bcr
Differential Revision: https://reviews.freebsd.org/D29901

usr.bin/ctlstat/Makefile

@@ -5,4 +5,6 @@ MAN=		ctlstat.8
 SDIR=		${SRCTOP}/sys
 CFLAGS+=	-I${SDIR}
 
+LIBADD=		sbuf bsdxml
+
 .include <bsd.prog.mk>

usr.bin/ctlstat/ctlstat.8

@@ -34,7 +34,7 @@
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
 .\" $FreeBSD$
 .\"
-.Dd January 9, 2017
+.Dd April 22, 2021
 .Dt CTLSTAT 8
 .Os
 .Sh NAME
@@ -48,6 +48,7 @@
 .Op Fl d
 .Op Fl D
 .Op Fl j
+.Op Fl P
 .Op Fl l Ar lun
 .Op Fl n Ar numdevs
 .Op Fl p Ar port
@@ -83,6 +84,19 @@ Suppress display of the header.
 JSON dump mode.
 Dump statistics every 30 seconds in JavaScript Object Notation (JSON) format.
 No statistics are computed in this mode, only raw numbers are displayed.
+.It Fl P
+Prometheus dump mode.
+Dump statistics in a format suitable for ingestion into Prometheus.
+When invoked with this option,
+.Nm
+dumps once, regardless of the
+.Fl t
+option.
+This option is especially useful when invoked by
+.Xr inetd 8 .
+See the comments in
+.Pa /etc/inetd.conf
+for an example configuration.
 .It Fl l Ar lun
 Request statistics for the specified LUN.
 .It Fl n Ar numdevs
@@ -116,7 +130,13 @@ every 10 seconds.
 .Xr camcontrol 8 ,
 .Xr ctladm 8 ,
 .Xr ctld 8 ,
-.Xr iostat 8
+.Xr iostat 8 ,
+.Lk
+Prometheus project:
+.Pa https://prometheus.io/ .
+.Pp
+Prometheus exposition formats:
+.Lk https://prometheus.io/docs/instrumenting/exposition_formats/ .
 .Sh AUTHORS
 .An Ken Merry Aq Mt ken@FreeBSD.org
 .An Will Andrews Aq Mt will@FreeBSD.org

usr.bin/ctlstat/ctlstat.c

@@ -41,19 +41,23 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/ioctl.h>
-#include <sys/types.h>
 #include <sys/param.h>
-#include <sys/time.h>
-#include <sys/sysctl.h>
-#include <sys/resource.h>
-#include <sys/queue.h>
 #include <sys/callout.h>
+#include <sys/ioctl.h>
+#include <sys/queue.h>
+#include <sys/resource.h>
+#include <sys/sbuf.h>
+#include <sys/socket.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include <bsdxml.h>
+#include <malloc_np.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
+#include <inttypes.h>
 #include <getopt.h>
 #include <string.h>
 #include <errno.h>
@@ -76,8 +80,8 @@ __FBSDID("$FreeBSD$");
 
 static int ctl_stat_bits;
 
-static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
-static const char *ctlstat_usage = "Usage:  ctlstat [-CDdjht] [-l lunnum]"
+static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:";
+static const char *ctlstat_usage = "Usage:  ctlstat [-CDPdjht] [-l lunnum]"
 				   "[-c count] [-n numdevs] [-w wait]\n";
 
 struct ctl_cpu_stats {
@@ -92,6 +96,7 @@ typedef enum {
 	CTLSTAT_MODE_STANDARD,
 	CTLSTAT_MODE_DUMP,
 	CTLSTAT_MODE_JSON,
+	CTLSTAT_MODE_PROMETHEUS,
 } ctlstat_mode_types;
 
 #define	CTLSTAT_FLAG_CPU		(1 << 0)
@@ -129,6 +134,15 @@ struct ctlstat_context {
 	int header_interval;
 };
 
+struct cctl_portlist_data {
+	int level;
+	struct sbuf *cur_sb[32];
+	int lun;
+	int ntargets;
+	char *target;
+	char **targets;
+};
+
 #ifndef min
 #define	min(x,y)	(((x) < (y)) ? (x) : (y))
 #endif
@@ -381,6 +395,200 @@ ctlstat_json(struct ctlstat_context *ctx) {
 	printf("]}");
 }
 
+#define CTLSTAT_PROMETHEUS_LOOP(field) \
+	for (i = n = 0; i < ctx->cur_items; i++) { \
+		if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+		    (int)stats[i].item) == 0) \
+			continue; \
+		for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+			int lun = stats[i].item; \
+			if (lun >= targdata.ntargets) \
+			    errx(1, "LUN %u out of range", lun); \
+			printf("iscsi_target_" #field "{" \
+			    "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+			    "\n", \
+			    lun, targdata.targets[lun], iotypes[iotype], \
+			    stats[i].field[iotype]); \
+		} \
+	} \
+
+#define CTLSTAT_PROMETHEUS_TIMELOOP(field) \
+	for (i = n = 0; i < ctx->cur_items; i++) { \
+		if (F_MASK(ctx) && bit_test(ctx->item_mask, \
+		    (int)stats[i].item) == 0) \
+			continue; \
+		for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \
+			uint64_t us; \
+			struct timespec ts; \
+			int lun = stats[i].item; \
+			if (lun >= targdata.ntargets) \
+			    errx(1, "LUN %u out of range", lun); \
+			bintime2timespec(&stats[i].field[iotype], &ts); \
+			us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \
+			printf("iscsi_target_" #field "{" \
+			    "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \
+			    "\n", \
+			    lun, targdata.targets[lun], iotypes[iotype], us); \
+		} \
+	} \
+
+static void
+cctl_start_pelement(void *user_data, const char *name, const char **attr __unused)
+{
+	struct cctl_portlist_data* targdata = user_data;
+
+	targdata->level++;
+	if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) /
+	    sizeof(targdata->cur_sb[0])))
+		errx(1, "%s: too many nesting levels, %zd max", __func__,
+		     sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0]));
+
+	targdata->cur_sb[targdata->level] = sbuf_new_auto();
+	if (targdata->cur_sb[targdata->level] == NULL)
+		err(1, "%s: Unable to allocate sbuf", __func__);
+
+	if (strcmp(name, "targ_port") == 0) {
+		targdata->lun = -1;
+		free(targdata->target);
+		targdata->target = NULL;
+	}
+}
+
+static void
+cctl_char_phandler(void *user_data, const XML_Char *str, int len)
+{
+	struct cctl_portlist_data *targdata = user_data;
+
+	sbuf_bcat(targdata->cur_sb[targdata->level], str, len);
+}
+
+static void
+cctl_end_pelement(void *user_data, const char *name)
+{
+	struct cctl_portlist_data* targdata = user_data;
+	char *str;
+
+	if (targdata->cur_sb[targdata->level] == NULL)
+		errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
+		     targdata->level, name);
+
+	if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0)
+		err(1, "%s: sbuf_finish", __func__);
+	str = strdup(sbuf_data(targdata->cur_sb[targdata->level]));
+	if (str == NULL)
+		err(1, "%s can't allocate %zd bytes for string", __func__,
+		    sbuf_len(targdata->cur_sb[targdata->level]));
+
+	sbuf_delete(targdata->cur_sb[targdata->level]);
+	targdata->cur_sb[targdata->level] = NULL;
+	targdata->level--;
+
+	if (strcmp(name, "target") == 0) {
+		free(targdata->target);
+		targdata->target = str;
+	} else if (strcmp(name, "lun") == 0) {
+		targdata->lun = atoi(str);
+		free(str);
+	} else if (strcmp(name, "targ_port") == 0) {
+		if (targdata->lun >= 0 && targdata->target != NULL) {
+			if (targdata->lun >= targdata->ntargets) {
+				/*
+				 * This can happen for example if there are
+				 * holes in CTL's lunlist.
+				 */
+				targdata->ntargets = MAX(targdata->ntargets * 2,
+					targdata->lun + 1);
+				size_t newsize = targdata->ntargets *
+					sizeof(char*);
+				targdata->targets = rallocx(targdata->targets,
+					newsize, MALLOCX_ZERO);
+			}
+			free(targdata->targets[targdata->lun]);
+			targdata->targets[targdata->lun] = targdata->target;
+			targdata->target = NULL;
+		}
+		free(str);
+	} else {
+		free(str);
+	}
+}
+
+static void
+ctlstat_prometheus(int fd, struct ctlstat_context *ctx) {
+	struct ctl_io_stats *stats = ctx->cur_stats;
+	struct ctl_lun_list list;
+	struct cctl_portlist_data targdata;
+	XML_Parser parser;
+	char *port_str = NULL;
+	int iotype, i, n, retval;
+	int port_len = 4096;
+
+	bzero(&targdata, sizeof(targdata));
+	targdata.ntargets = ctx->cur_items;
+	targdata.targets = calloc(targdata.ntargets, sizeof(char*));
+retry:
+	port_str = (char *)realloc(port_str, port_len);
+	bzero(&list, sizeof(list));
+	list.alloc_len = port_len;
+	list.status = CTL_LUN_LIST_NONE;
+	list.lun_xml = port_str;
+	if (ioctl(fd, CTL_PORT_LIST, &list) == -1)
+		err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__);
+	if (list.status == CTL_LUN_LIST_ERROR) {
+		warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s",
+		      __func__, list.error_str);
+	} else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
+		port_len <<= 1;
+		goto retry;
+	}
+
+	parser = XML_ParserCreate(NULL);
+	if (parser == NULL)
+		err(1, "%s: Unable to create XML parser", __func__);
+	XML_SetUserData(parser, &targdata);
+	XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
+	XML_SetCharacterDataHandler(parser, cctl_char_phandler);
+
+	retval = XML_Parse(parser, port_str, strlen(port_str), 1);
+	if (retval != 1) {
+		errx(1, "%s: Unable to parse XML: Error %d", __func__,
+		    XML_GetErrorCode(parser));
+	}
+	XML_ParserFree(parser);
+
+	/*
+	 * NB: Some clients will print a warning if we don't set Content-Length,
+	 * but they still work.  And the data still gets into Prometheus.
+	 */
+	printf("HTTP/1.1 200 OK\r\n"
+	       "Connection: close\r\n"
+	       "Content-Type: text/plain; version=0.0.4\r\n"
+	       "\r\n");
+
+	printf("# HELP iscsi_target_bytes Number of bytes\n"
+	       "# TYPE iscsi_target_bytes counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(bytes);
+	printf("# HELP iscsi_target_dmas Number of DMA\n"
+	       "# TYPE iscsi_target_dmas counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(dmas);
+	printf("# HELP iscsi_target_operations Number of operations\n"
+	       "# TYPE iscsi_target_operations counter\n");
+	CTLSTAT_PROMETHEUS_LOOP(operations);
+	printf("# HELP iscsi_target_time Cumulative operation time in us\n"
+	       "# TYPE iscsi_target_time counter\n");
+	CTLSTAT_PROMETHEUS_TIMELOOP(time);
+	printf("# HELP iscsi_target_dma_time Cumulative DMA time in us\n"
+	       "# TYPE iscsi_target_dma_time counter\n");
+	CTLSTAT_PROMETHEUS_TIMELOOP(dma_time);
+
+	for (i = 0; i < targdata.ntargets; i++)
+		free(targdata.targets[i]);
+	free(targdata.target);
+	free(targdata.targets);
+
+	fflush(stdout);
+}
+
 static void
 ctlstat_standard(struct ctlstat_context *ctx) {
 	long double etime;
@@ -659,6 +867,9 @@ main(int argc, char **argv)
 			ctx.flags |= CTLSTAT_FLAG_PORTS;
 			break;
 		}
+		case 'P':
+			ctx.mode = CTLSTAT_MODE_PROMETHEUS;
+			break;
 		case 't':
 			ctx.flags |= CTLSTAT_FLAG_TOTALS;
 			break;
@@ -676,6 +887,17 @@ main(int argc, char **argv)
 	if (F_LUNS(&ctx) && F_PORTS(&ctx))
 		errx(1, "Options -p and -l are exclusive.");
 
+	if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) {
+		if ((count != -1) ||
+			(waittime != 1) ||
+			/* NB: -P could be compatible with -t in the future */
+			(ctx.flags & CTLSTAT_FLAG_TOTALS))
+		{
+			errx(1, "Option -P is exclusive with -c, -w, and -t");
+		}
+		count = 1;
+	}
+
 	if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) {
 		if (F_TOTALS(&ctx))
 			ctx.flags |= CTLSTAT_FLAG_PORTS;
@@ -712,6 +934,9 @@ main(int argc, char **argv)
 		case CTLSTAT_MODE_JSON:
 			ctlstat_json(&ctx);
 			break;
+		case CTLSTAT_MODE_PROMETHEUS:
+			ctlstat_prometheus(fd, &ctx);
+			break;
 		default:
 			break;
 		}

usr.sbin/inetd/inetd.conf

@@ -120,6 +120,9 @@
 #
 #prom-sysctl	stream	tcp	nowait	nobody	/usr/sbin/prometheus_sysctl_exporter	prometheus_sysctl_exporter -dgh
 #
+# Example entry for the CTL exporter
+#prom-ctl       stream  tcp     nowait  root /usr/bin/ctlstat ctlstat -P
+#
 # Example entry for insecure rsync server
 # This is best combined with encrypted virtual tunnel interfaces, which can be
 # found with: apropos if_ | grep tunnel

usr.sbin/services_mkdb/services

@@ -2000,6 +2000,7 @@ bacula-sd	9103/udp   #Bacula Storage Daemon
 prom-sysctl	9124/tcp   #prometheus_sysctl_exporter(8)
 git		9418/tcp   #git pack transfer service
 git		9418/udp   #git pack transfer service
+prom-ctl	9572/tcp   #CTL prometheus
 odbcpathway	9628/tcp   #ODBC Pathway Service
 odbcpathway	9628/udp   #ODBC Pathway Service
 davsrc		9800/tcp   #WebDav Source Port