rpc.tlsservd: provide parallelism with help of pthread(3)

At normal NFS server runtime there is not much RPC traffic from kernel to
rpc.tlsservd.  But as Rick rmacklem@ explained, the notion of multiple
workers exists to handle a situation when a server reboots and it has
several hundred or thousands of TLS/TCP connections from clients.  Once it
comes back up, all the clients make TCP connections and do TLS handshakes.

So cleanup the remnants of the workers, that left after the conversion of
RPC over netlink(4) transport and restore desired parallelism with help of
pthread(3).

We are processing the TLS handshakes in separate threads, one per
handshake.  Number of concurrent threads is capped by hw.ncpu / 2, but this
can be overriden with -N.

Differential Revision:	https://reviews.freebsd.org/D48570

sys/rpc/rpcsec_tls.h

@@ -32,9 +32,6 @@
 #define	RPCTLS_SYSC_CLSOCKET	2
 #define	RPCTLS_SYSC_SRVSOCKET	5
 
-/* Max nprocs for SRV startup */
-#define	RPCTLS_SRV_MAXNPROCS	16
-
 /* System call used by the rpctlscd, rpctlssd daemons. */
 int	rpctls_syscall(int, const char *);
 

usr.sbin/rpc.tlsservd/Makefile

@@ -6,7 +6,7 @@ SRCS=	rpc.tlsservd.c rpc.tlscommon.c rpctlssd.h rpctlssd_svc.c rpctlssd_xdr.c
 
 CFLAGS+= -I.
 
-LIBADD=	ssl crypto util
+LIBADD=	ssl crypto util pthread
 
 CLEANFILES= rpctlssd_svc.c rpctlssd_xdr.c rpctlssd.h
 

usr.sbin/rpc.tlsservd/rpc.tlsservd.8

@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" Modified from gssd.8 for rpc.tlsservd.8 by Rick Macklem.
-.Dd November 10, 2022
+.Dd January 25, 2025
 .Dt RPC.TLSSERVD 8
 .Os
 .Sh NAME
@@ -39,7 +39,7 @@
 .Op Fl h
 .Op Fl l Ar CAfile
 .Op Fl m
-.Op Fl N Ar num_servers
+.Op Fl N Ar max_threads
 .Op Fl n Ar domain
 .Op Fl p Ar CApath
 .Op Fl r Ar CRLfile
@@ -236,16 +236,17 @@ may be used to require a client to provide a certificate
 that verifies.
 See
 .Xr exports 5 .
-.It Fl N Ar num_servers , Fl Fl numdaemons= Ns Ar num_servers
+.It Fl N Ar max_threads , Fl Fl maxthreads= Ns Ar max_threads
 For a server with a large number of NFS-over-TLS client mounts,
 this daemon might get overloaded after a reboot, when many
 clients attempt to do a TLS handshake at the same time.
-This option may be used to specify that
-.Dq num_servers
-daemons are to be run instead of a single daemon.
-When this is done, the TLS handshakes are spread across the
-.Dq num_servers
-daemons in a round robin fashion to spread out the load.
+To speed up recovery after reboot, the daemon always processes a TLS handshake
+in a separate spawned thread.
+By default the maximum number of concurrent threads (and thus
+parallel handshakes) is limited to
+.Va 1/2
+of available CPUs on a system.
+This option may be used to override this default.
 .It Fl n Ar domain , Fl Fl domain= Ns Ar domain
 This option specifies what the
 .Dq domain

usr.sbin/rpc.tlsservd/rpc.tlsservd.c

@@ -4,6 +4,7 @@
  * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
  * Authors: Doug Rabson <dfr@rabson.org>
  * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
+ * Copyright (c) 2025 Gleb Smirnoff <glebius@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -32,27 +33,20 @@
  * the server side of kernel RPC-over-TLS by Rick Macklem.
  */
 
-#include <sys/param.h>
 #include <sys/types.h>
 #include <sys/linker.h>
 #include <sys/module.h>
 #include <sys/queue.h>
-#include <sys/stat.h>
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
-#include <sys/time.h>
-#include <sys/wait.h>
+#include <assert.h>
 #include <err.h>
 #include <getopt.h>
 #include <libutil.h>
 #include <netdb.h>
+#include <pthread.h>
 #include <pwd.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
 #include <stdbool.h>
-#include <string.h>
 #include <unistd.h>
 
 #include <rpc/rpc.h>
@@ -86,7 +80,14 @@ const char		*rpctls_verify_cafile = NULL;
 const char		*rpctls_verify_capath = NULL;
 char			*rpctls_crlfile = NULL;
 bool			rpctls_gothup = false;
+
+static SVCXPRT		*xprt;
+static pthread_key_t	xidkey;
 struct ssl_list		rpctls_ssllist;
+static pthread_rwlock_t	rpctls_rwlock;
+static u_int		rpctls_nthreads = 0;
+static pthread_mutex_t	rpctls_mtx;
+static pthread_cond_t	rpctls_cv;
 
 static struct pidfh	*rpctls_pfh = NULL;
 static bool		rpctls_do_mutual = false;
@@ -98,9 +99,7 @@ static char		*rpctls_dnsname;
 static const char	*rpctls_cnuseroid = "1.3.6.1.4.1.2238.1.1.1";
 static const char	*rpctls_ciphers = NULL;
 static int		rpctls_mintls = TLS1_3_VERSION;
-static int		rpctls_procs = 1;
-static pid_t		rpctls_workers[RPCTLS_SRV_MAXNPROCS - 1];
-static bool		rpctls_im_a_worker = false;
+static u_int		rpctls_maxthreads;
 
 static void		rpctls_cleanup_term(int sig);
 static SSL_CTX		*rpctls_setup_ssl(const char *certdir);
@@ -114,6 +113,8 @@ static void		rpctls_huphandler(int sig __unused);
 
 extern void		rpctlssd_2(struct svc_req *rqstp, SVCXPRT *transp);
 
+static void *dummy_thread(void *v __unused) { return (NULL); }
+
 static struct option longopts[] = {
 	{ "allowtls1_2",	no_argument,		NULL,	'2' },
 	{ "ciphers",		required_argument,	NULL,	'C' },
@@ -122,7 +123,7 @@ static struct option longopts[] = {
 	{ "checkhost",		no_argument,		NULL,	'h' },
 	{ "verifylocs",		required_argument,	NULL,	'l' },
 	{ "mutualverf",		no_argument,		NULL,	'm' },
-	{ "numdaemons",		required_argument,	NULL,	'N' },
+	{ "maxthreads",		required_argument,	NULL,	'N' },
 	{ "domain",		required_argument,	NULL,	'n' },
 	{ "verifydir",		required_argument,	NULL,	'p' },
 	{ "crl",		required_argument,	NULL,	'r' },
@@ -136,13 +137,13 @@ static struct option longopts[] = {
 int
 main(int argc, char **argv)
 {
-	int ch, i;
-	SVCXPRT *xprt;
+	int ch;
 	char hostname[MAXHOSTNAMELEN + 2];
 	pid_t otherpid;
+	pthread_t tid;
 	bool tls_enable;
 	size_t tls_enable_len;
-	sigset_t signew;
+	u_int ncpu;
 
 	/* Check that another rpctlssd isn't already running. */
 	rpctls_pfh = pidfile_open(_PATH_RPCTLSSDPID, 0600, &otherpid);
@@ -166,6 +167,8 @@ main(int argc, char **argv)
 	}
 
 	rpctls_verbose = false;
+	rpctls_maxthreads = (ncpu = (u_int)sysconf(_SC_NPROCESSORS_ONLN)) / 2;
+
 	while ((ch = getopt_long(argc, argv, "2C:D:dhl:N:n:mp:r:uvWw", longopts,
 	    NULL)) != -1) {
 		switch (ch) {
@@ -191,11 +194,10 @@ main(int argc, char **argv)
 			rpctls_do_mutual = true;
 			break;
 		case 'N':
-			rpctls_procs = atoi(optarg);
-			if (rpctls_procs < 1 ||
-			    rpctls_procs > RPCTLS_SRV_MAXNPROCS)
-				errx(1, "numdaemons/-N must be between 1 and "
-				    "%d", RPCTLS_SRV_MAXNPROCS);
+			rpctls_maxthreads = atoi(optarg);
+			if (rpctls_maxthreads < 1 || rpctls_maxthreads > ncpu)
+				errx(1, "maximum threads must be between 1 and "
+				    "number of CPUs (%d)", ncpu);
 			break;
 		case 'n':
 			hostname[0] = '@';
@@ -233,7 +235,6 @@ main(int argc, char **argv)
 			    "[-D/--certdir certdir] [-d/--debuglevel] "
 			    "[-h/--checkhost] "
 			    "[-l/--verifylocs CAfile] [-m/--mutualverf] "
-			    "[-N/--numdaemons num] "
 			    "[-n/--domain domain_name] "
 			    "[-p/--verifydir CApath] [-r/--crl CRLfile] "
 			    "[-u/--certuser] [-v/--verbose] [-W/--multiwild] "
@@ -262,53 +263,20 @@ main(int argc, char **argv)
 		if (kldload("krpc") < 0 || modfind("krpc") < 0)
 			errx(1, "Kernel RPC is not available");
 	}
-
-	for (i = 0; i < rpctls_procs - 1; i++)
-		rpctls_workers[i] = -1;
-
-	if (rpctls_debug_level == 0) {
-		/*
-		 * Temporarily block SIGTERM and SIGCHLD, so workers[] can't
-		 * end up bogus.
-		 */
-		sigemptyset(&signew);
-		sigaddset(&signew, SIGTERM);
-		sigaddset(&signew, SIGCHLD);
-		sigprocmask(SIG_BLOCK, &signew, NULL);
-
-		if (daemon(0, 0) != 0)
-			err(1, "Can't daemonize");
-		signal(SIGINT, SIG_IGN);
-		signal(SIGQUIT, SIG_IGN);
-	}
 	signal(SIGPIPE, SIG_IGN);
 	signal(SIGHUP, rpctls_huphandler);
 	signal(SIGTERM, rpctls_cleanup_term);
-	signal(SIGCHLD, rpctls_cleanup_term);
 
+	if (rpctls_debug_level == 0 && daemon(0, 0) != 0)
+		err(1, "Can't daemonize");
 	pidfile_write(rpctls_pfh);
 
-	if (rpctls_debug_level == 0) {
-		/* Fork off the worker daemons. */
-		for (i = 0; i < rpctls_procs - 1; i++) {
-			rpctls_workers[i] = fork();
-			if (rpctls_workers[i] == 0) {
-				rpctls_im_a_worker = true;
-				setproctitle("server");
-				break;
-			} else if (rpctls_workers[i] < 0) {
-				syslog(LOG_ERR, "fork: %m");
-			}
-		}
-
-		if (!rpctls_im_a_worker && rpctls_procs > 1)
-			setproctitle("master");
-	}
-	sigemptyset(&signew);
-	sigaddset(&signew, SIGTERM);
-	sigaddset(&signew, SIGCHLD);
-	sigprocmask(SIG_UNBLOCK, &signew, NULL);
-
+	/*
+	 * XXX: Push libc internal state into threaded mode before creating
+	 * the threaded svc_nl xprt.
+	 */
+	(void)pthread_create(&tid, NULL, dummy_thread, NULL);
+	(void)pthread_join(tid, NULL);
 	if ((xprt = svc_nl_create("tlsserv")) == NULL) {
 		if (rpctls_debug_level == 0) {
 			syslog(LOG_ERR,
@@ -317,6 +285,8 @@ main(int argc, char **argv)
 		}
 		err(1, "Can't create transport for local rpctlssd socket");
 	}
+	if (!SVC_CONTROL(xprt, SVCNL_GET_XIDKEY, &xidkey))
+		err(1, "Failed to obtain pthread key for xid from svc_nl");
 	if (!svc_reg(xprt, RPCTLSSD, RPCTLSSDVERS, rpctlssd_2, NULL)) {
 		if (rpctls_debug_level == 0) {
 			syslog(LOG_ERR,
@@ -335,6 +305,9 @@ main(int argc, char **argv)
 		err(1, "Can't create SSL context");
 	}
 	rpctls_gothup = false;
+	pthread_rwlock_init(&rpctls_rwlock, NULL);
+	pthread_mutex_init(&rpctls_mtx, NULL);
+	pthread_cond_init(&rpctls_cv, NULL);
 	LIST_INIT(&rpctls_ssllist);
 
 	rpctls_svc_run();
@@ -352,24 +325,90 @@ rpctlssd_null_2_svc(__unused void *argp, __unused void *result,
 	return (TRUE);
 }
 
+/*
+ * To parallelize SSL handshakes we will launch a thread per handshake.  Thread
+ * creation/destruction shall be order(s) of magnitude cheaper than a crypto
+ * handshake, so we are not keeping a pool of workers here.
+ *
+ * Marrying rpc(3) and pthread(3):
+ *
+ * Normally the rpcgen(1) generated rpctlssd_V() calls rpctlssd_connect_V_svc(),
+ * and the latter processes the RPC all the way to the end and returns a TRUE
+ * value and populates the result.  The generated code immediately calls
+ * svc_sendreply() transmitting the result back.
+ *
+ * We will make a private copy of arguments and return FALSE.  Then it is our
+ * obligation to call svc_sendreply() once we do the work in the thread.
+ */
+
+static void * rpctlssd_connect_thread(void *);
+struct rpctlssd_connect_thread_ctx {
+	struct rpctlssd_connect_arg arg;
+	uint32_t xid;
+};
+
 bool_t
 rpctlssd_connect_2_svc(struct rpctlssd_connect_arg *argp,
-    struct rpctlssd_connect_res *result, __unused struct svc_req *rqstp)
+    struct rpctlssd_connect_res *result __unused, struct svc_req *rqstp)
 {
+	struct rpctlssd_connect_thread_ctx *ctx;
+	pthread_t tid;
+
+	assert(rqstp->rq_xprt == xprt);
+
+	ctx = malloc(sizeof(*ctx));
+	memcpy(&ctx->arg, argp, sizeof(ctx->arg));
+	ctx->xid = *(uint32_t *)pthread_getspecific(xidkey);
+
+	pthread_mutex_lock(&rpctls_mtx);
+	while (rpctls_nthreads >= rpctls_maxthreads)
+		pthread_cond_wait(&rpctls_cv, &rpctls_mtx);
+	rpctls_nthreads++;
+	pthread_mutex_unlock(&rpctls_mtx);
+
+	rpctls_verbose_out("rpctlsd_connect_svc: xid %u thread %u\n",
+	    ctx->xid, rpctls_nthreads);
+
+	if (pthread_create(&tid, NULL, rpctlssd_connect_thread, ctx) != 0)
+		warn("failed to start handshake thread");
+
+	/* Intentionally, so that RPC generated code doesn't try to reply. */
+	return (FALSE);
+}
+
+static void *
+rpctlssd_connect_thread(void *v)
+{
+	struct rpctlssd_connect_thread_ctx *ctx = v;
+	struct rpctlssd_connect_res result;
+	uint64_t socookie;
 	int ngrps, s;
 	SSL *ssl;
 	uint32_t flags;
 	struct ssl_entry *newslp;
-	uint32_t uid;
+	uint32_t xid, uid;
 	uint32_t *gidp;
 	X509 *cert;
 
-	rpctls_verbose_out("rpctlsd_connect_svc: started\n");
-	memset(result, 0, sizeof(*result));
+	socookie = ctx->arg.socookie;
+	xid = ctx->xid;
+	free(ctx);
+	ctx = NULL;
+	pthread_detach(pthread_self());
+
+	if (pthread_setspecific(xidkey, &xid) != 0) {
+		rpctls_verbose_out("rpctlssd_connect_svc: pthread_setspecific "
+		    "failed\n");
+		goto out;
+	}
+
 	/* Get the socket fd from the kernel. */
-	s = rpctls_syscall(RPCTLS_SYSC_SRVSOCKET, (char *)argp->socookie);
-	if (s < 0)
-		return (FALSE);
+	s = rpctls_syscall(RPCTLS_SYSC_SRVSOCKET, (char *)socookie);
+	if (s < 0) {
+		rpctls_verbose_out("rpctlssd_connect_svc: rpctls_syscall "
+		    "accept failed\n");
+		goto out;
+	}
 
 	/* Do the server side of a TLS handshake. */
 	gidp = calloc(NGROUPS, sizeof(*gidp));
@@ -383,20 +422,24 @@ rpctlssd_connect_2_svc(struct rpctlssd_connect_arg *argp,
 		 * to close off the socket upon handshake failure.
 		 */
 		close(s);
-		return (FALSE);
+		goto out;
 	} else {
 		rpctls_verbose_out("rpctlssd_connect_svc: "
 		    "succeeded flags=0x%x\n", flags);
-		result->flags = flags;
-		if ((flags & RPCTLS_FLAGS_CERTUSER) != 0) {
-			result->uid = uid;
-			result->gid.gid_len = ngrps;
-			result->gid.gid_val = gidp;
-		} else {
-			result->uid = 0;
-			result->gid.gid_len = 0;
-			result->gid.gid_val = gidp;
-		}
+		if ((flags & RPCTLS_FLAGS_CERTUSER) != 0)
+			result = (struct rpctlssd_connect_res){
+				.flags = flags,
+				.uid = uid,
+				.gid.gid_len = ngrps,
+				.gid.gid_val = gidp,
+			};
+		else
+			result = (struct rpctlssd_connect_res){
+				.flags = flags,
+				.uid = 0,
+				.gid.gid_len = 0,
+				.gid.gid_val = gidp,
+			};
 	}
 
 	/* Maintain list of all current SSL *'s */
@@ -404,10 +447,27 @@ rpctlssd_connect_2_svc(struct rpctlssd_connect_arg *argp,
 	newslp->ssl = ssl;
 	newslp->s = s;
 	newslp->shutoff = false;
-	newslp->cookie = argp->socookie;
+	newslp->cookie = socookie;
 	newslp->cert = cert;
+	pthread_rwlock_wrlock(&rpctls_rwlock);
 	LIST_INSERT_HEAD(&rpctls_ssllist, newslp, next);
-	return (TRUE);
+	pthread_rwlock_unlock(&rpctls_rwlock);
+
+	if (!svc_sendreply(xprt, (xdrproc_t)xdr_rpctlssd_connect_res, &result))
+		svcerr_systemerr(xprt);
+
+	free(result.gid.gid_val);
+	rpctls_verbose_out("rpctlsd_connect_svc: xid %u: thread finished\n",
+	    xid);
+
+out:
+	pthread_mutex_lock(&rpctls_mtx);
+	if (rpctls_nthreads-- >= rpctls_maxthreads) {
+		pthread_mutex_unlock(&rpctls_mtx);
+		pthread_cond_signal(&rpctls_cv);
+	} else
+		pthread_mutex_unlock(&rpctls_mtx);
+	return (NULL);
 }
 
 bool_t
@@ -418,9 +478,11 @@ rpctlssd_handlerecord_2_svc(struct rpctlssd_handlerecord_arg *argp,
 	int ret;
 	char junk;
 
+	pthread_rwlock_rdlock(&rpctls_rwlock);
 	LIST_FOREACH(slp, &rpctls_ssllist, next)
 		if (slp->cookie == argp->socookie)
 			break;
+	pthread_rwlock_unlock(&rpctls_rwlock);
 
 	if (slp != NULL) {
 		rpctls_verbose_out("rpctlssd_handlerecord fd=%d\n",
@@ -455,14 +517,17 @@ rpctlssd_disconnect_2_svc(struct rpctlssd_disconnect_arg *argp,
 	struct ssl_entry *slp;
 	int ret;
 
+	pthread_rwlock_wrlock(&rpctls_rwlock);
 	LIST_FOREACH(slp, &rpctls_ssllist, next)
-		if (slp->cookie == argp->socookie)
+		if (slp->cookie == argp->socookie) {
+			LIST_REMOVE(slp, next);
 			break;
+		}
+	pthread_rwlock_unlock(&rpctls_rwlock);
 
 	if (slp != NULL) {
 		rpctls_verbose_out("rpctlssd_disconnect fd=%d closed\n",
 		    slp->s);
-		LIST_REMOVE(slp, next);
 		if (!slp->shutoff) {
 			ret = SSL_get_shutdown(slp->ssl);
 			/*
@@ -490,15 +555,9 @@ rpctlssd_disconnect_2_svc(struct rpctlssd_disconnect_arg *argp,
 }
 
 int
-rpctlssd_2_freeresult(__unused SVCXPRT *transp, xdrproc_t xdr_result,
-    caddr_t result)
+rpctlssd_2_freeresult(__unused SVCXPRT *transp, xdrproc_t xdr_result __unused,
+    caddr_t result __unused)
 {
-	rpctlssd_connect_res *res;
-
-	if (xdr_result == (xdrproc_t)xdr_rpctlssd_connect_res) {
-		res = (rpctlssd_connect_res *)(void *)result;
-		free(res->gid.gid_val);
-	}
 	return (TRUE);
 }
 
@@ -506,36 +565,14 @@ rpctlssd_2_freeresult(__unused SVCXPRT *transp, xdrproc_t xdr_result,
  * cleanup_term() called via SIGTERM (or SIGCHLD if a child dies).
  */
 static void
-rpctls_cleanup_term(int sig)
+rpctls_cleanup_term(int sig __unused)
 {
 	struct ssl_entry *slp;
-	int i, cnt;
 
-	if (rpctls_im_a_worker && sig == SIGCHLD)
-		return;
 	LIST_FOREACH(slp, &rpctls_ssllist, next)
 		shutdown(slp->s, SHUT_RD);
 	SSL_CTX_free(rpctls_ctx);
 	EVP_cleanup();
-
-	if (rpctls_im_a_worker)
-		exit(0);
-
-	/* I'm the server, so terminate the workers. */
-	cnt = 0;
-	for (i = 0; i < rpctls_procs - 1; i++) {
-		if (rpctls_workers[i] != -1) {
-			cnt++;
-			kill(rpctls_workers[i], SIGTERM);
-		}
-	}
-
-	/*
-	 * Wait for them to die.
-	 */
-	for (i = 0; i < cnt; i++)
-		wait3(NULL, 0, NULL);
-
 	pidfile_remove(rpctls_pfh);
 
 	exit(0);