sendfile: Fix bug when using headers with SW KTLS offload

When using SW KTLS, we must account for the headers in sf_iodone()
in terms of either freeing or enqueuing them for TLS work.
Not doing so can lead to a situation where we enqueue only
the payload, and not the header, for encryption. Rather than
leaking the header, the socket is left "hung" with the header marked
M_NOTREADY.

Sponsored by: Netflix
Reviewed by: glebius, kib
Differential Revision: https://reviews.freebsd.org/D57134
MFC After: 14 days

sys/kern/kern_sendfile.c

@@ -95,6 +95,8 @@ struct sf_io {
 	vm_pindex_t	pindex0;
 #ifdef KERN_TLS
 	struct ktls_session *tls;
+	struct mbuf	*tls_m;
+	int		tls_enq_cnt;
 #endif
 	vm_page_t	pa[];
 };
@@ -338,6 +340,11 @@ sendfile_iodone(void *arg, vm_page_t *pa, int count, int error)
 		so->so_proto->pr_abort(so);
 		so->so_error = EIO;
 
+#ifdef KERN_TLS
+		if (sfio->tls_m != NULL)
+			mb_free_notready(sfio->tls_m, sfio->tls_enq_cnt);
+		else
+#endif
 			mb_free_notready(sfio->m, sfio->npages);
 #ifdef KERN_TLS
 	} else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) {
@@ -350,6 +357,9 @@ sendfile_iodone(void *arg, vm_page_t *pa, int count, int error)
 		 * Donate the socket reference from sfio to rather
 		 * than explicitly invoking soref().
 		 */
+		if (sfio->tls_m != NULL)
+			ktls_enqueue(sfio->tls_m, so, sfio->tls_enq_cnt);
+		else
 			ktls_enqueue(sfio->m, so, sfio->npages);
 		goto out_with_ref;
 #endif
@@ -897,6 +907,8 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
 		 * for all of sfio's lifetime.
 		 */
 		sfio->tls = tls;
+		sfio->tls_m = NULL;
+		sfio->tls_enq_cnt = 0;
 #endif
 		vm_object_pip_add(obj, 1);
 		error = sendfile_swapin(obj, sfio, &nios, off, space, rhpages,
@@ -1125,6 +1137,13 @@ vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
 		} else {
 			sfio->so = so;
 			sfio->m = m0;
+#ifdef KERN_TLS
+			if (hdrlen != 0 && tls != NULL &&
+			    tls->mode == TCP_TLS_MODE_SW) {
+				sfio->tls_m = m;
+				sfio->tls_enq_cnt = tls_enq_cnt;
+			}
+#endif
 			soref(so);
 			error = pr->pr_send(so, PRUS_NOTREADY, m, NULL, NULL,
 			    td);