ossl: Add GCM support on powerpc64/powerpc64le (POWER8+)

Separate ossl's existing AES-NI GCM implementation into a common
ossl_aes_gcm.c and add conditionals to switch between OpenSSL's AES-NI
and POWER8 GCM routines depending on the architecture. Since the
existing AVX-512 implementation is less agnostic, move it into a
separate ossl_aes_gcm_avx512.c.

Reviewed by:	markj
MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D44274

sys/conf/files.amd64

@@ -107,7 +107,8 @@ crypto/openssl/amd64/poly1305-x86_64.S	optional ossl
 crypto/openssl/amd64/sha1-x86_64.S	optional ossl
 crypto/openssl/amd64/sha256-x86_64.S	optional ossl
 crypto/openssl/amd64/sha512-x86_64.S	optional ossl
-crypto/openssl/amd64/ossl_aes_gcm.c	optional ossl
+crypto/openssl/amd64/ossl_aes_gcm_avx512.c	optional ossl
+crypto/openssl/ossl_aes_gcm.c	optional ossl
 dev/amdgpio/amdgpio.c		optional	amdgpio
 dev/axgbe/if_axgbe_pci.c	optional	axp
 dev/axgbe/xgbe-desc.c		optional	axp

sys/conf/files.arm

@@ -132,7 +132,7 @@ libkern/udivdi3.c		standard
 libkern/umoddi3.c		standard
 
 crypto/openssl/ossl_arm.c		optional ossl
-crypto/openssl/arm/ossl_aes_gcm.c	optional ossl
+crypto/openssl/arm/ossl_aes_gcm_neon.c	optional ossl
 crypto/openssl/arm/aes-armv4.S		optional ossl	\
 	compile-with	"${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl"
 crypto/openssl/arm/bsaes-armv7.S	optional ossl	\

sys/conf/files.powerpc

@@ -31,9 +31,11 @@ zfs-sha512-ppc.o		optional	zfs			\
 
 # openssl ppc common files
 crypto/openssl/ossl_ppc.c	optional	ossl powerpc64 | ossl powerpc64le
+crypto/openssl/ossl_aes_gcm.c	optional	ossl powerpc64 | ossl powerpc64le
 
 # openssl assembly files (powerpc64le)
 crypto/openssl/powerpc64le/aes-ppc.S			optional	ossl powerpc64le
+crypto/openssl/powerpc64le/aes-gcm-ppc.S			optional	ossl powerpc64le
 crypto/openssl/powerpc64le/aesp8-ppc.S			optional	ossl powerpc64le
 crypto/openssl/powerpc64le/chacha-ppc.S			optional	ossl powerpc64le
 crypto/openssl/powerpc64le/ecp_nistz256-ppc64.S		optional	ossl powerpc64le
@@ -54,6 +56,7 @@ crypto/openssl/powerpc64le/x25519-ppc64.S		optional	ossl powerpc64le
 
 # openssl assembly files (powerpc64)
 crypto/openssl/powerpc64/aes-ppc.S			optional	ossl powerpc64
+crypto/openssl/powerpc64/aes-gcm-ppc.S			optional	ossl powerpc64
 crypto/openssl/powerpc64/aesp8-ppc.S			optional	ossl powerpc64
 crypto/openssl/powerpc64/chacha-ppc.S			optional	ossl powerpc64
 crypto/openssl/powerpc64/ecp_nistz256-ppc64.S		optional	ossl powerpc64

sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c

@@ -0,0 +1,232 @@
+/*
+ * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright (c) 2021, Intel Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/*
+ * This file contains an AES-GCM wrapper implementation from OpenSSL, using
+ * VAES extensions. It was ported from cipher_aes_gcm_hw_vaes_avx512.inc.
+ */
+
+#include <sys/endian.h>
+#include <sys/systm.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_aes_gcm.h>
+#include <crypto/openssl/ossl_cipher.h>
+
+#include <opencrypto/cryptodev.h>
+
+_Static_assert(
+    sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context),
+    "ossl_gcm_context too large");
+
+void aesni_set_encrypt_key(const void *key, int bits, void *ctx);
+
+static void
+gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
+{
+	KASSERT(keylen == 128 || keylen == 192 || keylen == 256,
+	    ("%s: invalid key length %zu", __func__, keylen));
+
+	memset(&ctx->gcm, 0, sizeof(ctx->gcm));
+	memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks));
+	aesni_set_encrypt_key(key, keylen, &ctx->aes_ks);
+	ctx->ops->init(ctx, key, keylen);
+}
+
+static void
+gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len)
+{
+	(void)ctx->ops->finish(ctx, NULL, 0);
+	memcpy(tag, ctx->gcm.Xi.c, len);
+}
+
+void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx);
+void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx);
+void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx,
+    const unsigned char *iv, size_t ivlen);
+void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad,
+    size_t len);
+void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx,
+    unsigned int *pblocklen, const unsigned char *in, size_t len,
+    unsigned char *out);
+void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx,
+    unsigned int *pblocklen, const unsigned char *in, size_t len,
+    unsigned char *out);
+void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen);
+
+static void
+gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
+{
+	ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm);
+}
+
+static void
+gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv,
+    size_t len)
+{
+	KASSERT(len == AES_GCM_IV_LEN,
+	    ("%s: invalid IV length %zu", __func__, len));
+
+	ctx->gcm.Yi.u[0] = 0;		/* Current counter */
+	ctx->gcm.Yi.u[1] = 0;
+	ctx->gcm.Xi.u[0] = 0;		/* AAD hash */
+	ctx->gcm.Xi.u[1] = 0;
+	ctx->gcm.len.u[0] = 0;		/* AAD length */
+	ctx->gcm.len.u[1] = 0;		/* Message length */
+	ctx->gcm.ares = 0;
+	ctx->gcm.mres = 0;
+
+	ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len);
+}
+
+static int
+gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad,
+    size_t len)
+{
+	uint64_t alen = ctx->gcm.len.u[0];
+	size_t lenblks;
+	unsigned int ares;
+
+	/* Bad sequence: call of AAD update after message processing */
+	if (ctx->gcm.len.u[1])
+		return -2;
+
+	alen += len;
+	/* AAD is limited by 2^64 bits, thus 2^61 bytes */
+	if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len))
+		return -1;
+	ctx->gcm.len.u[0] = alen;
+
+	ares = ctx->gcm.ares;
+	/* Partial AAD block left from previous AAD update calls */
+	if (ares > 0) {
+		/*
+		 * Fill partial block buffer till full block
+		 * (note, the hash is stored reflected)
+		 */
+		while (ares > 0 && len > 0) {
+			ctx->gcm.Xi.c[15 - ares] ^= *(aad++);
+			--len;
+			ares = (ares + 1) % AES_BLOCK_LEN;
+		}
+		/* Full block gathered */
+		if (ares == 0) {
+			ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx);
+		} else { /* no more AAD */
+			ctx->gcm.ares = ares;
+			return 0;
+		}
+	}
+
+	/* Bulk AAD processing */
+	lenblks = len & ((size_t)(-AES_BLOCK_LEN));
+	if (lenblks > 0) {
+		ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks);
+		aad += lenblks;
+		len -= lenblks;
+	}
+
+	/* Add remaining AAD to the hash (note, the hash is stored reflected) */
+	if (len > 0) {
+		ares = (unsigned int)len;
+		for (size_t i = 0; i < len; ++i)
+			ctx->gcm.Xi.c[15 - i] ^= aad[i];
+	}
+
+	ctx->gcm.ares = ares;
+
+	return 0;
+}
+
+static int
+_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
+    unsigned char *out, size_t len, bool encrypt)
+{
+	uint64_t mlen = ctx->gcm.len.u[1];
+
+	mlen += len;
+	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))
+		return -1;
+
+	ctx->gcm.len.u[1] = mlen;
+
+	/* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */
+	if (ctx->gcm.ares > 0) {
+		ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx);
+		ctx->gcm.ares = 0;
+	}
+
+	if (encrypt) {
+		ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres,
+		    in, len, out);
+	} else {
+		ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres,
+		    in, len, out);
+	}
+
+	return 0;
+}
+
+static int
+gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
+    unsigned char *out, size_t len)
+{
+	return _gcm_encrypt_avx512(ctx, in, out, len, true);
+}
+
+static int
+gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
+    unsigned char *out, size_t len)
+{
+	return _gcm_encrypt_avx512(ctx, in, out, len, false);
+}
+
+static int
+gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag,
+    size_t len)
+{
+	unsigned int *res = &ctx->gcm.mres;
+
+	/* Finalize AAD processing */
+	if (ctx->gcm.ares > 0)
+		res = &ctx->gcm.ares;
+
+	ossl_aes_gcm_finalize_avx512(ctx, *res);
+
+	ctx->gcm.ares = ctx->gcm.mres = 0;
+
+	if (tag != NULL)
+		return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);
+	return 0;
+}
+
+static const struct ossl_aes_gcm_ops gcm_ops_avx512 = {
+	.init = gcm_init_avx512,
+	.setiv = gcm_setiv_avx512,
+	.aad = gcm_aad_avx512,
+	.encrypt = gcm_encrypt_avx512,
+	.decrypt = gcm_decrypt_avx512,
+	.finish = gcm_finish_avx512,
+	.tag = gcm_tag,
+};
+
+int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx);
+
+int
+ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen,
+    void *_ctx)
+{
+	struct ossl_gcm_context *ctx;
+
+	ctx = _ctx;
+	ctx->ops = &gcm_ops_avx512;
+	gcm_init(ctx, key, klen);
+	return (0);
+}

sys/crypto/openssl/ossl_aes_gcm.c

@@ -1,6 +1,7 @@
 /*
  * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved.
  * Copyright (c) 2021, Intel Corporation. All Rights Reserved.
+ * Copyright (c) 2023, Raptor Engineering, LLC. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -9,11 +10,10 @@
  */
 
 /*
- * This file contains 2 AES-GCM wrapper implementations from OpenSSL, using
- * AES-NI and VAES extensions respectively.  These were ported from
- * cipher_aes_gcm_hw_aesni.inc and cipher_aes_gcm_hw_vaes_avx512.inc.  The
- * AES-NI implementation makes use of a generic C implementation for partial
- * blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
+ * This file contains an AES-GCM wrapper implementation from OpenSSL, using
+ * AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from
+ * cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation
+ * for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
  */
 
 #include <sys/endian.h>
@@ -29,7 +29,117 @@ _Static_assert(
     sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context),
     "ossl_gcm_context too large");
 
-void aesni_set_encrypt_key(const void *key, int bits, void *ctx);
+#if defined(__amd64__) || defined(__i386__)
+#define	AES_set_encrypt_key	aesni_set_encrypt_key
+#define	AES_gcm_encrypt	aesni_gcm_encrypt
+#define	AES_gcm_decrypt	aesni_gcm_decrypt
+#define	AES_encrypt	aesni_encrypt
+#define	AES_ctr32_encrypt_blocks	aesni_ctr32_encrypt_blocks
+#define	GCM_init 	gcm_init_avx
+#define	GCM_gmult	gcm_gmult_avx
+#define	GCM_ghash	gcm_ghash_avx
+
+void AES_set_encrypt_key(const void *key, int bits, void *ctx);
+size_t AES_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
+    const void *key, unsigned char ivec[16], uint64_t *Xi);
+size_t AES_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
+    const void *key, unsigned char ivec[16], uint64_t *Xi);
+void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks);
+void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+    size_t blocks, void *ks, const unsigned char *iv);
+
+void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]);
+void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
+void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
+    size_t len);
+
+#elif defined(__powerpc64__)
+#define	AES_set_encrypt_key	aes_p8_set_encrypt_key
+#define AES_gcm_encrypt(i,o,l,k,v,x) 	ppc_aes_gcm_crypt(i,o,l,k,v,x,1)
+#define AES_gcm_decrypt(i,o,l,k,v,x) 	ppc_aes_gcm_crypt(i,o,l,k,v,x,0)
+#define	AES_encrypt	aes_p8_encrypt
+#define	AES_ctr32_encrypt_blocks	aes_p8_ctr32_encrypt_blocks
+#define	GCM_init	gcm_init_p8
+#define	GCM_gmult	gcm_gmult_p8
+#define	GCM_ghash	gcm_ghash_p8
+
+size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
+    const void *key, unsigned char ivec[16], uint64_t *Xi);
+size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
+    const void *key, unsigned char ivec[16], uint64_t *Xi);
+
+void AES_set_encrypt_key(const void *key, int bits, void *ctx);
+void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks);
+void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+    size_t blocks, void *ks, const unsigned char *iv);
+
+void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]);
+void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
+void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
+    size_t len);
+
+static size_t
+ppc_aes_gcm_crypt(const unsigned char *in, unsigned char *out,
+    size_t len, const void *key, unsigned char ivec_[16], uint64_t *Xi,
+    int encrypt)
+{
+	union {
+		uint32_t d[4];
+		uint8_t c[16];
+	} *ivec = (void *)ivec_;
+	int s = 0;
+	int ndone = 0;
+	int ctr_reset = 0;
+	uint32_t ivec_val;
+	uint64_t blocks_unused;
+	uint64_t nb = len / 16;
+	uint64_t next_ctr = 0;
+	unsigned char ctr_saved[12];
+
+	memcpy(ctr_saved, ivec, 12);
+
+	while (nb) {
+		ivec_val = ivec->d[3];
+#if BYTE_ORDER == LITTLE_ENDIAN
+		ivec_val = bswap32(ivec_val);
+#endif
+
+		blocks_unused = (uint64_t)0xffffffffU + 1 - (uint64_t)ivec_val;
+		if (nb > blocks_unused) {
+			len = blocks_unused * 16;
+			nb -= blocks_unused;
+			next_ctr = blocks_unused;
+			ctr_reset = 1;
+		} else {
+			len = nb * 16;
+			next_ctr = nb;
+			nb = 0;
+		}
+
+		s = encrypt ? ppc_aes_gcm_encrypt(in, out, len, key, ivec->c, Xi) :
+		    ppc_aes_gcm_decrypt(in, out, len, key, ivec->c, Xi);
+
+		/* add counter to ivec */
+#if BYTE_ORDER == LITTLE_ENDIAN
+		ivec->d[3] = bswap32(ivec_val + next_ctr);
+#else
+		ivec->d[3] += next_ctr;
+#endif
+		if (ctr_reset) {
+			ctr_reset = 0;
+			in += len;
+			out += len;
+		}
+		memcpy(ivec, ctr_saved, 12);
+		ndone += s;
+	}
+
+	return ndone;
+}
+
+#else
+#error "Unsupported architecture!"
+#endif
 
 static void
 gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
@@ -39,215 +149,32 @@ gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
 
 	memset(&ctx->gcm, 0, sizeof(ctx->gcm));
 	memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks));
-	aesni_set_encrypt_key(key, keylen, &ctx->aes_ks);
+	AES_set_encrypt_key(key, keylen, &ctx->aes_ks);
 	ctx->ops->init(ctx, key, keylen);
 }
 
 static void
-gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len)
+gcm_tag_op(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len)
 {
 	(void)ctx->ops->finish(ctx, NULL, 0);
 	memcpy(tag, ctx->gcm.Xi.c, len);
 }
 
-void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx);
-void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx);
-void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx,
-    const unsigned char *iv, size_t ivlen);
-void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad,
-    size_t len);
-void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx,
-    unsigned int *pblocklen, const unsigned char *in, size_t len,
-    unsigned char *out);
-void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx,
-    unsigned int *pblocklen, const unsigned char *in, size_t len,
-    unsigned char *out);
-void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen);
-
 static void
-gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
+gcm_init_op(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
 {
-	ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm);
-}
-
-static void
-gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv,
-    size_t len)
-{
-	KASSERT(len == AES_GCM_IV_LEN,
-	    ("%s: invalid IV length %zu", __func__, len));
-
-	ctx->gcm.Yi.u[0] = 0;		/* Current counter */
-	ctx->gcm.Yi.u[1] = 0;
-	ctx->gcm.Xi.u[0] = 0;		/* AAD hash */
-	ctx->gcm.Xi.u[1] = 0;
-	ctx->gcm.len.u[0] = 0;		/* AAD length */
-	ctx->gcm.len.u[1] = 0;		/* Message length */
-	ctx->gcm.ares = 0;
-	ctx->gcm.mres = 0;
-
-	ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len);
-}
-
-static int
-gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad,
-    size_t len)
-{
-	uint64_t alen = ctx->gcm.len.u[0];
-	size_t lenblks;
-	unsigned int ares;
-
-	/* Bad sequence: call of AAD update after message processing */
-	if (ctx->gcm.len.u[1])
-		return -2;
-
-	alen += len;
-	/* AAD is limited by 2^64 bits, thus 2^61 bytes */
-	if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len))
-		return -1;
-	ctx->gcm.len.u[0] = alen;
-
-	ares = ctx->gcm.ares;
-	/* Partial AAD block left from previous AAD update calls */
-	if (ares > 0) {
-		/*
-		 * Fill partial block buffer till full block
-		 * (note, the hash is stored reflected)
-		 */
-		while (ares > 0 && len > 0) {
-			ctx->gcm.Xi.c[15 - ares] ^= *(aad++);
-			--len;
-			ares = (ares + 1) % AES_BLOCK_LEN;
-		}
-		/* Full block gathered */
-		if (ares == 0) {
-			ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx);
-		} else { /* no more AAD */
-			ctx->gcm.ares = ares;
-			return 0;
-		}
-	}
-
-	/* Bulk AAD processing */
-	lenblks = len & ((size_t)(-AES_BLOCK_LEN));
-	if (lenblks > 0) {
-		ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks);
-		aad += lenblks;
-		len -= lenblks;
-	}
-
-	/* Add remaining AAD to the hash (note, the hash is stored reflected) */
-	if (len > 0) {
-		ares = (unsigned int)len;
-		for (size_t i = 0; i < len; ++i)
-			ctx->gcm.Xi.c[15 - i] ^= aad[i];
-	}
-
-	ctx->gcm.ares = ares;
-
-	return 0;
-}
-
-static int
-_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
-    unsigned char *out, size_t len, bool encrypt)
-{
-	uint64_t mlen = ctx->gcm.len.u[1];
-
-	mlen += len;
-	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))
-		return -1;
-
-	ctx->gcm.len.u[1] = mlen;
-
-	/* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */
-	if (ctx->gcm.ares > 0) {
-		ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx);
-		ctx->gcm.ares = 0;
-	}
-
-	if (encrypt) {
-		ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres,
-		    in, len, out);
-	} else {
-		ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres,
-		    in, len, out);
-	}
-
-	return 0;
-}
-
-static int
-gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
-    unsigned char *out, size_t len)
-{
-	return _gcm_encrypt_avx512(ctx, in, out, len, true);
-}
-
-static int
-gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in,
-    unsigned char *out, size_t len)
-{
-	return _gcm_encrypt_avx512(ctx, in, out, len, false);
-}
-
-static int
-gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag,
-    size_t len)
-{
-	unsigned int *res = &ctx->gcm.mres;
-
-	/* Finalize AAD processing */
-	if (ctx->gcm.ares > 0)
-		res = &ctx->gcm.ares;
-
-	ossl_aes_gcm_finalize_avx512(ctx, *res);
-
-	ctx->gcm.ares = ctx->gcm.mres = 0;
-
-	if (tag != NULL)
-		return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);
-	return 0;
-}
-
-static const struct ossl_aes_gcm_ops gcm_ops_avx512 = {
-	.init = gcm_init_avx512,
-	.setiv = gcm_setiv_avx512,
-	.aad = gcm_aad_avx512,
-	.encrypt = gcm_encrypt_avx512,
-	.decrypt = gcm_decrypt_avx512,
-	.finish = gcm_finish_avx512,
-	.tag = gcm_tag,
-};
-
-size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
-    const void *key, unsigned char ivec[16], uint64_t *Xi);
-size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
-    const void *key, unsigned char ivec[16], uint64_t *Xi);
-void aesni_encrypt(const unsigned char *in, unsigned char *out, void *ks);
-void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
-    size_t blocks, void *ks, const unsigned char *iv);
-
-void gcm_init_avx(__uint128_t Htable[16], uint64_t Xi[2]);
-void gcm_gmult_avx(uint64_t Xi[2], const __uint128_t Htable[16]);
-void gcm_ghash_avx(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
-    size_t len);
-
-static void
-gcm_init_aesni(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
-{
-	aesni_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks);
+	AES_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks);
 
 #if BYTE_ORDER == LITTLE_ENDIAN
 	ctx->gcm.H.u[0] = bswap64(ctx->gcm.H.u[0]);
 	ctx->gcm.H.u[1] = bswap64(ctx->gcm.H.u[1]);
 #endif
 
-	gcm_init_avx(ctx->gcm.Htable, ctx->gcm.H.u);
+	GCM_init(ctx->gcm.Htable, ctx->gcm.H.u);
 }
 
 static void
-gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv,
+gcm_setiv_op(struct ossl_gcm_context *ctx, const unsigned char *iv,
     size_t len)
 {
 	uint32_t ctr;
@@ -269,7 +196,7 @@ gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv,
 	ctx->gcm.Xi.u[0] = 0;
 	ctx->gcm.Xi.u[1] = 0;
 
-	aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks);
+	AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks);
 	ctr++;
 
 #if BYTE_ORDER == LITTLE_ENDIAN
@@ -280,7 +207,7 @@ gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv,
 }
 
 static int
-gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad,
+gcm_aad_op(struct ossl_gcm_context *ctx, const unsigned char *aad,
     size_t len)
 {
 	size_t i;
@@ -303,14 +230,14 @@ gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad,
 			n = (n + 1) % 16;
 		}
 		if (n == 0)
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 		else {
 			ctx->gcm.ares = n;
 			return 0;
 		}
 	}
 	if ((i = (len & (size_t)-AES_BLOCK_LEN))) {
-		gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i);
+		GCM_ghash(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i);
 		aad += i;
 		len -= i;
 	}
@@ -341,7 +268,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 
 	if (ctx->gcm.ares) {
 		/* First call to encrypt finalizes GHASH(AAD) */
-		gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+		GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 		ctx->gcm.ares = 0;
 	}
 
@@ -354,7 +281,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 	n = mres % 16;
 	for (i = 0; i < len; ++i) {
 		if (n == 0) {
-			aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
+			AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
 			    &ctx->aes_ks);
 			++ctr;
 #if BYTE_ORDER == LITTLE_ENDIAN
@@ -366,7 +293,7 @@ gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 		ctx->gcm.Xi.c[n] ^= out[i] = in[i] ^ ctx->gcm.EKi.c[n];
 		mres = n = (n + 1) % 16;
 		if (n == 0)
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 	}
 
 	ctx->gcm.mres = mres;
@@ -390,7 +317,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 
 	if (ctx->gcm.ares) {
 		/* First call to encrypt finalizes GHASH(AAD) */
-		gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+		GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 		ctx->gcm.ares = 0;
 	}
 
@@ -408,7 +335,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 			n = (n + 1) % 16;
 		}
 		if (n == 0) {
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 			mres = 0;
 		} else {
 			ctx->gcm.mres = n;
@@ -418,7 +345,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 	if ((i = (len & (size_t)-16))) {
 		size_t j = i / 16;
 
-		aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c);
+		AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c);
 		ctr += (unsigned int)j;
 #if BYTE_ORDER == LITTLE_ENDIAN
 		ctx->gcm.Yi.d[3] = bswap32(ctr);
@@ -430,12 +357,12 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 		while (j--) {
 			for (i = 0; i < 16; ++i)
 				ctx->gcm.Xi.c[i] ^= out[i];
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 			out += 16;
 		}
 	}
 	if (len) {
-		aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
+		AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
 		++ctr;
 #if BYTE_ORDER == LITTLE_ENDIAN
 		ctx->gcm.Yi.d[3] = bswap32(ctr);
@@ -453,7 +380,7 @@ gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 }
 
 static int
-gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
+gcm_encrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in,
     unsigned char *out, size_t len)
 {
 	size_t bulk = 0, res;
@@ -463,7 +390,7 @@ gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
 	if ((error = gcm_encrypt(ctx, in, out, res)) != 0)
 		return error;
 
-	bulk = aesni_gcm_encrypt(in + res, out + res, len - res,
+	bulk = AES_gcm_encrypt(in + res, out + res, len - res,
 	    &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u);
 	ctx->gcm.len.u[1] += bulk;
 	bulk += res;
@@ -492,7 +419,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 
 	if (ctx->gcm.ares) {
 		/* First call to encrypt finalizes GHASH(AAD) */
-		gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+		GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 		ctx->gcm.ares = 0;
 	}
 
@@ -506,7 +433,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 	for (i = 0; i < len; ++i) {
 		uint8_t c;
 		if (n == 0) {
-			aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
+			AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
 			    &ctx->aes_ks);
 			++ctr;
 #if BYTE_ORDER == LITTLE_ENDIAN
@@ -520,7 +447,7 @@ gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
 		ctx->gcm.Xi.c[n] ^= c;
 		mres = n = (n + 1) % 16;
 		if (n == 0)
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 	}
 
 	ctx->gcm.mres = mres;
@@ -544,7 +471,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 
 	if (ctx->gcm.ares) {
 		/* First call to decrypt finalizes GHASH(AAD) */
-		gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+		GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 		ctx->gcm.ares = 0;
 	}
 
@@ -564,7 +491,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 			n = (n + 1) % 16;
 		}
 		if (n == 0) {
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 			mres = 0;
 		} else {
 			ctx->gcm.mres = n;
@@ -578,12 +505,12 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 			size_t k;
 			for (k = 0; k < 16; ++k)
 				ctx->gcm.Xi.c[k] ^= in[k];
-			gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+			GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 			in += 16;
 		}
 		j = i / 16;
 		in -= i;
-		aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c);
+		AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c);
 		ctr += (unsigned int)j;
 #if BYTE_ORDER == LITTLE_ENDIAN
 		ctx->gcm.Yi.d[3] = bswap32(ctr);
@@ -595,7 +522,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 		len -= i;
 	}
 	if (len) {
-		aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
+		AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks);
 		++ctr;
 #if BYTE_ORDER == LITTLE_ENDIAN
 		ctx->gcm.Yi.d[3] = bswap32(ctr);
@@ -615,7 +542,7 @@ gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in,
 }
 
 static int
-gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
+gcm_decrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in,
     unsigned char *out, size_t len)
 {
 	size_t bulk = 0, res;
@@ -625,7 +552,7 @@ gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
 	if ((error = gcm_decrypt(ctx, in, out, res)) != 0)
 		return error;
 
-	bulk = aesni_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks,
+	bulk = AES_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks,
 	    ctx->gcm.Yi.c, ctx->gcm.Xi.u);
 	ctx->gcm.len.u[1] += bulk;
 	bulk += res;
@@ -637,14 +564,14 @@ gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
 }
 
 static int
-gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag,
+gcm_finish_op(struct ossl_gcm_context *ctx, const unsigned char *tag,
     size_t len)
 {
 	uint64_t alen = ctx->gcm.len.u[0] << 3;
 	uint64_t clen = ctx->gcm.len.u[1] << 3;
 
 	if (ctx->gcm.mres || ctx->gcm.ares)
-		gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+		GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 
 #if BYTE_ORDER == LITTLE_ENDIAN
 	alen = bswap64(alen);
@@ -653,7 +580,7 @@ gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag,
 
 	ctx->gcm.Xi.u[0] ^= alen;
 	ctx->gcm.Xi.u[1] ^= clen;
-	gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+	GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable);
 
 	ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0];
 	ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1];
@@ -663,40 +590,26 @@ gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag,
 	return 0;
 }
 
-static const struct ossl_aes_gcm_ops gcm_ops_aesni = {
-	.init = gcm_init_aesni,
-	.setiv = gcm_setiv_aesni,
-	.aad = gcm_aad_aesni,
-	.encrypt = gcm_encrypt_aesni,
-	.decrypt = gcm_decrypt_aesni,
-	.finish = gcm_finish_aesni,
-	.tag = gcm_tag,
+static const struct ossl_aes_gcm_ops gcm_ops = {
+	.init = gcm_init_op,
+	.setiv = gcm_setiv_op,
+	.aad = gcm_aad_op,
+	.encrypt = gcm_encrypt_op,
+	.decrypt = gcm_decrypt_op,
+	.finish = gcm_finish_op,
+	.tag = gcm_tag_op,
 };
 
-int ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, void *_ctx);
+int ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx);
 
 int
-ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen,
+ossl_aes_gcm_setkey(const unsigned char *key, int klen,
     void *_ctx)
 {
 	struct ossl_gcm_context *ctx;
 
 	ctx = _ctx;
-	ctx->ops = &gcm_ops_aesni;
-	gcm_init(ctx, key, klen);
-	return (0);
-}
-
-int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx);
-
-int
-ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen,
-    void *_ctx)
-{
-	struct ossl_gcm_context *ctx;
-
-	ctx = _ctx;
-	ctx->ops = &gcm_ops_avx512;
+	ctx->ops = &gcm_ops;
 	gcm_init(ctx, key, klen);
 	return (0);
 }

sys/crypto/openssl/ossl_ppc.c

@@ -38,9 +38,12 @@ unsigned int OPENSSL_ppccap_P = 0;
 
 ossl_cipher_setkey_t aes_p8_set_encrypt_key;
 ossl_cipher_setkey_t aes_p8_set_decrypt_key;
+
 ossl_cipher_setkey_t vpaes_set_encrypt_key;
 ossl_cipher_setkey_t vpaes_set_decrypt_key;
 
+ossl_cipher_setkey_t ossl_aes_gcm_setkey;
+
 void
 ossl_cpuid(struct ossl_softc *sc)
 {
@@ -75,7 +78,11 @@ ossl_cpuid(struct ossl_softc *sc)
 		ossl_cipher_aes_cbc.set_encrypt_key = aes_p8_set_encrypt_key;
 		ossl_cipher_aes_cbc.set_decrypt_key = aes_p8_set_decrypt_key;
 		sc->has_aes = true;
-	} else if (OPENSSL_ppccap_P & PPC_ALTIVEC) {
+
+		ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey;
+		ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey;
+		sc->has_aes_gcm = true;
+    } else if (OPENSSL_ppccap_P & PPC_ALTIVEC) {
 		ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key;
 		ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key;
 		sc->has_aes = true;

sys/crypto/openssl/ossl_x86.c

@@ -56,7 +56,7 @@ ossl_cipher_setkey_t aesni_set_decrypt_key;
 
 #ifdef __amd64__
 int ossl_vaes_vpclmulqdq_capable(void);
-ossl_cipher_setkey_t ossl_aes_gcm_setkey_aesni;
+ossl_cipher_setkey_t ossl_aes_gcm_setkey;
 ossl_cipher_setkey_t ossl_aes_gcm_setkey_avx512;
 #endif
 
@@ -141,8 +141,8 @@ ossl_cpuid(struct ossl_softc *sc)
 	} else if ((cpu_feature2 &
 	    (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) ==
 	    (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) {
-		ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_aesni;
-		ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_aesni;
+		ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey;
+		ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey;
 		sc->has_aes_gcm = true;
 	} else {
 		sc->has_aes_gcm = false;

sys/modules/ossl/Makefile

@@ -25,7 +25,7 @@ SRCS.arm= \
 	sha256-armv4.S \
 	sha512-armv4.S \
 	ossl_arm.c \
-	ossl_aes_gcm.c
+	ossl_aes_gcm_neon.c
 
 SRCS.aarch64= \
 	chacha-armv8.S \
@@ -48,6 +48,7 @@ SRCS.amd64= \
 	sha256-x86_64.S \
 	sha512-x86_64.S \
 	ossl_aes_gcm.c \
+	ossl_aes_gcm_avx512.c \
 	ossl_x86.c
 
 SRCS.i386= \
@@ -60,6 +61,8 @@ SRCS.i386= \
 	ossl_x86.c
 
 SRCS.powerpc64le= \
+	aes-gcm-ppc.S \
+	ossl_aes_gcm.c \
 	ossl_ppccap.c \
 	aes-ppc.S \
 	aesp8-ppc.S \
@@ -81,6 +84,8 @@ SRCS.powerpc64le= \
 	x25519-ppc64.S
 
 SRCS.powerpc64= \
+	aes-gcm-ppc.S \
+	ossl_aes_gcm.c \
 	ossl_ppccap.c \
 	aes-ppc.S \
 	aesp8-ppc.S \