Merge tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers:

 - Add support for verifying ML-DSA signatures.

   ML-DSA (Module-Lattice-Based Digital Signature Algorithm) is a
   recently-standardized post-quantum (quantum-resistant) signature
   algorithm. It was known as Dilithium pre-standardization.

   The first use case in the kernel will be module signing. But there
   are also other users of RSA and ECDSA signatures in the kernel that
   might want to upgrade to ML-DSA eventually.

 - Improve the AES library:

     - Make the AES key expansion and single block encryption and
       decryption functions use the architecture-optimized AES code.
       Enable these optimizations by default.

     - Support preparing an AES key for encryption-only, using about
       half as much memory as a bidirectional key.

     - Replace the existing two generic implementations of AES with a
       single one.

 - Simplify how Adiantum message hashing is implemented. Remove the
   "nhpoly1305" crypto_shash in favor of direct lib/crypto/ support for
   NH hashing, and enable optimizations by default.

* tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (53 commits)
  lib/crypto: mldsa: Clarify the documentation for mldsa_verify() slightly
  lib/crypto: aes: Drop 'volatile' from aes_sbox and aes_inv_sbox
  lib/crypto: aes: Remove old AES en/decryption functions
  lib/crypto: aesgcm: Use new AES library API
  lib/crypto: aescfb: Use new AES library API
  crypto: omap - Use new AES library API
  crypto: inside-secure - Use new AES library API
  crypto: drbg - Use new AES library API
  crypto: crypto4xx - Use new AES library API
  crypto: chelsio - Use new AES library API
  crypto: ccp - Use new AES library API
  crypto: x86/aes-gcm - Use new AES library API
  crypto: arm64/ghash - Use new AES library API
  crypto: arm/ghash - Use new AES library API
  staging: rtl8723bs: core: Use new AES library API
  net: phy: mscc: macsec: Use new AES library API
  chelsio: Use new AES library API
  Bluetooth: SMP: Use new AES library API
  crypto: x86/aes - Remove the superseded AES-NI crypto_cipher
  lib/crypto: x86/aes: Add AES-NI optimization
  ...

16 files changed, 19 insertions, 5763 deletions

diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig
index 662aed46f9c7..2d056f1fc90f 100644
--- a/arch/powerpc/crypto/Kconfig
+++ b/arch/powerpc/crypto/Kconfig
@@ -5,9 +5,9 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)"
 config CRYPTO_AES_PPC_SPE
 	tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)"
 	depends on SPE
+	select CRYPTO_LIB_AES
 	select CRYPTO_SKCIPHER
 	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
 	  Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes
 
 	  Architecture: powerpc using:
diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile
index 5960e5300db7..3ac0886282a2 100644
--- a/arch/powerpc/crypto/Makefile
+++ b/arch/powerpc/crypto/Makefile
@@ -9,9 +9,9 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o
 obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
 obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
 
-aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
+aes-ppc-spe-y := aes-spe-glue.o
 aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
-vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
+vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
 
 ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
 override flavour := linux-ppc64le
@@ -26,15 +26,14 @@ endif
 quiet_cmd_perl = PERL    $@
       cmd_perl = $(PERL) $< $(flavour) > $@
 
-targets += aesp10-ppc.S ghashp10-ppc.S aesp8-ppc.S ghashp8-ppc.S
+targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S
 
 $(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perl)
 
-$(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
+$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perl)
 
 OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y
 OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y
-OBJECT_FILES_NON_STANDARD_aesp8-ppc.o := y
 OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y
diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c
index 85f4fd4b1bdc..f3417436d3f7 100644
--- a/arch/powerpc/crypto/aes-gcm-p10-glue.c
+++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c
@@ -44,7 +44,7 @@ asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
 			      unsigned char *aad, unsigned int alen);
 asmlinkage void gcm_update(u8 *iv, void *Xi);
 
-struct aes_key {
+struct p10_aes_key {
 	u8 key[AES_MAX_KEYLENGTH];
 	u64 rounds;
 };
@@ -63,7 +63,7 @@ struct Hash_ctx {
 };
 
 struct p10_aes_gcm_ctx {
-	struct aes_key enc_key;
+	struct p10_aes_key enc_key;
 	u8 nonce[RFC4106_NONCE_SIZE];
 };
 
diff --git a/arch/powerpc/crypto/aes-spe-core.S b/arch/powerpc/crypto/aes-spe-core.S
deleted file mode 100644
index 8e00eccc352b..000000000000
--- a/arch/powerpc/crypto/aes-spe-core.S
+++ /dev/null
@@ -1,346 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Fast AES implementation for SPE instruction set (PPC)
- *
- * This code makes use of the SPE SIMD instruction set as defined in
- * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
- * Implementation is based on optimization guide notes from
- * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-#include "aes-spe-regs.h"
-
-#define	EAD(in, bpos) \
-	rlwimi		rT0,in,28-((bpos+3)%4)*8,20,27;
-
-#define DAD(in, bpos) \
-	rlwimi		rT1,in,24-((bpos+3)%4)*8,24,31;
-
-#define LWH(out, off) \
-	evlwwsplat	out,off(rT0);	/* load word high		*/
-
-#define LWL(out, off) \
-	lwz		out,off(rT0);	/* load word low		*/
-
-#define LBZ(out, tab, off) \
-	lbz		out,off(tab);	/* load byte			*/
-
-#define LAH(out, in, bpos, off) \
-	EAD(in, bpos)			/* calc addr + load word high	*/ \
-	LWH(out, off)
-
-#define LAL(out, in, bpos, off) \
-	EAD(in, bpos)			/* calc addr + load word low	*/ \
-	LWL(out, off)
-
-#define LAE(out, in, bpos) \
-	EAD(in, bpos)			/* calc addr + load enc byte	*/ \
-	LBZ(out, rT0, 8)
-
-#define LBE(out) \
-	LBZ(out, rT0, 8)		/* load enc byte		*/
-
-#define LAD(out, in, bpos) \
-	DAD(in, bpos)			/* calc addr + load dec byte	*/ \
-	LBZ(out, rT1, 0)
-
-#define LBD(out) \
-	LBZ(out, rT1, 0)
-
-/*
- * ppc_encrypt_block: The central encryption function for a single 16 bytes
- * block. It does no stack handling or register saving to support fast calls
- * via bl/blr. It expects that caller has pre-xored input data with first
- * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
- * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
- * and rW0-rW3 and caller must execute a final xor on the output registers.
- * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
- *
- */
-_GLOBAL(ppc_encrypt_block)
-	LAH(rW4, rD1, 2, 4)
-	LAH(rW6, rD0, 3, 0)
-	LAH(rW3, rD0, 1, 8)
-ppc_encrypt_block_loop:
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,16(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 2)
-	evxor		rW3,rW3,rW5
-	LWH(rW4, 4)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,32(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,40(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 2)
-	evxor		rW3,rW3,rW5
-	LWH(rW4, 4)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	addi		rKP,rKP,32
-	bdnz		ppc_encrypt_block_loop
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,16(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW5
-	LBE(rW2)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 1)
-	evxor		rD3,rD3,rW3
-	LBE(rW6)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 0)
-	evmergehi	rD2,rD2,rD3
-	LBE(rW1)
-	LAE(rW0, rD3, 0)
-	LAE(rW1, rD0, 0)
-	LAE(rW4, rD2, 1)
-	LAE(rW5, rD3, 1)
-	LAE(rW3, rD2, 0)
-	LAE(rW7, rD1, 1)
-	rlwimi		rW0,rW4,8,16,23
-	rlwimi		rW1,rW5,8,16,23
-	LAE(rW4, rD1, 2)
-	LAE(rW5, rD2, 2)
-	rlwimi		rW2,rW6,8,16,23
-	rlwimi		rW3,rW7,8,16,23
-	LAE(rW6, rD3, 2)
-	LAE(rW7, rD0, 2)
-	rlwimi		rW0,rW4,16,8,15
-	rlwimi		rW1,rW5,16,8,15
-	LAE(rW4, rD0, 3)
-	LAE(rW5, rD1, 3)
-	rlwimi		rW2,rW6,16,8,15
-	lwz		rD0,32(rKP)
-	rlwimi		rW3,rW7,16,8,15
-	lwz		rD1,36(rKP)
-	LAE(rW6, rD2, 3)
-	LAE(rW7, rD3, 3)
-	rlwimi		rW0,rW4,24,0,7
-	lwz		rD2,40(rKP)
-	rlwimi		rW1,rW5,24,0,7
-	lwz		rD3,44(rKP)
-	rlwimi		rW2,rW6,24,0,7
-	rlwimi		rW3,rW7,24,0,7
-	blr
-
-/*
- * ppc_decrypt_block: The central decryption function for a single 16 bytes
- * block. It does no stack handling or register saving to support fast calls
- * via bl/blr. It expects that caller has pre-xored input data with first
- * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
- * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
- * and rW0-rW3 and caller must execute a final xor on the output registers.
- * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
- *
- */
-_GLOBAL(ppc_decrypt_block)
-	LAH(rW0, rD1, 0, 12)
-	LAH(rW6, rD0, 3, 0)
-	LAH(rW3, rD0, 1, 8)
-ppc_decrypt_block_loop:
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		rD1,16(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LWH(rW0, 12)
-	evxor		rW3,rW3,rW1
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW5
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		 rD1,32(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,40(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LWH(rW0, 12)
-	evxor		rW3,rW3,rW1
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW5
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	addi		rKP,rKP,32
-	bdnz		ppc_decrypt_block_loop
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		 rD1,16(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	DAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LBD(rW0)
-	evxor		rW3,rW3,rW1
-	DAD(rD0, 1)
-	evxor		rD3,rD3,rW3
-	LBD(rW6)
-	evxor		rD3,rD3,rW5
-	DAD(rD0, 0)
-	evmergehi	rD2,rD2,rD3
-	LBD(rW3)
-	LAD(rW2, rD3, 0)
-	LAD(rW1, rD2, 0)
-	LAD(rW4, rD2, 1)
-	LAD(rW5, rD3, 1)
-	LAD(rW7, rD1, 1)
-	rlwimi		rW0,rW4,8,16,23
-	rlwimi		rW1,rW5,8,16,23
-	LAD(rW4, rD3, 2)
-	LAD(rW5, rD0, 2)
-	rlwimi		rW2,rW6,8,16,23
-	rlwimi		rW3,rW7,8,16,23
-	LAD(rW6, rD1, 2)
-	LAD(rW7, rD2, 2)
-	rlwimi		rW0,rW4,16,8,15
-	rlwimi		rW1,rW5,16,8,15
-	LAD(rW4, rD0, 3)
-	LAD(rW5, rD1, 3)
-	rlwimi		rW2,rW6,16,8,15
-	lwz		rD0,32(rKP)
-	rlwimi		rW3,rW7,16,8,15
-	lwz		rD1,36(rKP)
-	LAD(rW6, rD2, 3)
-	LAD(rW7, rD3, 3)
-	rlwimi		rW0,rW4,24,0,7
-	lwz		rD2,40(rKP)
-	rlwimi		rW1,rW5,24,0,7
-	lwz		rD3,44(rKP)
-	rlwimi		rW2,rW6,24,0,7
-	rlwimi		rW3,rW7,24,0,7
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c
index efab78a3a8f6..7d2827e65240 100644
--- a/arch/powerpc/crypto/aes-spe-glue.c
+++ b/arch/powerpc/crypto/aes-spe-glue.c
@@ -51,30 +51,6 @@ struct ppc_xts_ctx {
 	u32 rounds;
 };
 
-extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
-extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
-extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes);
-extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes);
-extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv, u32 *key_twk);
-extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes, u8 *iv, u32 *key_twk);
-
-extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
-
-extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
-				     unsigned int key_len);
-
 static void spe_begin(void)
 {
 	/* disable preemption and save users SPE registers if required */
@@ -89,10 +65,10 @@ static void spe_end(void)
 	preempt_enable();
 }
 
-static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-		unsigned int key_len)
+static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
+				   const u8 *in_key, unsigned int key_len)
 {
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 
 	switch (key_len) {
 	case AES_KEYSIZE_128:
@@ -116,12 +92,6 @@ static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
-				   const u8 *in_key, unsigned int key_len)
-{
-	return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
-}
-
 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 		   unsigned int key_len)
 {
@@ -159,24 +129,6 @@ static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 	return 0;
 }
 
-static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	spe_begin();
-	ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
-	spe_end();
-}
-
-static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	spe_begin();
-	ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
-	spe_end();
-}
-
 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -410,26 +362,6 @@ static int ppc_xts_decrypt(struct skcipher_request *req)
  * with kmalloc() in the crypto infrastructure
  */
 
-static struct crypto_alg aes_cipher_alg = {
-	.cra_name		=	"aes",
-	.cra_driver_name	=	"aes-ppc-spe",
-	.cra_priority		=	300,
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		=	AES_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct ppc_aes_ctx),
-	.cra_alignmask		=	0,
-	.cra_module		=	THIS_MODULE,
-	.cra_u			=	{
-		.cipher = {
-			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
-			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
-			.cia_setkey		=	ppc_aes_setkey,
-			.cia_encrypt		=	ppc_aes_encrypt,
-			.cia_decrypt		=	ppc_aes_decrypt
-		}
-	}
-};
-
 static struct skcipher_alg aes_skcipher_algs[] = {
 	{
 		.base.cra_name		=	"ecb(aes)",
@@ -488,22 +420,12 @@ static struct skcipher_alg aes_skcipher_algs[] = {
 
 static int __init ppc_aes_mod_init(void)
 {
-	int err;
-
-	err = crypto_register_alg(&aes_cipher_alg);
-	if (err)
-		return err;
-
-	err = crypto_register_skciphers(aes_skcipher_algs,
-					ARRAY_SIZE(aes_skcipher_algs));
-	if (err)
-		crypto_unregister_alg(&aes_cipher_alg);
-	return err;
+	return crypto_register_skciphers(aes_skcipher_algs,
+					 ARRAY_SIZE(aes_skcipher_algs));
 }
 
 static void __exit ppc_aes_mod_fini(void)
 {
-	crypto_unregister_alg(&aes_cipher_alg);
 	crypto_unregister_skciphers(aes_skcipher_algs,
 				    ARRAY_SIZE(aes_skcipher_algs));
 }
diff --git a/arch/powerpc/crypto/aes-spe-keys.S b/arch/powerpc/crypto/aes-spe-keys.S
deleted file mode 100644
index 2e1bc0d099bf..000000000000
--- a/arch/powerpc/crypto/aes-spe-keys.S
+++ /dev/null
@@ -1,278 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Key handling functions for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-
-#ifdef __BIG_ENDIAN__
-#define LOAD_KEY(d, s, off) \
-	lwz		d,off(s);
-#else
-#define LOAD_KEY(d, s, off) \
-	li		r0,off; \
-	lwbrx		d,s,r0;
-#endif
-
-#define INITIALIZE_KEY \
-	stwu		r1,-32(r1);	/* create stack frame		*/ \
-	stw		r14,8(r1);	/* save registers		*/ \
-	stw		r15,12(r1);					   \
-	stw		r16,16(r1);
-
-#define FINALIZE_KEY \
-	lwz		r14,8(r1);	/* restore registers		*/ \
-	lwz		r15,12(r1);					   \
-	lwz		r16,16(r1);					   \
-	xor		r5,r5,r5;	/* clear sensitive data		*/ \
-	xor		r6,r6,r6;					   \
-	xor		r7,r7,r7;					   \
-	xor		r8,r8,r8;					   \
-	xor		r9,r9,r9;					   \
-	xor		r10,r10,r10;					   \
-	xor		r11,r11,r11;					   \
-	xor		r12,r12,r12;					   \
-	addi		r1,r1,32;	/* cleanup stack		*/
-
-#define LS_BOX(r, t1, t2) \
-	lis		t2,PPC_AES_4K_ENCTAB@h;				   \
-	ori		t2,t2,PPC_AES_4K_ENCTAB@l;			   \
-	rlwimi		t2,r,4,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,0,24,31;					   \
-	rlwimi		t2,r,28,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,8,16,23;					   \
-	rlwimi		t2,r,20,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,16,8,15;					   \
-	rlwimi		t2,r,12,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,24,0,7;
-
-#define GF8_MUL(out, in, t1, t2) \
-	lis t1,0x8080;			/* multiplication in GF8	*/ \
-	ori t1,t1,0x8080; 						   \
-	and t1,t1,in; 							   \
-	srwi t1,t1,7; 							   \
-	mulli t1,t1,0x1b; 						   \
-	lis t2,0x7f7f; 							   \
-	ori t2,t2,0x7f7f; 						   \
-	and t2,t2,in; 							   \
-	slwi t2,t2,1; 							   \
-	xor out,t1,t2;
-
-/*
- * ppc_expand_key_128(u32 *key_enc, const u8 *key)
- *
- * Expand 128 bit key into 176 bytes encryption key. It consists of
- * key itself plus 10 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_128)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	stw		r5,0(r3)	/* key[0..3] = input data	*/
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	li		r16,10		/* 10 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_128_loop:
-	addi		r3,r3,16
-	mr		r14,r8		/* apply LS_BOX to 4th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor next 4 keys		*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	stw		r5,0(r3)	/* store next 4 keys		*/
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO by 2 in GF	*/
-	subi		r16,r16,1
-	cmpwi		r16,0
-	bt		eq,ppc_expand_128_end
-	b		ppc_expand_128_loop
-ppc_expand_128_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_expand_key_192(u32 *key_enc, const u8 *key)
- *
- * Expand 192 bit key into 208 bytes encryption key. It consists of key
- * itself plus 12 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_192)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	LOAD_KEY(r9,r4,16)
-	LOAD_KEY(r10,r4,20)
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	li		r16,8		/* 8 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_192_loop:
-	addi		r3,r3,24
-	mr		r14,r10		/* apply LS_BOX to 6th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor next 6 keys		*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	xor		r9,r9,r8
-	xor		r10,r10,r9
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	subi		r16,r16,1
-	cmpwi		r16,0		/* last round early kick out	*/
-	bt		eq,ppc_expand_192_end
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO GF8		*/
-	b		ppc_expand_192_loop
-ppc_expand_192_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_expand_key_256(u32 *key_enc, const u8 *key)
- *
- * Expand 256 bit key into 240 bytes encryption key. It consists of key
- * itself plus 14 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_256)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	LOAD_KEY(r9,r4,16)
-	LOAD_KEY(r10,r4,20)
-	LOAD_KEY(r11,r4,24)
-	LOAD_KEY(r12,r4,28)
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	stw		r11,24(r3)
-	stw		r12,28(r3)
-	li		r16,7		/* 7 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_256_loop:
-	addi		r3,r3,32
-	mr		r14,r12		/* apply LS_BOX to 8th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor 4 keys			*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	mr		r14,r8
-	LS_BOX(r14, r15, r4)		/* apply LS_BOX to 4th temp	*/
-	xor		r9,r9,r14	/* xor 4 keys			*/
-	xor		r10,r10,r9
-	xor		r11,r11,r10
-	xor		r12,r12,r11
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	subi		r16,r16,1
-	cmpwi		r16,0		/* last round early kick out	*/
-	bt		eq,ppc_expand_256_end
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	stw		r11,24(r3)
-	stw		r12,28(r3)
-	GF8_MUL(r0, r0, r4, r14)
-	b		ppc_expand_256_loop
-ppc_expand_256_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_generate_decrypt_key: derive decryption key from encryption key
- * number of bytes to handle are calculated from length of key (16/24/32)
- *
- */
-_GLOBAL(ppc_generate_decrypt_key)
-	addi		r6,r5,24
-	slwi		r6,r6,2
-	lwzx		r7,r4,r6	/* first/last 4 words are same	*/
-	stw		r7,0(r3)
-	lwz		r7,0(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,4(r3)
-	lwz		r7,4(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,8(r3)
-	lwz		r7,8(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,12(r3)
-	lwz		r7,12(r4)
-	stwx		r7,r3,r6
-	addi		r3,r3,16
-	add		r4,r4,r6
-	subi		r4,r4,28
-	addi		r5,r5,20
-	srwi		r5,r5,2
-ppc_generate_decrypt_block:
-	li	r6,4
-	mtctr	r6
-ppc_generate_decrypt_word:
-	lwz		r6,0(r4)
-	GF8_MUL(r7, r6, r0, r7)
-	GF8_MUL(r8, r7, r0, r8)
-	GF8_MUL(r9, r8, r0, r9)
-	xor		r10,r9,r6
-	xor		r11,r7,r8
-	xor		r11,r11,r9
-	xor		r12,r7,r10
-	rotrwi		r12,r12,24
-	xor		r11,r11,r12
-	xor		r12,r8,r10
-	rotrwi		r12,r12,16
-	xor		r11,r11,r12
-	rotrwi		r12,r10,8
-	xor		r11,r11,r12
-	stw		r11,0(r3)
-	addi		r3,r3,4
-	addi		r4,r4,4
-	bdnz		ppc_generate_decrypt_word
-	subi		r4,r4,32
-	subi		r5,r5,1
-	cmpwi		r5,0
-	bt		gt,ppc_generate_decrypt_block
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-modes.S b/arch/powerpc/crypto/aes-spe-modes.S
deleted file mode 100644
index 3f92a6a85785..000000000000
--- a/arch/powerpc/crypto/aes-spe-modes.S
+++ /dev/null
@@ -1,625 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-#include "aes-spe-regs.h"
-
-#ifdef __BIG_ENDIAN__			/* Macros for big endian builds	*/
-
-#define LOAD_DATA(reg, off) \
-	lwz		reg,off(rSP);	/* load with offset		*/
-#define SAVE_DATA(reg, off) \
-	stw		reg,off(rDP);	/* save with offset		*/
-#define NEXT_BLOCK \
-	addi		rSP,rSP,16;	/* increment pointers per bloc	*/ \
-	addi		rDP,rDP,16;
-#define LOAD_IV(reg, off) \
-	lwz		reg,off(rIP);	/* IV loading with offset	*/
-#define SAVE_IV(reg, off) \
-	stw		reg,off(rIP);	/* IV saving with offset	*/
-#define START_IV			/* nothing to reset		*/
-#define CBC_DEC 16			/* CBC decrement per block	*/
-#define CTR_DEC 1			/* CTR decrement one byte	*/
-
-#else					/* Macros for little endian	*/
-
-#define LOAD_DATA(reg, off) \
-	lwbrx		reg,0,rSP;	/* load reversed		*/ \
-	addi		rSP,rSP,4;	/* and increment pointer	*/
-#define SAVE_DATA(reg, off) \
-	stwbrx		reg,0,rDP;	/* save reversed		*/ \
-	addi		rDP,rDP,4;	/* and increment pointer	*/
-#define NEXT_BLOCK			/* nothing todo			*/
-#define LOAD_IV(reg, off) \
-	lwbrx		reg,0,rIP;	/* load reversed		*/ \
-	addi		rIP,rIP,4;	/* and increment pointer	*/
-#define SAVE_IV(reg, off) \
-	stwbrx		reg,0,rIP;	/* load reversed		*/ \
-	addi		rIP,rIP,4;	/* and increment pointer	*/
-#define START_IV \
-	subi		rIP,rIP,16;	/* must reset pointer		*/
-#define CBC_DEC 32			/* 2 blocks because of incs	*/
-#define CTR_DEC 17			/* 1 block because of incs	*/
-
-#endif
-
-#define SAVE_0_REGS
-#define LOAD_0_REGS
-
-#define SAVE_4_REGS \
-	stw		rI0,96(r1);	/* save 32 bit registers	*/ \
-	stw		rI1,100(r1);					   \
-	stw		rI2,104(r1);					   \
-	stw		rI3,108(r1);
-
-#define LOAD_4_REGS \
-	lwz		rI0,96(r1);	/* restore 32 bit registers	*/ \
-	lwz		rI1,100(r1);					   \
-	lwz		rI2,104(r1);					   \
-	lwz		rI3,108(r1);
-
-#define SAVE_8_REGS \
-	SAVE_4_REGS							   \
-	stw		rG0,112(r1);	/* save 32 bit registers	*/ \
-	stw		rG1,116(r1);					   \
-	stw		rG2,120(r1);					   \
-	stw		rG3,124(r1);
-
-#define LOAD_8_REGS \
-	LOAD_4_REGS							   \
-	lwz		rG0,112(r1);	/* restore 32 bit registers	*/ \
-	lwz		rG1,116(r1);					   \
-	lwz		rG2,120(r1);					   \
-	lwz		rG3,124(r1);
-
-#define INITIALIZE_CRYPT(tab,nr32bitregs) \
-	mflr		r0;						   \
-	stwu		r1,-160(r1);	/* create stack frame		*/ \
-	lis		rT0,tab@h;	/* en-/decryption table pointer	*/ \
-	stw		r0,8(r1);	/* save link register		*/ \
-	ori		rT0,rT0,tab@l;					   \
-	evstdw		r14,16(r1);					   \
-	mr		rKS,rKP;					   \
-	evstdw		r15,24(r1);	/* We must save non volatile	*/ \
-	evstdw		r16,32(r1);	/* registers. Take the chance	*/ \
-	evstdw		r17,40(r1);	/* and save the SPE part too	*/ \
-	evstdw		r18,48(r1);					   \
-	evstdw		r19,56(r1);					   \
-	evstdw		r20,64(r1);					   \
-	evstdw		r21,72(r1);					   \
-	evstdw		r22,80(r1);					   \
-	evstdw		r23,88(r1);					   \
-	SAVE_##nr32bitregs##_REGS
-
-#define FINALIZE_CRYPT(nr32bitregs) \
-	lwz		r0,8(r1);					   \
-	evldw		r14,16(r1);	/* restore SPE registers	*/ \
-	evldw		r15,24(r1);					   \
-	evldw		r16,32(r1);					   \
-	evldw		r17,40(r1);					   \
-	evldw		r18,48(r1);					   \
-	evldw		r19,56(r1);					   \
-	evldw		r20,64(r1);					   \
-	evldw		r21,72(r1);					   \
-	evldw		r22,80(r1);					   \
-	evldw		r23,88(r1);					   \
-	LOAD_##nr32bitregs##_REGS					   \
-	mtlr		r0;		/* restore link register	*/ \
-	xor		r0,r0,r0;					   \
-	stw		r0,16(r1);	/* delete sensitive data	*/ \
-	stw		r0,24(r1);	/* that we might have pushed	*/ \
-	stw		r0,32(r1);	/* from other context that runs	*/ \
-	stw		r0,40(r1);	/* the same code		*/ \
-	stw		r0,48(r1);					   \
-	stw		r0,56(r1);					   \
-	stw		r0,64(r1);					   \
-	stw		r0,72(r1);					   \
-	stw		r0,80(r1);					   \
-	stw		r0,88(r1);					   \
-	addi		r1,r1,160;	/* cleanup stack frame		*/
-
-#define ENDIAN_SWAP(t0, t1, s0, s1) \
-	rotrwi		t0,s0,8;	/* swap endianness for 2 GPRs	*/ \
-	rotrwi		t1,s1,8;					   \
-	rlwimi		t0,s0,8,8,15;					   \
-	rlwimi		t1,s1,8,8,15;					   \
-	rlwimi		t0,s0,8,24,31;					   \
-	rlwimi		t1,s1,8,24,31;
-
-#define GF128_MUL(d0, d1, d2, d3, t0) \
-	li		t0,0x87;	/* multiplication in GF128	*/ \
-	cmpwi		d3,-1;						   \
-	iselgt		t0,0,t0;					   \
-	rlwimi		d3,d2,0,0,0;	/* propagate "carry" bits	*/ \
-	rotlwi		d3,d3,1;					   \
-	rlwimi		d2,d1,0,0,0;					   \
-	rotlwi		d2,d2,1;					   \
-	rlwimi		d1,d0,0,0,0;					   \
-	slwi		d0,d0,1;	/* shift left 128 bit		*/ \
-	rotlwi		d1,d1,1;					   \
-	xor		d0,d0,t0;
-
-#define START_KEY(d0, d1, d2, d3) \
-	lwz		rW0,0(rKP);					   \
-	mtctr		rRR;						   \
-	lwz		rW1,4(rKP);					   \
-	lwz		rW2,8(rKP);					   \
-	lwz		rW3,12(rKP);					   \
-	xor		rD0,d0,rW0;					   \
-	xor		rD1,d1,rW1;					   \
-	xor		rD2,d2,rW2;					   \
-	xor		rD3,d3,rW3;
-
-/*
- * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds)
- *
- * called from glue layer to encrypt a single 16 byte block
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_aes)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
-	LOAD_DATA(rD0, 0)
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds)
- *
- * called from glue layer to decrypt a single 16 byte block
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_aes)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
-	LOAD_DATA(rD0, 0)
-	addi		rT1,rT0,4096
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds, u32 bytes);
- *
- * called from glue layer to encrypt multiple blocks via ECB
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_ecb)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
-ppc_encrypt_ecb_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_ecb_loop
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 bytes);
- *
- * called from glue layer to decrypt multiple blocks via ECB
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_ecb)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
-	addi		rT1,rT0,4096
-ppc_decrypt_ecb_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_decrypt_ecb_loop
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
- *		   32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to encrypt multiple blocks via CBC
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_cbc)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	LOAD_IV(rI3, 12)
-ppc_encrypt_cbc_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	SAVE_DATA(rI0, 0)
-	xor		rI1,rD1,rW1
-	SAVE_DATA(rI1, 4)
-	xor		rI2,rD2,rW2
-	SAVE_DATA(rI2, 8)
-	xor		rI3,rD3,rW3
-	SAVE_DATA(rI3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_cbc_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to decrypt multiple blocks via CBC
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_cbc)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
-	li		rT1,15
-	LOAD_IV(rI0, 0)
-	andc		rLN,rLN,rT1
-	LOAD_IV(rI1, 4)
-	subi		rLN,rLN,16
-	LOAD_IV(rI2, 8)
-	add		rSP,rSP,rLN	/* reverse processing		*/
-	LOAD_IV(rI3, 12)
-	add		rDP,rDP,rLN
-	LOAD_DATA(rD0, 0)
-	addi		rT1,rT0,4096
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_IV
-	SAVE_IV(rD0, 0)
-	SAVE_IV(rD1, 4)
-	SAVE_IV(rD2, 8)
-	cmpwi		rLN,16
-	SAVE_IV(rD3, 12)
-	bt		lt,ppc_decrypt_cbc_end
-ppc_decrypt_cbc_loop:
-	mr		rKP,rKS
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	subi		rLN,rLN,16
-	subi		rSP,rSP,CBC_DEC
-	xor		rW0,rD0,rW0
-	LOAD_DATA(rD0, 0)
-	xor		rW1,rD1,rW1
-	LOAD_DATA(rD1, 4)
-	xor		rW2,rD2,rW2
-	LOAD_DATA(rD2, 8)
-	xor		rW3,rD3,rW3
-	LOAD_DATA(rD3, 12)
-	xor		rW0,rW0,rD0
-	SAVE_DATA(rW0, 0)
-	xor		rW1,rW1,rD1
-	SAVE_DATA(rW1, 4)
-	xor		rW2,rW2,rD2
-	SAVE_DATA(rW2, 8)
-	xor		rW3,rW3,rD3
-	SAVE_DATA(rW3, 12)
-	cmpwi		rLN,15
-	subi		rDP,rDP,CBC_DEC
-	bt		gt,ppc_decrypt_cbc_loop
-ppc_decrypt_cbc_end:
-	mr		rKP,rKS
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rW0,rW0,rD0
-	xor		rW1,rW1,rD1
-	xor		rW2,rW2,rD2
-	xor		rW3,rW3,rD3
-	xor		rW0,rW0,rI0	/* decrypt with initial IV	*/
-	SAVE_DATA(rW0, 0)
-	xor		rW1,rW1,rI1
-	SAVE_DATA(rW1, 4)
-	xor		rW2,rW2,rI2
-	SAVE_DATA(rW2, 8)
-	xor		rW3,rW3,rI3
-	SAVE_DATA(rW3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
- *		 u32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to encrypt/decrypt multiple blocks
- * via CTR. Number of bytes does not need to be a multiple of
- * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_crypt_ctr)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rLN,16
-	LOAD_IV(rI3, 12)
-	START_IV
-	bt		lt,ppc_crypt_ctr_partial
-ppc_crypt_ctr_loop:
-	mr		rKP,rKS
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rW0,rD0,rW0
-	xor		rW1,rD1,rW1
-	xor		rW2,rD2,rW2
-	xor		rW3,rD3,rW3
-	LOAD_DATA(rD0, 0)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	addic		rI3,rI3,1	/* increase counter			*/
-	addze		rI2,rI2
-	addze		rI1,rI1
-	addze		rI0,rI0
-	NEXT_BLOCK
-	cmpwi		rLN,15
-	bt		gt,ppc_crypt_ctr_loop
-ppc_crypt_ctr_partial:
-	cmpwi		rLN,0
-	bt		eq,ppc_crypt_ctr_end
-	mr		rKP,rKS
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rW0,rD0,rW0
-	SAVE_IV(rW0, 0)
-	xor		rW1,rD1,rW1
-	SAVE_IV(rW1, 4)
-	xor		rW2,rD2,rW2
-	SAVE_IV(rW2, 8)
-	xor		rW3,rD3,rW3
-	SAVE_IV(rW3, 12)
-	mtctr		rLN
-	subi		rIP,rIP,CTR_DEC
-	subi		rSP,rSP,1
-	subi		rDP,rDP,1
-ppc_crypt_ctr_xorbyte:
-	lbzu		rW4,1(rIP)	/* bytewise xor for partial block	*/
-	lbzu		rW5,1(rSP)
-	xor		rW4,rW4,rW5
-	stbu		rW4,1(rDP)
-	bdnz		ppc_crypt_ctr_xorbyte
-	subf		rIP,rLN,rIP
-	addi		rIP,rIP,1
-	addic		rI3,rI3,1
-	addze		rI2,rI2
-	addze		rI1,rI1
-	addze		rI0,rI0
-ppc_crypt_ctr_end:
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
- *
- * called from glue layer to encrypt multiple blocks via XTS
- * If key_twk is given, the initial IV encryption will be
- * processed too. Round values are AES128 = 4, AES192 = 5,
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_xts)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rKT,0
-	LOAD_IV(rI3, 12)
-	bt		eq,ppc_encrypt_xts_notweak
-	mr		rKP,rKT
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	xor		rI1,rD1,rW1
-	xor		rI2,rD2,rW2
-	xor		rI3,rD3,rW3
-ppc_encrypt_xts_notweak:
-	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
-	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
-ppc_encrypt_xts_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	xor		rD1,rD1,rW1
-	xor		rD2,rD2,rW2
-	xor		rD3,rD3,rW3
-	xor		rD0,rD0,rI0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rI1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rI2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rI3
-	SAVE_DATA(rD3, 12)
-	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
-	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
-	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
-	cmpwi		rLN,0
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_xts_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(8)
-	blr
-
-/*
- * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
- *
- * called from glue layer to decrypt multiple blocks via XTS
- * If key_twk is given, the initial IV encryption will be
- * processed too. Round values are AES128 = 4, AES192 = 5,
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_xts)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
-	LOAD_IV(rI0, 0)
-	addi		rT1,rT0,4096
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rKT,0
-	LOAD_IV(rI3, 12)
-	bt		eq,ppc_decrypt_xts_notweak
-	subi		rT0,rT0,4096
-	mr		rKP,rKT
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	xor		rI1,rD1,rW1
-	xor		rI2,rD2,rW2
-	xor		rI3,rD3,rW3
-	addi		rT0,rT0,4096
-ppc_decrypt_xts_notweak:
-	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
-	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
-ppc_decrypt_xts_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	xor		rD1,rD1,rW1
-	xor		rD2,rD2,rW2
-	xor		rD3,rD3,rW3
-	xor		rD0,rD0,rI0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rI1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rI2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rI3
-	SAVE_DATA(rD3, 12)
-	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
-	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
-	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
-	cmpwi		rLN,0
-	NEXT_BLOCK
-	bt		gt,ppc_decrypt_xts_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(8)
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-regs.h b/arch/powerpc/crypto/aes-spe-regs.h
deleted file mode 100644
index 2eb4c9b94152..000000000000
--- a/arch/powerpc/crypto/aes-spe-regs.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Common registers for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#define rKS r0	/* copy of en-/decryption key pointer			*/
-#define rDP r3	/* destination pointer					*/
-#define rSP r4	/* source pointer					*/
-#define rKP r5	/* pointer to en-/decryption key pointer		*/
-#define rRR r6	/* en-/decryption rounds				*/
-#define rLN r7	/* length of data to be processed			*/
-#define rIP r8	/* potiner to IV (CBC/CTR/XTS modes)			*/
-#define rKT r9	/* pointer to tweak key (XTS mode)			*/
-#define rT0 r11	/* pointers to en-/decryption tables			*/
-#define rT1 r10
-#define rD0 r9	/* data 						*/
-#define rD1 r14
-#define rD2 r12
-#define rD3 r15
-#define rW0 r16	/* working registers					*/
-#define rW1 r17
-#define rW2 r18
-#define rW3 r19
-#define rW4 r20
-#define rW5 r21
-#define rW6 r22
-#define rW7 r23
-#define rI0 r24	/* IV							*/
-#define rI1 r25
-#define rI2 r26
-#define rI3 r27
-#define rG0 r28	/* endian reversed tweak (XTS mode)			*/
-#define rG1 r29
-#define rG2 r30
-#define rG3 r31
diff --git a/arch/powerpc/crypto/aes-tab-4k.S b/arch/powerpc/crypto/aes-tab-4k.S
deleted file mode 100644
index ceb604bc6f72..000000000000
--- a/arch/powerpc/crypto/aes-tab-4k.S
+++ /dev/null
@@ -1,326 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * 4K AES tables for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-/*
- * These big endian AES encryption/decryption tables have been taken from
- * crypto/aes_generic.c and are designed to be simply accessed by a combination
- * of rlwimi/lwz instructions with a minimum of table registers (usually only
- * one required). Thus they are aligned to 4K. The locality of rotated values
- * is derived from the reduced offsets that are available in the SPE load
- * instructions. E.g. evldw, evlwwsplat, ...
- *
- * For the safety-conscious it has to be noted that they might be vulnerable
- * to cache timing attacks because of their size. Nevertheless in contrast to
- * the generic tables they have been reduced from 16KB to 8KB + 256 bytes.
- * This is a quite good tradeoff for low power devices (e.g. routers) without
- * dedicated encryption hardware where we usually have no multiuser
- * environment.
- *
- */
-
-#define R(a, b, c, d) \
-	0x##a##b##c##d, 0x##d##a##b##c, 0x##c##d##a##b, 0x##b##c##d##a
-
-.data
-.align 12
-.globl PPC_AES_4K_ENCTAB
-PPC_AES_4K_ENCTAB:
-/* encryption table, same as crypto_ft_tab in crypto/aes-generic.c */
-	.long R(c6, 63, 63, a5), R(f8, 7c, 7c, 84)
-	.long R(ee, 77, 77, 99), R(f6, 7b, 7b, 8d)
-	.long R(ff, f2, f2, 0d), R(d6, 6b, 6b, bd)
-	.long R(de, 6f, 6f, b1), R(91, c5, c5, 54)
-	.long R(60, 30, 30, 50), R(02, 01, 01, 03)
-	.long R(ce, 67, 67, a9), R(56, 2b, 2b, 7d)
-	.long R(e7, fe, fe, 19), R(b5, d7, d7, 62)
-	.long R(4d, ab, ab, e6), R(ec, 76, 76, 9a)
-	.long R(8f, ca, ca, 45), R(1f, 82, 82, 9d)
-	.long R(89, c9, c9, 40), R(fa, 7d, 7d, 87)
-	.long R(ef, fa, fa, 15), R(b2, 59, 59, eb)
-	.long R(8e, 47, 47, c9), R(fb, f0, f0, 0b)
-	.long R(41, ad, ad, ec), R(b3, d4, d4, 67)
-	.long R(5f, a2, a2, fd), R(45, af, af, ea)
-	.long R(23, 9c, 9c, bf), R(53, a4, a4, f7)
-	.long R(e4, 72, 72, 96), R(9b, c0, c0, 5b)
-	.long R(75, b7, b7, c2), R(e1, fd, fd, 1c)
-	.long R(3d, 93, 93, ae), R(4c, 26, 26, 6a)
-	.long R(6c, 36, 36, 5a), R(7e, 3f, 3f, 41)
-	.long R(f5, f7, f7, 02), R(83, cc, cc, 4f)
-	.long R(68, 34, 34, 5c), R(51, a5, a5, f4)
-	.long R(d1, e5, e5, 34), R(f9, f1, f1, 08)
-	.long R(e2, 71, 71, 93), R(ab, d8, d8, 73)
-	.long R(62, 31, 31, 53), R(2a, 15, 15, 3f)
-	.long R(08, 04, 04, 0c), R(95, c7, c7, 52)
-	.long R(46, 23, 23, 65), R(9d, c3, c3, 5e)
-	.long R(30, 18, 18, 28), R(37, 96, 96, a1)
-	.long R(0a, 05, 05, 0f), R(2f, 9a, 9a, b5)
-	.long R(0e, 07, 07, 09), R(24, 12, 12, 36)
-	.long R(1b, 80, 80, 9b), R(df, e2, e2, 3d)
-	.long R(cd, eb, eb, 26), R(4e, 27, 27, 69)
-	.long R(7f, b2, b2, cd), R(ea, 75, 75, 9f)
-	.long R(12, 09, 09, 1b), R(1d, 83, 83, 9e)
-	.long R(58, 2c, 2c, 74), R(34, 1a, 1a, 2e)
-	.long R(36, 1b, 1b, 2d), R(dc, 6e, 6e, b2)
-	.long R(b4, 5a, 5a, ee), R(5b, a0, a0, fb)
-	.long R(a4, 52, 52, f6), R(76, 3b, 3b, 4d)
-	.long R(b7, d6, d6, 61), R(7d, b3, b3, ce)
-	.long R(52, 29, 29, 7b), R(dd, e3, e3, 3e)
-	.long R(5e, 2f, 2f, 71), R(13, 84, 84, 97)
-	.long R(a6, 53, 53, f5), R(b9, d1, d1, 68)
-	.long R(00, 00, 00, 00), R(c1, ed, ed, 2c)
-	.long R(40, 20, 20, 60), R(e3, fc, fc, 1f)
-	.long R(79, b1, b1, c8), R(b6, 5b, 5b, ed)
-	.long R(d4, 6a, 6a, be), R(8d, cb, cb, 46)
-	.long R(67, be, be, d9), R(72, 39, 39, 4b)
-	.long R(94, 4a, 4a, de), R(98, 4c, 4c, d4)
-	.long R(b0, 58, 58, e8), R(85, cf, cf, 4a)
-	.long R(bb, d0, d0, 6b), R(c5, ef, ef, 2a)
-	.long R(4f, aa, aa, e5), R(ed, fb, fb, 16)
-	.long R(86, 43, 43, c5), R(9a, 4d, 4d, d7)
-	.long R(66, 33, 33, 55), R(11, 85, 85, 94)
-	.long R(8a, 45, 45, cf), R(e9, f9, f9, 10)
-	.long R(04, 02, 02, 06), R(fe, 7f, 7f, 81)
-	.long R(a0, 50, 50, f0), R(78, 3c, 3c, 44)
-	.long R(25, 9f, 9f, ba), R(4b, a8, a8, e3)
-	.long R(a2, 51, 51, f3), R(5d, a3, a3, fe)
-	.long R(80, 40, 40, c0), R(05, 8f, 8f, 8a)
-	.long R(3f, 92, 92, ad), R(21, 9d, 9d, bc)
-	.long R(70, 38, 38, 48), R(f1, f5, f5, 04)
-	.long R(63, bc, bc, df), R(77, b6, b6, c1)
-	.long R(af, da, da, 75), R(42, 21, 21, 63)
-	.long R(20, 10, 10, 30), R(e5, ff, ff, 1a)
-	.long R(fd, f3, f3, 0e), R(bf, d2, d2, 6d)
-	.long R(81, cd, cd, 4c), R(18, 0c, 0c, 14)
-	.long R(26, 13, 13, 35), R(c3, ec, ec, 2f)
-	.long R(be, 5f, 5f, e1), R(35, 97, 97, a2)
-	.long R(88, 44, 44, cc), R(2e, 17, 17, 39)
-	.long R(93, c4, c4, 57), R(55, a7, a7, f2)
-	.long R(fc, 7e, 7e, 82), R(7a, 3d, 3d, 47)
-	.long R(c8, 64, 64, ac), R(ba, 5d, 5d, e7)
-	.long R(32, 19, 19, 2b), R(e6, 73, 73, 95)
-	.long R(c0, 60, 60, a0), R(19, 81, 81, 98)
-	.long R(9e, 4f, 4f, d1), R(a3, dc, dc, 7f)
-	.long R(44, 22, 22, 66), R(54, 2a, 2a, 7e)
-	.long R(3b, 90, 90, ab), R(0b, 88, 88, 83)
-	.long R(8c, 46, 46, ca), R(c7, ee, ee, 29)
-	.long R(6b, b8, b8, d3), R(28, 14, 14, 3c)
-	.long R(a7, de, de, 79), R(bc, 5e, 5e, e2)
-	.long R(16, 0b, 0b, 1d), R(ad, db, db, 76)
-	.long R(db, e0, e0, 3b), R(64, 32, 32, 56)
-	.long R(74, 3a, 3a, 4e), R(14, 0a, 0a, 1e)
-	.long R(92, 49, 49, db), R(0c, 06, 06, 0a)
-	.long R(48, 24, 24, 6c), R(b8, 5c, 5c, e4)
-	.long R(9f, c2, c2, 5d), R(bd, d3, d3, 6e)
-	.long R(43, ac, ac, ef), R(c4, 62, 62, a6)
-	.long R(39, 91, 91, a8), R(31, 95, 95, a4)
-	.long R(d3, e4, e4, 37), R(f2, 79, 79, 8b)
-	.long R(d5, e7, e7, 32), R(8b, c8, c8, 43)
-	.long R(6e, 37, 37, 59), R(da, 6d, 6d, b7)
-	.long R(01, 8d, 8d, 8c), R(b1, d5, d5, 64)
-	.long R(9c, 4e, 4e, d2), R(49, a9, a9, e0)
-	.long R(d8, 6c, 6c, b4), R(ac, 56, 56, fa)
-	.long R(f3, f4, f4, 07), R(cf, ea, ea, 25)
-	.long R(ca, 65, 65, af), R(f4, 7a, 7a, 8e)
-	.long R(47, ae, ae, e9), R(10, 08, 08, 18)
-	.long R(6f, ba, ba, d5), R(f0, 78, 78, 88)
-	.long R(4a, 25, 25, 6f), R(5c, 2e, 2e, 72)
-	.long R(38, 1c, 1c, 24), R(57, a6, a6, f1)
-	.long R(73, b4, b4, c7), R(97, c6, c6, 51)
-	.long R(cb, e8, e8, 23), R(a1, dd, dd, 7c)
-	.long R(e8, 74, 74, 9c), R(3e, 1f, 1f, 21)
-	.long R(96, 4b, 4b, dd), R(61, bd, bd, dc)
-	.long R(0d, 8b, 8b, 86), R(0f, 8a, 8a, 85)
-	.long R(e0, 70, 70, 90), R(7c, 3e, 3e, 42)
-	.long R(71, b5, b5, c4), R(cc, 66, 66, aa)
-	.long R(90, 48, 48, d8), R(06, 03, 03, 05)
-	.long R(f7, f6, f6, 01), R(1c, 0e, 0e, 12)
-	.long R(c2, 61, 61, a3), R(6a, 35, 35, 5f)
-	.long R(ae, 57, 57, f9), R(69, b9, b9, d0)
-	.long R(17, 86, 86, 91), R(99, c1, c1, 58)
-	.long R(3a, 1d, 1d, 27), R(27, 9e, 9e, b9)
-	.long R(d9, e1, e1, 38), R(eb, f8, f8, 13)
-	.long R(2b, 98, 98, b3), R(22, 11, 11, 33)
-	.long R(d2, 69, 69, bb), R(a9, d9, d9, 70)
-	.long R(07, 8e, 8e, 89), R(33, 94, 94, a7)
-	.long R(2d, 9b, 9b, b6), R(3c, 1e, 1e, 22)
-	.long R(15, 87, 87, 92), R(c9, e9, e9, 20)
-	.long R(87, ce, ce, 49), R(aa, 55, 55, ff)
-	.long R(50, 28, 28, 78), R(a5, df, df, 7a)
-	.long R(03, 8c, 8c, 8f), R(59, a1, a1, f8)
-	.long R(09, 89, 89, 80), R(1a, 0d, 0d, 17)
-	.long R(65, bf, bf, da), R(d7, e6, e6, 31)
-	.long R(84, 42, 42, c6), R(d0, 68, 68, b8)
-	.long R(82, 41, 41, c3), R(29, 99, 99, b0)
-	.long R(5a, 2d, 2d, 77), R(1e, 0f, 0f, 11)
-	.long R(7b, b0, b0, cb), R(a8, 54, 54, fc)
-	.long R(6d, bb, bb, d6), R(2c, 16, 16, 3a)
-.globl PPC_AES_4K_DECTAB
-PPC_AES_4K_DECTAB:
-/* decryption table, same as crypto_it_tab in crypto/aes-generic.c */
-	.long R(51, f4, a7, 50), R(7e, 41, 65, 53)
-	.long R(1a, 17, a4, c3), R(3a, 27, 5e, 96)
-	.long R(3b, ab, 6b, cb), R(1f, 9d, 45, f1)
-	.long R(ac, fa, 58, ab), R(4b, e3, 03, 93)
-	.long R(20, 30, fa, 55), R(ad, 76, 6d, f6)
-	.long R(88, cc, 76, 91), R(f5, 02, 4c, 25)
-	.long R(4f, e5, d7, fc), R(c5, 2a, cb, d7)
-	.long R(26, 35, 44, 80), R(b5, 62, a3, 8f)
-	.long R(de, b1, 5a, 49), R(25, ba, 1b, 67)
-	.long R(45, ea, 0e, 98), R(5d, fe, c0, e1)
-	.long R(c3, 2f, 75, 02), R(81, 4c, f0, 12)
-	.long R(8d, 46, 97, a3), R(6b, d3, f9, c6)
-	.long R(03, 8f, 5f, e7), R(15, 92, 9c, 95)
-	.long R(bf, 6d, 7a, eb), R(95, 52, 59, da)
-	.long R(d4, be, 83, 2d), R(58, 74, 21, d3)
-	.long R(49, e0, 69, 29), R(8e, c9, c8, 44)
-	.long R(75, c2, 89, 6a), R(f4, 8e, 79, 78)
-	.long R(99, 58, 3e, 6b), R(27, b9, 71, dd)
-	.long R(be, e1, 4f, b6), R(f0, 88, ad, 17)
-	.long R(c9, 20, ac, 66), R(7d, ce, 3a, b4)
-	.long R(63, df, 4a, 18), R(e5, 1a, 31, 82)
-	.long R(97, 51, 33, 60), R(62, 53, 7f, 45)
-	.long R(b1, 64, 77, e0), R(bb, 6b, ae, 84)
-	.long R(fe, 81, a0, 1c), R(f9, 08, 2b, 94)
-	.long R(70, 48, 68, 58), R(8f, 45, fd, 19)
-	.long R(94, de, 6c, 87), R(52, 7b, f8, b7)
-	.long R(ab, 73, d3, 23), R(72, 4b, 02, e2)
-	.long R(e3, 1f, 8f, 57), R(66, 55, ab, 2a)
-	.long R(b2, eb, 28, 07), R(2f, b5, c2, 03)
-	.long R(86, c5, 7b, 9a), R(d3, 37, 08, a5)
-	.long R(30, 28, 87, f2), R(23, bf, a5, b2)
-	.long R(02, 03, 6a, ba), R(ed, 16, 82, 5c)
-	.long R(8a, cf, 1c, 2b), R(a7, 79, b4, 92)
-	.long R(f3, 07, f2, f0), R(4e, 69, e2, a1)
-	.long R(65, da, f4, cd), R(06, 05, be, d5)
-	.long R(d1, 34, 62, 1f), R(c4, a6, fe, 8a)
-	.long R(34, 2e, 53, 9d), R(a2, f3, 55, a0)
-	.long R(05, 8a, e1, 32), R(a4, f6, eb, 75)
-	.long R(0b, 83, ec, 39), R(40, 60, ef, aa)
-	.long R(5e, 71, 9f, 06), R(bd, 6e, 10, 51)
-	.long R(3e, 21, 8a, f9), R(96, dd, 06, 3d)
-	.long R(dd, 3e, 05, ae), R(4d, e6, bd, 46)
-	.long R(91, 54, 8d, b5), R(71, c4, 5d, 05)
-	.long R(04, 06, d4, 6f), R(60, 50, 15, ff)
-	.long R(19, 98, fb, 24), R(d6, bd, e9, 97)
-	.long R(89, 40, 43, cc), R(67, d9, 9e, 77)
-	.long R(b0, e8, 42, bd), R(07, 89, 8b, 88)
-	.long R(e7, 19, 5b, 38), R(79, c8, ee, db)
-	.long R(a1, 7c, 0a, 47), R(7c, 42, 0f, e9)
-	.long R(f8, 84, 1e, c9), R(00, 00, 00, 00)
-	.long R(09, 80, 86, 83), R(32, 2b, ed, 48)
-	.long R(1e, 11, 70, ac), R(6c, 5a, 72, 4e)
-	.long R(fd, 0e, ff, fb), R(0f, 85, 38, 56)
-	.long R(3d, ae, d5, 1e), R(36, 2d, 39, 27)
-	.long R(0a, 0f, d9, 64), R(68, 5c, a6, 21)
-	.long R(9b, 5b, 54, d1), R(24, 36, 2e, 3a)
-	.long R(0c, 0a, 67, b1), R(93, 57, e7, 0f)
-	.long R(b4, ee, 96, d2), R(1b, 9b, 91, 9e)
-	.long R(80, c0, c5, 4f), R(61, dc, 20, a2)
-	.long R(5a, 77, 4b, 69), R(1c, 12, 1a, 16)
-	.long R(e2, 93, ba, 0a), R(c0, a0, 2a, e5)
-	.long R(3c, 22, e0, 43), R(12, 1b, 17, 1d)
-	.long R(0e, 09, 0d, 0b), R(f2, 8b, c7, ad)
-	.long R(2d, b6, a8, b9), R(14, 1e, a9, c8)
-	.long R(57, f1, 19, 85), R(af, 75, 07, 4c)
-	.long R(ee, 99, dd, bb), R(a3, 7f, 60, fd)
-	.long R(f7, 01, 26, 9f), R(5c, 72, f5, bc)
-	.long R(44, 66, 3b, c5), R(5b, fb, 7e, 34)
-	.long R(8b, 43, 29, 76), R(cb, 23, c6, dc)
-	.long R(b6, ed, fc, 68), R(b8, e4, f1, 63)
-	.long R(d7, 31, dc, ca), R(42, 63, 85, 10)
-	.long R(13, 97, 22, 40), R(84, c6, 11, 20)
-	.long R(85, 4a, 24, 7d), R(d2, bb, 3d, f8)
-	.long R(ae, f9, 32, 11), R(c7, 29, a1, 6d)
-	.long R(1d, 9e, 2f, 4b), R(dc, b2, 30, f3)
-	.long R(0d, 86, 52, ec), R(77, c1, e3, d0)
-	.long R(2b, b3, 16, 6c), R(a9, 70, b9, 99)
-	.long R(11, 94, 48, fa), R(47, e9, 64, 22)
-	.long R(a8, fc, 8c, c4), R(a0, f0, 3f, 1a)
-	.long R(56, 7d, 2c, d8), R(22, 33, 90, ef)
-	.long R(87, 49, 4e, c7), R(d9, 38, d1, c1)
-	.long R(8c, ca, a2, fe), R(98, d4, 0b, 36)
-	.long R(a6, f5, 81, cf), R(a5, 7a, de, 28)
-	.long R(da, b7, 8e, 26), R(3f, ad, bf, a4)
-	.long R(2c, 3a, 9d, e4), R(50, 78, 92, 0d)
-	.long R(6a, 5f, cc, 9b), R(54, 7e, 46, 62)
-	.long R(f6, 8d, 13, c2), R(90, d8, b8, e8)
-	.long R(2e, 39, f7, 5e), R(82, c3, af, f5)
-	.long R(9f, 5d, 80, be), R(69, d0, 93, 7c)
-	.long R(6f, d5, 2d, a9), R(cf, 25, 12, b3)
-	.long R(c8, ac, 99, 3b), R(10, 18, 7d, a7)
-	.long R(e8, 9c, 63, 6e), R(db, 3b, bb, 7b)
-	.long R(cd, 26, 78, 09), R(6e, 59, 18, f4)
-	.long R(ec, 9a, b7, 01), R(83, 4f, 9a, a8)
-	.long R(e6, 95, 6e, 65), R(aa, ff, e6, 7e)
-	.long R(21, bc, cf, 08), R(ef, 15, e8, e6)
-	.long R(ba, e7, 9b, d9), R(4a, 6f, 36, ce)
-	.long R(ea, 9f, 09, d4), R(29, b0, 7c, d6)
-	.long R(31, a4, b2, af), R(2a, 3f, 23, 31)
-	.long R(c6, a5, 94, 30), R(35, a2, 66, c0)
-	.long R(74, 4e, bc, 37), R(fc, 82, ca, a6)
-	.long R(e0, 90, d0, b0), R(33, a7, d8, 15)
-	.long R(f1, 04, 98, 4a), R(41, ec, da, f7)
-	.long R(7f, cd, 50, 0e), R(17, 91, f6, 2f)
-	.long R(76, 4d, d6, 8d), R(43, ef, b0, 4d)
-	.long R(cc, aa, 4d, 54), R(e4, 96, 04, df)
-	.long R(9e, d1, b5, e3), R(4c, 6a, 88, 1b)
-	.long R(c1, 2c, 1f, b8), R(46, 65, 51, 7f)
-	.long R(9d, 5e, ea, 04), R(01, 8c, 35, 5d)
-	.long R(fa, 87, 74, 73), R(fb, 0b, 41, 2e)
-	.long R(b3, 67, 1d, 5a), R(92, db, d2, 52)
-	.long R(e9, 10, 56, 33), R(6d, d6, 47, 13)
-	.long R(9a, d7, 61, 8c), R(37, a1, 0c, 7a)
-	.long R(59, f8, 14, 8e), R(eb, 13, 3c, 89)
-	.long R(ce, a9, 27, ee), R(b7, 61, c9, 35)
-	.long R(e1, 1c, e5, ed), R(7a, 47, b1, 3c)
-	.long R(9c, d2, df, 59), R(55, f2, 73, 3f)
-	.long R(18, 14, ce, 79), R(73, c7, 37, bf)
-	.long R(53, f7, cd, ea), R(5f, fd, aa, 5b)
-	.long R(df, 3d, 6f, 14), R(78, 44, db, 86)
-	.long R(ca, af, f3, 81), R(b9, 68, c4, 3e)
-	.long R(38, 24, 34, 2c), R(c2, a3, 40, 5f)
-	.long R(16, 1d, c3, 72), R(bc, e2, 25, 0c)
-	.long R(28, 3c, 49, 8b), R(ff, 0d, 95, 41)
-	.long R(39, a8, 01, 71), R(08, 0c, b3, de)
-	.long R(d8, b4, e4, 9c), R(64, 56, c1, 90)
-	.long R(7b, cb, 84, 61), R(d5, 32, b6, 70)
-	.long R(48, 6c, 5c, 74), R(d0, b8, 57, 42)
-.globl PPC_AES_4K_DECTAB2
-PPC_AES_4K_DECTAB2:
-/* decryption table, same as crypto_il_tab in crypto/aes-generic.c */
-	.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
-	.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
-	.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
-	.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
-	.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
-	.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
-	.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
-	.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
-	.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
-	.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
-	.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
-	.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
-	.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
-	.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
-	.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
-	.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
-	.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
-	.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
-	.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
-	.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
-	.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
-	.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
-	.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
-	.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
-	.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
-	.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
-	.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
-	.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
-	.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
-	.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
-	.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
-	.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
diff --git a/arch/powerpc/crypto/aes.c b/arch/powerpc/crypto/aes.c
deleted file mode 100644
index 3f1e5e894902..000000000000
--- a/arch/powerpc/crypto/aes.c
+++ /dev/null
@@ -1,134 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * AES routines supporting VMX instructions on the Power 8
- *
- * Copyright (C) 2015 International Business Machines Inc.
- *
- * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
- */
-
-#include <asm/simd.h>
-#include <asm/switch_to.h>
-#include <crypto/aes.h>
-#include <crypto/internal/cipher.h>
-#include <crypto/internal/simd.h>
-#include <linux/err.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/uaccess.h>
-
-#include "aesp8-ppc.h"
-
-struct p8_aes_ctx {
-	struct crypto_cipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
-};
-
-static int p8_aes_init(struct crypto_tfm *tfm)
-{
-	const char *alg = crypto_tfm_alg_name(tfm);
-	struct crypto_cipher *fallback;
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
-	if (IS_ERR(fallback)) {
-		printk(KERN_ERR
-		       "Failed to allocate transformation for '%s': %ld\n",
-		       alg, PTR_ERR(fallback));
-		return PTR_ERR(fallback);
-	}
-
-	crypto_cipher_set_flags(fallback,
-				crypto_cipher_get_flags((struct
-							 crypto_cipher *)
-							tfm));
-	ctx->fallback = fallback;
-
-	return 0;
-}
-
-static void p8_aes_exit(struct crypto_tfm *tfm)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (ctx->fallback) {
-		crypto_free_cipher(ctx->fallback);
-		ctx->fallback = NULL;
-	}
-}
-
-static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,
-			 unsigned int keylen)
-{
-	int ret;
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	preempt_disable();
-	pagefault_disable();
-	enable_kernel_vsx();
-	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
-	ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
-	disable_kernel_vsx();
-	pagefault_enable();
-	preempt_enable();
-
-	ret |= crypto_cipher_setkey(ctx->fallback, key, keylen);
-
-	return ret ? -EINVAL : 0;
-}
-
-static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		crypto_cipher_encrypt_one(ctx->fallback, dst, src);
-	} else {
-		preempt_disable();
-		pagefault_disable();
-		enable_kernel_vsx();
-		aes_p8_encrypt(src, dst, &ctx->enc_key);
-		disable_kernel_vsx();
-		pagefault_enable();
-		preempt_enable();
-	}
-}
-
-static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		crypto_cipher_decrypt_one(ctx->fallback, dst, src);
-	} else {
-		preempt_disable();
-		pagefault_disable();
-		enable_kernel_vsx();
-		aes_p8_decrypt(src, dst, &ctx->dec_key);
-		disable_kernel_vsx();
-		pagefault_enable();
-		preempt_enable();
-	}
-}
-
-struct crypto_alg p8_aes_alg = {
-	.cra_name = "aes",
-	.cra_driver_name = "p8_aes",
-	.cra_module = THIS_MODULE,
-	.cra_priority = 1000,
-	.cra_type = NULL,
-	.cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK,
-	.cra_alignmask = 0,
-	.cra_blocksize = AES_BLOCK_SIZE,
-	.cra_ctxsize = sizeof(struct p8_aes_ctx),
-	.cra_init = p8_aes_init,
-	.cra_exit = p8_aes_exit,
-	.cra_cipher = {
-		       .cia_min_keysize = AES_MIN_KEY_SIZE,
-		       .cia_max_keysize = AES_MAX_KEY_SIZE,
-		       .cia_setkey = p8_aes_setkey,
-		       .cia_encrypt = p8_aes_encrypt,
-		       .cia_decrypt = p8_aes_decrypt,
-	},
-};
diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c
index 5f2a4f375eef..4a9f285f0970 100644
--- a/arch/powerpc/crypto/aes_cbc.c
+++ b/arch/powerpc/crypto/aes_cbc.c
@@ -21,8 +21,8 @@
 
 struct p8_aes_cbc_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
+	struct p8_aes_key enc_key;
+	struct p8_aes_key dec_key;
 };
 
 static int p8_aes_cbc_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c
index e27c4036e711..7dbd06f442db 100644
--- a/arch/powerpc/crypto/aes_ctr.c
+++ b/arch/powerpc/crypto/aes_ctr.c
@@ -21,7 +21,7 @@
 
 struct p8_aes_ctr_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
+	struct p8_aes_key enc_key;
 };
 
 static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aes_xts.c b/arch/powerpc/crypto/aes_xts.c
index 9440e771cede..b4c760e465ea 100644
--- a/arch/powerpc/crypto/aes_xts.c
+++ b/arch/powerpc/crypto/aes_xts.c
@@ -22,9 +22,9 @@
 
 struct p8_aes_xts_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
-	struct aes_key tweak_key;
+	struct p8_aes_key enc_key;
+	struct p8_aes_key dec_key;
+	struct p8_aes_key tweak_key;
 };
 
 static int p8_aes_xts_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aesp8-ppc.h b/arch/powerpc/crypto/aesp8-ppc.h
index 5764d4438388..6862c605cc33 100644
--- a/arch/powerpc/crypto/aesp8-ppc.h
+++ b/arch/powerpc/crypto/aesp8-ppc.h
@@ -2,29 +2,7 @@
 #include <linux/types.h>
 #include <crypto/aes.h>
 
-struct aes_key {
-	u8 key[AES_MAX_KEYLENGTH];
-	int rounds;
-};
-
 extern struct shash_alg p8_ghash_alg;
-extern struct crypto_alg p8_aes_alg;
 extern struct skcipher_alg p8_aes_cbc_alg;
 extern struct skcipher_alg p8_aes_ctr_alg;
 extern struct skcipher_alg p8_aes_xts_alg;
-
-int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,
-			   struct aes_key *key);
-int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
-			   struct aes_key *key);
-void aes_p8_encrypt(const u8 *in, u8 *out, const struct aes_key *key);
-void aes_p8_decrypt(const u8 *in, u8 *out, const struct aes_key *key);
-void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key, u8 *iv, const int enc);
-void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out,
-				 size_t len, const struct aes_key *key,
-				 const u8 *iv);
-void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
-void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
diff --git a/arch/powerpc/crypto/aesp8-ppc.pl b/arch/powerpc/crypto/aesp8-ppc.pl
deleted file mode 100644
index f729589d792e..000000000000
--- a/arch/powerpc/crypto/aesp8-ppc.pl
+++ /dev/null
@@ -1,3889 +0,0 @@
-#! /usr/bin/env perl
-# SPDX-License-Identifier: GPL-2.0
-
-# This code is taken from CRYPTOGAMs[1] and is included here using the option
-# in the license to distribute the code under the GPL. Therefore this program
-# is free software; you can redistribute it and/or modify it under the terms of
-# the GNU General Public License version 2 as published by the Free Software
-# Foundation.
-#
-# [1] https://www.openssl.org/~appro/cryptogams/
-
-# Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-#       * Redistributions of source code must retain copyright notices,
-#         this list of conditions and the following disclaimer.
-#
-#       * Redistributions in binary form must reproduce the above
-#         copyright notice, this list of conditions and the following
-#         disclaimer in the documentation and/or other materials
-#         provided with the distribution.
-#
-#       * Neither the name of the CRYPTOGAMS nor the names of its
-#         copyright holder and contributors may be used to endorse or
-#         promote products derived from this software without specific
-#         prior written permission.
-#
-# ALTERNATIVELY, provided that this notice is retained in full, this
-# product may be distributed under the terms of the GNU General Public
-# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
-# those given above.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see https://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# This module implements support for AES instructions as per PowerISA
-# specification version 2.07, first implemented by POWER8 processor.
-# The module is endian-agnostic in sense that it supports both big-
-# and little-endian cases. Data alignment in parallelizable modes is
-# handled with VSX loads and stores, which implies MSR.VSX flag being
-# set. It should also be noted that ISA specification doesn't prohibit
-# alignment exceptions for these instructions on page boundaries.
-# Initially alignment was handled in pure AltiVec/VMX way [when data
-# is aligned programmatically, which in turn guarantees exception-
-# free execution], but it turned to hamper performance when vcipher
-# instructions are interleaved. It's reckoned that eventual
-# misalignment penalties at page boundaries are in average lower
-# than additional overhead in pure AltiVec approach.
-#
-# May 2016
-#
-# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
-# systems were measured.
-#
-######################################################################
-# Current large-block performance in cycles per byte processed with
-# 128-bit key (less is better).
-#
-#		CBC en-/decrypt	CTR	XTS
-# POWER8[le]	3.96/0.72	0.74	1.1
-# POWER8[be]	3.75/0.65	0.66	1.0
-
-$flavour = shift;
-
-if ($flavour =~ /64/) {
-	$SIZE_T	=8;
-	$LRSAVE	=2*$SIZE_T;
-	$STU	="stdu";
-	$POP	="ld";
-	$PUSH	="std";
-	$UCMP	="cmpld";
-	$SHL	="sldi";
-} elsif ($flavour =~ /32/) {
-	$SIZE_T	=4;
-	$LRSAVE	=$SIZE_T;
-	$STU	="stwu";
-	$POP	="lwz";
-	$PUSH	="stw";
-	$UCMP	="cmplw";
-	$SHL	="slwi";
-} else { die "nonsense $flavour"; }
-
-$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
-
-$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
-( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
-die "can't locate ppc-xlate.pl";
-
-open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
-
-$FRAME=8*$SIZE_T;
-$prefix="aes_p8";
-
-$sp="r1";
-$vrsave="r12";
-
-#########################################################################
-{{{	# Key setup procedures						#
-my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
-my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
-my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
-
-$code.=<<___;
-.machine	"any"
-
-.text
-
-.align	7
-rcon:
-.long	0x01000000, 0x01000000, 0x01000000, 0x01000000	?rev
-.long	0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000	?rev
-.long	0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c	?rev
-.long	0,0,0,0						?asis
-.long	0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe
-Lconsts:
-	mflr	r0
-	bcl	20,31,\$+4
-	mflr	$ptr	 #vvvvv "distance between . and rcon
-	addi	$ptr,$ptr,-0x58
-	mtlr	r0
-	blr
-	.long	0
-	.byte	0,12,0x14,0,0,0,0,0
-.asciz	"AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
-
-.globl	.${prefix}_set_encrypt_key
-Lset_encrypt_key:
-	mflr		r11
-	$PUSH		r11,$LRSAVE($sp)
-
-	li		$ptr,-1
-	${UCMP}i	$inp,0
-	beq-		Lenc_key_abort		# if ($inp==0) return -1;
-	${UCMP}i	$out,0
-	beq-		Lenc_key_abort		# if ($out==0) return -1;
-	li		$ptr,-2
-	cmpwi		$bits,128
-	blt-		Lenc_key_abort
-	cmpwi		$bits,256
-	bgt-		Lenc_key_abort
-	andi.		r0,$bits,0x3f
-	bne-		Lenc_key_abort
-
-	lis		r0,0xfff0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	bl		Lconsts
-	mtlr		r11
-
-	neg		r9,$inp
-	lvx		$in0,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	lvsr		$key,0,r9		# borrow $key
-	li		r8,0x20
-	cmpwi		$bits,192
-	lvx		$in1,0,$inp
-	le?vspltisb	$mask,0x0f		# borrow $mask
-	lvx		$rcon,0,$ptr
-	le?vxor		$key,$key,$mask		# adjust for byte swap
-	lvx		$mask,r8,$ptr
-	addi		$ptr,$ptr,0x10
-	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]
-	li		$cnt,8
-	vxor		$zero,$zero,$zero
-	mtctr		$cnt
-
-	?lvsr		$outperm,0,$out
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$zero,$outmask,$outperm
-
-	blt		Loop128
-	addi		$inp,$inp,8
-	beq		L192
-	addi		$inp,$inp,8
-	b		L256
-
-.align	4
-Loop128:
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-	bdnz		Loop128
-
-	lvx		$rcon,0,$ptr		# last two round keys
-
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vxor		$in0,$in0,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-
-	addi		$inp,$out,15		# 15 is not typo
-	addi		$out,$out,0x50
-
-	li		$rounds,10
-	b		Ldone
-
-.align	4
-L192:
-	lvx		$tmp,0,$inp
-	li		$cnt,4
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
-	vspltisb	$key,8			# borrow $key
-	mtctr		$cnt
-	vsububm		$mask,$mask,$key	# adjust the mask
-
-Loop192:
-	vperm		$key,$in1,$in1,$mask	# roate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	vcipherlast	$key,$key,$rcon
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-
-	 vsldoi		$stage,$zero,$in1,8
-	vspltw		$tmp,$in0,3
-	vxor		$tmp,$tmp,$in1
-	vsldoi		$in1,$zero,$in1,12	# >>32
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in1,$in1,$tmp
-	vxor		$in0,$in0,$key
-	vxor		$in1,$in1,$key
-	 vsldoi		$stage,$stage,$in0,8
-
-	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$stage,$stage,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	 vsldoi		$stage,$in0,$in1,8
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	 vperm		$outtail,$stage,$stage,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vspltw		$tmp,$in0,3
-	vxor		$tmp,$tmp,$in1
-	vsldoi		$in1,$zero,$in1,12	# >>32
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in1,$in1,$tmp
-	vxor		$in0,$in0,$key
-	vxor		$in1,$in1,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$inp,$out,15		# 15 is not typo
-	 addi		$out,$out,16
-	bdnz		Loop192
-
-	li		$rounds,12
-	addi		$out,$out,0x20
-	b		Ldone
-
-.align	4
-L256:
-	lvx		$tmp,0,$inp
-	li		$cnt,7
-	li		$rounds,14
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
-	mtctr		$cnt
-
-Loop256:
-	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in1,$in1,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$inp,$out,15		# 15 is not typo
-	 addi		$out,$out,16
-	bdz		Ldone
-
-	vspltw		$key,$in0,3		# just splat
-	vsldoi		$tmp,$zero,$in1,12	# >>32
-	vsbox		$key,$key
-
-	vxor		$in1,$in1,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in1,$in1,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in1,$in1,$tmp
-
-	vxor		$in1,$in1,$key
-	b		Loop256
-
-.align	4
-Ldone:
-	lvx		$in1,0,$inp		# redundant in aligned case
-	vsel		$in1,$outhead,$in1,$outmask
-	stvx		$in1,0,$inp
-	li		$ptr,0
-	mtspr		256,$vrsave
-	stw		$rounds,0($out)
-
-Lenc_key_abort:
-	mr		r3,$ptr
-	blr
-	.long		0
-	.byte		0,12,0x14,1,0,0,3,0
-	.long		0
-.size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
-
-.globl	.${prefix}_set_decrypt_key
-	$STU		$sp,-$FRAME($sp)
-	mflr		r10
-	$PUSH		r10,$FRAME+$LRSAVE($sp)
-	bl		Lset_encrypt_key
-	mtlr		r10
-
-	cmpwi		r3,0
-	bne-		Ldec_key_abort
-
-	slwi		$cnt,$rounds,4
-	subi		$inp,$out,240		# first round key
-	srwi		$rounds,$rounds,1
-	add		$out,$inp,$cnt		# last round key
-	mtctr		$rounds
-
-Ldeckey:
-	lwz		r0, 0($inp)
-	lwz		r6, 4($inp)
-	lwz		r7, 8($inp)
-	lwz		r8, 12($inp)
-	addi		$inp,$inp,16
-	lwz		r9, 0($out)
-	lwz		r10,4($out)
-	lwz		r11,8($out)
-	lwz		r12,12($out)
-	stw		r0, 0($out)
-	stw		r6, 4($out)
-	stw		r7, 8($out)
-	stw		r8, 12($out)
-	subi		$out,$out,16
-	stw		r9, -16($inp)
-	stw		r10,-12($inp)
-	stw		r11,-8($inp)
-	stw		r12,-4($inp)
-	bdnz		Ldeckey
-
-	xor		r3,r3,r3		# return value
-Ldec_key_abort:
-	addi		$sp,$sp,$FRAME
-	blr
-	.long		0
-	.byte		0,12,4,1,0x80,0,3,0
-	.long		0
-.size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
-___
-}}}
-#########################################################################
-{{{	# Single block en- and decrypt procedures			#
-sub gen_block () {
-my $dir = shift;
-my $n   = $dir eq "de" ? "n" : "";
-my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
-
-$code.=<<___;
-.globl	.${prefix}_${dir}crypt
-	lwz		$rounds,240($key)
-	lis		r0,0xfc00
-	mfspr		$vrsave,256
-	li		$idx,15			# 15 is not typo
-	mtspr		256,r0
-
-	lvx		v0,0,$inp
-	neg		r11,$out
-	lvx		v1,$idx,$inp
-	lvsl		v2,0,$inp		# inpperm
-	le?vspltisb	v4,0x0f
-	?lvsl		v3,0,r11		# outperm
-	le?vxor		v2,v2,v4
-	li		$idx,16
-	vperm		v0,v0,v1,v2		# align [and byte swap in LE]
-	lvx		v1,0,$key
-	?lvsl		v5,0,$key		# keyperm
-	srwi		$rounds,$rounds,1
-	lvx		v2,$idx,$key
-	addi		$idx,$idx,16
-	subi		$rounds,$rounds,1
-	?vperm		v1,v1,v2,v5		# align round key
-
-	vxor		v0,v0,v1
-	lvx		v1,$idx,$key
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Loop_${dir}c:
-	?vperm		v2,v2,v1,v5
-	v${n}cipher	v0,v0,v2
-	lvx		v2,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		v1,v1,v2,v5
-	v${n}cipher	v0,v0,v1
-	lvx		v1,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_${dir}c
-
-	?vperm		v2,v2,v1,v5
-	v${n}cipher	v0,v0,v2
-	lvx		v2,$idx,$key
-	?vperm		v1,v1,v2,v5
-	v${n}cipherlast	v0,v0,v1
-
-	vspltisb	v2,-1
-	vxor		v1,v1,v1
-	li		$idx,15			# 15 is not typo
-	?vperm		v2,v1,v2,v3		# outmask
-	le?vxor		v3,v3,v4
-	lvx		v1,0,$out		# outhead
-	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]
-	vsel		v1,v1,v0,v2
-	lvx		v4,$idx,$out
-	stvx		v1,0,$out
-	vsel		v0,v0,v4,v2
-	stvx		v0,$idx,$out
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,3,0
-	.long		0
-.size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
-___
-}
-&gen_block("en");
-&gen_block("de");
-}}}
-#########################################################################
-{{{	# CBC en- and decrypt procedures				#
-my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
-my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
-						map("v$_",(4..10));
-$code.=<<___;
-.globl	.${prefix}_cbc_encrypt
-	${UCMP}i	$len,16
-	bltlr-
-
-	cmpwi		$enc,0			# test direction
-	lis		r0,0xffe0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	li		$idx,15
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	le?vspltisb	$tmp,0x0f
-
-	lvx		$ivec,0,$ivp		# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$ivec,$ivec,$inptail,$inpperm
-
-	neg		r11,$inp
-	?lvsl		$keyperm,0,$key		# prepare for unaligned key
-	lwz		$rounds,240($key)
-
-	lvsr		$inpperm,0,r11		# prepare for unaligned load
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	?lvsr		$outperm,0,$out		# prepare for unaligned store
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-
-	srwi		$rounds,$rounds,1
-	li		$idx,16
-	subi		$rounds,$rounds,1
-	beq		Lcbc_dec
-
-Lcbc_enc:
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	mtctr		$rounds
-	subi		$len,$len,16		# len-=16
-
-	lvx		$rndkey0,0,$key
-	 vperm		$inout,$inout,$inptail,$inpperm
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	vxor		$inout,$inout,$ivec
-
-Loop_cbc_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_cbc_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$ivec,$inout,$rndkey0
-	${UCMP}i	$len,16
-
-	vperm		$tmp,$ivec,$ivec,$outperm
-	vsel		$inout,$outhead,$tmp,$outmask
-	vmr		$outhead,$tmp
-	stvx		$inout,0,$out
-	addi		$out,$out,16
-	bge		Lcbc_enc
-
-	b		Lcbc_done
-
-.align	4
-Lcbc_dec:
-	${UCMP}i	$len,128
-	bge		_aesp8_cbc_decrypt8x
-	vmr		$tmp,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	mtctr		$rounds
-	subi		$len,$len,16		# len-=16
-
-	lvx		$rndkey0,0,$key
-	 vperm		$tmp,$tmp,$inptail,$inpperm
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$tmp,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-
-Loop_cbc_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_cbc_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipherlast	$inout,$inout,$rndkey0
-	${UCMP}i	$len,16
-
-	vxor		$inout,$inout,$ivec
-	vmr		$ivec,$tmp
-	vperm		$tmp,$inout,$inout,$outperm
-	vsel		$inout,$outhead,$tmp,$outmask
-	vmr		$outhead,$tmp
-	stvx		$inout,0,$out
-	addi		$out,$out,16
-	bge		Lcbc_dec
-
-Lcbc_done:
-	addi		$out,$out,-1
-	lvx		$inout,0,$out		# redundant in aligned case
-	vsel		$inout,$outhead,$inout,$outmask
-	stvx		$inout,0,$out
-
-	neg		$enc,$ivp		# write [unaligned] iv
-	li		$idx,15			# 15 is not typo
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	vspltisb	$outmask,-1
-	le?vspltisb	$tmp,0x0f
-	?lvsl		$outperm,0,$enc
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-	lvx		$outhead,0,$ivp
-	vperm		$ivec,$ivec,$ivec,$outperm
-	vsel		$inout,$outhead,$ivec,$outmask
-	lvx		$inptail,$idx,$ivp
-	stvx		$inout,0,$ivp
-	vsel		$inout,$ivec,$inptail,$outmask
-	stvx		$inout,$idx,$ivp
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,6,0
-	.long		0
-___
-#########################################################################
-{{	# Optimized CBC decrypt procedure				#
-my $key_="r11";
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
-my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
-my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
-
-$code.=<<___;
-.align	5
-_aesp8_cbc_decrypt8x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	li		r10,`$FRAME+8*16+15`
-	li		r11,`$FRAME+8*16+31`
-	stvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	stvx		v21,r11,$sp
-	addi		r11,r11,32
-	stvx		v22,r10,$sp
-	addi		r10,r10,32
-	stvx		v23,r11,$sp
-	addi		r11,r11,32
-	stvx		v24,r10,$sp
-	addi		r10,r10,32
-	stvx		v25,r11,$sp
-	addi		r11,r11,32
-	stvx		v26,r10,$sp
-	addi		r10,r10,32
-	stvx		v27,r11,$sp
-	addi		r11,r11,32
-	stvx		v28,r10,$sp
-	addi		r10,r10,32
-	stvx		v29,r11,$sp
-	addi		r11,r11,32
-	stvx		v30,r10,$sp
-	stvx		v31,r11,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	subi		$rounds,$rounds,3	# -4 in total
-	subi		$len,$len,128		# bias
-
-	lvx		$rndkey0,$x00,$key	# load key schedule
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	lvx		v31,$x00,$key
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_cbc_dec_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_cbc_dec_key
-
-	lvx		v26,$x10,$key
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$out0,$x70,$key		# borrow $out0
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$out0,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	#lvx		$inptail,0,$inp		# "caller" already did this
-	#addi		$inp,$inp,15		# 15 is not typo
-	subi		$inp,$inp,15		# undo "caller"
-
-	 le?li		$idx,8
-	lvx_u		$in0,$x00,$inp		# load first 8 "words"
-	 le?lvsl	$inpperm,0,$idx
-	 le?vspltisb	$tmp,0x0f
-	lvx_u		$in1,$x10,$inp
-	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
-	lvx_u		$in2,$x20,$inp
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	lvx_u		$in3,$x30,$inp
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	lvx_u		$in4,$x40,$inp
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	vxor		$out0,$in0,$rndkey0
-	lvx_u		$in5,$x50,$inp
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	vxor		$out1,$in1,$rndkey0
-	lvx_u		$in6,$x60,$inp
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	vxor		$out2,$in2,$rndkey0
-	lvx_u		$in7,$x70,$inp
-	addi		$inp,$inp,0x80
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	vxor		$out3,$in3,$rndkey0
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	vxor		$out4,$in4,$rndkey0
-	 le?vperm	$in7,$in7,$in7,$inpperm
-	vxor		$out5,$in5,$rndkey0
-	vxor		$out6,$in6,$rndkey0
-	vxor		$out7,$in7,$rndkey0
-
-	mtctr		$rounds
-	b		Loop_cbc_dec8x
-.align	5
-Loop_cbc_dec8x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_cbc_dec8x
-
-	subic		$len,$len,128		# $len-=128
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-
-	and		r0,r0,$len
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-	vncipher	$out6,$out6,v26
-	vncipher	$out7,$out7,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in7 are loaded
-						# with last "words"
-	vncipher	$out0,$out0,v27
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-	vncipher	$out6,$out6,v27
-	vncipher	$out7,$out7,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	vncipher	$out6,$out6,v28
-	vncipher	$out7,$out7,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	vncipher	$out6,$out6,v29
-	vncipher	$out7,$out7,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-
-	vncipher	$out0,$out0,v30
-	 vxor		$ivec,$ivec,v31		# xor with last round key
-	vncipher	$out1,$out1,v30
-	 vxor		$in0,$in0,v31
-	vncipher	$out2,$out2,v30
-	 vxor		$in1,$in1,v31
-	vncipher	$out3,$out3,v30
-	 vxor		$in2,$in2,v31
-	vncipher	$out4,$out4,v30
-	 vxor		$in3,$in3,v31
-	vncipher	$out5,$out5,v30
-	 vxor		$in4,$in4,v31
-	vncipher	$out6,$out6,v30
-	 vxor		$in5,$in5,v31
-	vncipher	$out7,$out7,v30
-	 vxor		$in6,$in6,v31
-
-	vncipherlast	$out0,$out0,$ivec
-	vncipherlast	$out1,$out1,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	vncipherlast	$out2,$out2,$in1
-	 lvx_u		$in1,$x10,$inp
-	vncipherlast	$out3,$out3,$in2
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	 lvx_u		$in2,$x20,$inp
-	vncipherlast	$out4,$out4,$in3
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	 lvx_u		$in3,$x30,$inp
-	vncipherlast	$out5,$out5,$in4
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	 lvx_u		$in4,$x40,$inp
-	vncipherlast	$out6,$out6,$in5
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	 lvx_u		$in5,$x50,$inp
-	vncipherlast	$out7,$out7,$in6
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	 lvx_u		$in6,$x60,$inp
-	vmr		$ivec,$in7
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	 lvx_u		$in7,$x70,$inp
-	 addi		$inp,$inp,0x80
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	 vxor		$out0,$in0,$rndkey0
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	 le?vperm	$in7,$in7,$in7,$inpperm
-	 vxor		$out1,$in1,$rndkey0
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$rndkey0
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$rndkey0
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$rndkey0
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	 vxor		$out5,$in5,$rndkey0
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x60,$out
-	 vxor		$out6,$in6,$rndkey0
-	stvx_u		$out7,$x70,$out
-	addi		$out,$out,0x80
-	 vxor		$out7,$in7,$rndkey0
-
-	mtctr		$rounds
-	beq		Loop_cbc_dec8x		# did $len-=128 borrow?
-
-	addic.		$len,$len,128
-	beq		Lcbc_dec8x_done
-	nop
-	nop
-
-Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_cbc_dec8x_tail
-
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-	vncipher	$out6,$out6,v26
-	vncipher	$out7,$out7,v26
-
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-	vncipher	$out6,$out6,v27
-	vncipher	$out7,$out7,v27
-
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	vncipher	$out6,$out6,v28
-	vncipher	$out7,$out7,v28
-
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	vncipher	$out6,$out6,v29
-	vncipher	$out7,$out7,v29
-
-	vncipher	$out1,$out1,v30
-	 vxor		$ivec,$ivec,v31		# last round key
-	vncipher	$out2,$out2,v30
-	 vxor		$in1,$in1,v31
-	vncipher	$out3,$out3,v30
-	 vxor		$in2,$in2,v31
-	vncipher	$out4,$out4,v30
-	 vxor		$in3,$in3,v31
-	vncipher	$out5,$out5,v30
-	 vxor		$in4,$in4,v31
-	vncipher	$out6,$out6,v30
-	 vxor		$in5,$in5,v31
-	vncipher	$out7,$out7,v30
-	 vxor		$in6,$in6,v31
-
-	cmplwi		$len,32			# switch($len)
-	blt		Lcbc_dec8x_one
-	nop
-	beq		Lcbc_dec8x_two
-	cmplwi		$len,64
-	blt		Lcbc_dec8x_three
-	nop
-	beq		Lcbc_dec8x_four
-	cmplwi		$len,96
-	blt		Lcbc_dec8x_five
-	nop
-	beq		Lcbc_dec8x_six
-
-Lcbc_dec8x_seven:
-	vncipherlast	$out1,$out1,$ivec
-	vncipherlast	$out2,$out2,$in1
-	vncipherlast	$out3,$out3,$in2
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out1,$out1,$out1,$inpperm
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x00,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x10,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x20,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x30,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x40,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x50,$out
-	stvx_u		$out7,$x60,$out
-	addi		$out,$out,0x70
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_six:
-	vncipherlast	$out2,$out2,$ivec
-	vncipherlast	$out3,$out3,$in2
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out2,$out2,$out2,$inpperm
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x00,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x10,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x20,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x30,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x40,$out
-	stvx_u		$out7,$x50,$out
-	addi		$out,$out,0x60
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_five:
-	vncipherlast	$out3,$out3,$ivec
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out3,$out3,$out3,$inpperm
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x00,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x10,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x20,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x30,$out
-	stvx_u		$out7,$x40,$out
-	addi		$out,$out,0x50
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_four:
-	vncipherlast	$out4,$out4,$ivec
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out4,$out4,$out4,$inpperm
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x00,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x10,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x20,$out
-	stvx_u		$out7,$x30,$out
-	addi		$out,$out,0x40
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_three:
-	vncipherlast	$out5,$out5,$ivec
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out5,$out5,$out5,$inpperm
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x00,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x10,$out
-	stvx_u		$out7,$x20,$out
-	addi		$out,$out,0x30
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_two:
-	vncipherlast	$out6,$out6,$ivec
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out6,$out6,$out6,$inpperm
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x00,$out
-	stvx_u		$out7,$x10,$out
-	addi		$out,$out,0x20
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_one:
-	vncipherlast	$out7,$out7,$ivec
-	vmr		$ivec,$in7
-
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out7,0,$out
-	addi		$out,$out,0x10
-
-Lcbc_dec8x_done:
-	le?vperm	$ivec,$ivec,$ivec,$inpperm
-	stvx_u		$ivec,0,$ivp		# write [unaligned] iv
-
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$inpperm,r10,$sp	# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
-___
-}}	}}}
-
-#########################################################################
-{{{	# CTR procedure[s]						#
-
-####################### WARNING: Here be dragons! #######################
-#
-# This code is written as 'ctr32', based on a 32-bit counter used
-# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
-# a 128-bit counter.
-#
-# This leads to subtle changes from the upstream code: the counter
-# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
-# both the bulk (8 blocks at a time) path, and in the individual block
-# path. Be aware of this when doing updates.
-#
-# See:
-# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
-# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
-# https://github.com/openssl/openssl/pull/8942
-#
-#########################################################################
-my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
-my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
-						map("v$_",(4..11));
-my $dat=$tmp;
-
-$code.=<<___;
-.globl	.${prefix}_ctr32_encrypt_blocks
-	${UCMP}i	$len,1
-	bltlr-
-
-	lis		r0,0xfff0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	li		$idx,15
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	le?vspltisb	$tmp,0x0f
-
-	lvx		$ivec,0,$ivp		# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	 vspltisb	$one,1
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$ivec,$ivec,$inptail,$inpperm
-	 vsldoi		$one,$rndkey0,$one,1
-
-	neg		r11,$inp
-	?lvsl		$keyperm,0,$key		# prepare for unaligned key
-	lwz		$rounds,240($key)
-
-	lvsr		$inpperm,0,r11		# prepare for unaligned load
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	srwi		$rounds,$rounds,1
-	li		$idx,16
-	subi		$rounds,$rounds,1
-
-	${UCMP}i	$len,8
-	bge		_aesp8_ctr32_encrypt8x
-
-	?lvsr		$outperm,0,$out		# prepare for unaligned store
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-
-	lvx		$rndkey0,0,$key
-	mtctr		$rounds
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$ivec,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	b		Loop_ctr32_enc
-
-.align	5
-Loop_ctr32_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_ctr32_enc
-
-	vadduqm		$ivec,$ivec,$one	# Kernel change for 128-bit
-	 vmr		$dat,$inptail
-	 lvx		$inptail,0,$inp
-	 addi		$inp,$inp,16
-	 subic.		$len,$len,1		# blocks--
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	 vperm		$dat,$dat,$inptail,$inpperm
-	 li		$idx,16
-	?vperm		$rndkey1,$rndkey0,$rndkey1,$keyperm
-	 lvx		$rndkey0,0,$key
-	vxor		$dat,$dat,$rndkey1	# last round key
-	vcipherlast	$inout,$inout,$dat
-
-	 lvx		$rndkey1,$idx,$key
-	 addi		$idx,$idx,16
-	vperm		$inout,$inout,$inout,$outperm
-	vsel		$dat,$outhead,$inout,$outmask
-	 mtctr		$rounds
-	 ?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vmr		$outhead,$inout
-	 vxor		$inout,$ivec,$rndkey0
-	 lvx		$rndkey0,$idx,$key
-	 addi		$idx,$idx,16
-	stvx		$dat,0,$out
-	addi		$out,$out,16
-	bne		Loop_ctr32_enc
-
-	addi		$out,$out,-1
-	lvx		$inout,0,$out		# redundant in aligned case
-	vsel		$inout,$outhead,$inout,$outmask
-	stvx		$inout,0,$out
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,6,0
-	.long		0
-___
-#########################################################################
-{{	# Optimized CTR procedure					#
-my $key_="r11";
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
-my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
-my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
-my ($two,$three,$four)=($outhead,$outperm,$outmask);
-
-$code.=<<___;
-.align	5
-_aesp8_ctr32_encrypt8x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	li		r10,`$FRAME+8*16+15`
-	li		r11,`$FRAME+8*16+31`
-	stvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	stvx		v21,r11,$sp
-	addi		r11,r11,32
-	stvx		v22,r10,$sp
-	addi		r10,r10,32
-	stvx		v23,r11,$sp
-	addi		r11,r11,32
-	stvx		v24,r10,$sp
-	addi		r10,r10,32
-	stvx		v25,r11,$sp
-	addi		r11,r11,32
-	stvx		v26,r10,$sp
-	addi		r10,r10,32
-	stvx		v27,r11,$sp
-	addi		r11,r11,32
-	stvx		v28,r10,$sp
-	addi		r10,r10,32
-	stvx		v29,r11,$sp
-	addi		r11,r11,32
-	stvx		v30,r10,$sp
-	stvx		v31,r11,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key	# load key schedule
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	lvx		v31,$x00,$key
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_ctr32_enc_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_ctr32_enc_key
-
-	lvx		v26,$x10,$key
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$out0,$x70,$key		# borrow $out0
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$out0,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	vadduqm		$two,$one,$one
-	subi		$inp,$inp,15		# undo "caller"
-	$SHL		$len,$len,4
-
-	vadduqm		$out1,$ivec,$one	# counter values ...
-	vadduqm		$out2,$ivec,$two	# (do all ctr adds as 128-bit)
-	vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
-	 le?li		$idx,8
-	vadduqm		$out3,$out1,$two
-	vxor		$out1,$out1,$rndkey0
-	 le?lvsl	$inpperm,0,$idx
-	vadduqm		$out4,$out2,$two
-	vxor		$out2,$out2,$rndkey0
-	 le?vspltisb	$tmp,0x0f
-	vadduqm		$out5,$out3,$two
-	vxor		$out3,$out3,$rndkey0
-	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
-	vadduqm		$out6,$out4,$two
-	vxor		$out4,$out4,$rndkey0
-	vadduqm		$out7,$out5,$two
-	vxor		$out5,$out5,$rndkey0
-	vadduqm		$ivec,$out6,$two	# next counter value
-	vxor		$out6,$out6,$rndkey0
-	vxor		$out7,$out7,$rndkey0
-
-	mtctr		$rounds
-	b		Loop_ctr32_enc8x
-.align	5
-Loop_ctr32_enc8x:
-	vcipher 	$out0,$out0,v24
-	vcipher 	$out1,$out1,v24
-	vcipher 	$out2,$out2,v24
-	vcipher 	$out3,$out3,v24
-	vcipher 	$out4,$out4,v24
-	vcipher 	$out5,$out5,v24
-	vcipher 	$out6,$out6,v24
-	vcipher 	$out7,$out7,v24
-Loop_ctr32_enc8x_middle:
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher 	$out0,$out0,v25
-	vcipher 	$out1,$out1,v25
-	vcipher 	$out2,$out2,v25
-	vcipher 	$out3,$out3,v25
-	vcipher 	$out4,$out4,v25
-	vcipher 	$out5,$out5,v25
-	vcipher 	$out6,$out6,v25
-	vcipher 	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_ctr32_enc8x
-
-	subic		r11,$len,256		# $len-256, borrow $key_
-	vcipher 	$out0,$out0,v24
-	vcipher 	$out1,$out1,v24
-	vcipher 	$out2,$out2,v24
-	vcipher 	$out3,$out3,v24
-	vcipher 	$out4,$out4,v24
-	vcipher 	$out5,$out5,v24
-	vcipher 	$out6,$out6,v24
-	vcipher 	$out7,$out7,v24
-
-	subfe		r0,r0,r0		# borrow?-1:0
-	vcipher 	$out0,$out0,v25
-	vcipher 	$out1,$out1,v25
-	vcipher 	$out2,$out2,v25
-	vcipher 	$out3,$out3,v25
-	vcipher 	$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-	vcipher		$out6,$out6,v25
-	vcipher		$out7,$out7,v25
-
-	and		r0,r0,r11
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v26
-	vcipher		$out1,$out1,v26
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	vcipher		$out4,$out4,v26
-	vcipher		$out5,$out5,v26
-	vcipher		$out6,$out6,v26
-	vcipher		$out7,$out7,v26
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	subic		$len,$len,129		# $len-=129
-	vcipher		$out0,$out0,v27
-	addi		$len,$len,1		# $len-=128 really
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	vcipher		$out4,$out4,v27
-	vcipher		$out5,$out5,v27
-	vcipher		$out6,$out6,v27
-	vcipher		$out7,$out7,v27
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-
-	vcipher		$out0,$out0,v28
-	 lvx_u		$in0,$x00,$inp		# load input
-	vcipher		$out1,$out1,v28
-	 lvx_u		$in1,$x10,$inp
-	vcipher		$out2,$out2,v28
-	 lvx_u		$in2,$x20,$inp
-	vcipher		$out3,$out3,v28
-	 lvx_u		$in3,$x30,$inp
-	vcipher		$out4,$out4,v28
-	 lvx_u		$in4,$x40,$inp
-	vcipher		$out5,$out5,v28
-	 lvx_u		$in5,$x50,$inp
-	vcipher		$out6,$out6,v28
-	 lvx_u		$in6,$x60,$inp
-	vcipher		$out7,$out7,v28
-	 lvx_u		$in7,$x70,$inp
-	 addi		$inp,$inp,0x80
-
-	vcipher		$out0,$out0,v29
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	vcipher		$out1,$out1,v29
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	vcipher		$out2,$out2,v29
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	vcipher		$out3,$out3,v29
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	vcipher		$out4,$out4,v29
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	vcipher		$out5,$out5,v29
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	vcipher		$out6,$out6,v29
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	vcipher		$out7,$out7,v29
-	 le?vperm	$in7,$in7,$in7,$inpperm
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in7 are loaded
-						# with last "words"
-	subfe.		r0,r0,r0		# borrow?-1:0
-	vcipher		$out0,$out0,v30
-	 vxor		$in0,$in0,v31		# xor with last round key
-	vcipher		$out1,$out1,v30
-	 vxor		$in1,$in1,v31
-	vcipher		$out2,$out2,v30
-	 vxor		$in2,$in2,v31
-	vcipher		$out3,$out3,v30
-	 vxor		$in3,$in3,v31
-	vcipher		$out4,$out4,v30
-	 vxor		$in4,$in4,v31
-	vcipher		$out5,$out5,v30
-	 vxor		$in5,$in5,v31
-	vcipher		$out6,$out6,v30
-	 vxor		$in6,$in6,v31
-	vcipher		$out7,$out7,v30
-	 vxor		$in7,$in7,v31
-
-	bne		Lctr32_enc8x_break	# did $len-129 borrow?
-
-	vcipherlast	$in0,$out0,$in0
-	vcipherlast	$in1,$out1,$in1
-	 vadduqm	$out1,$ivec,$one	# counter values ...
-	vcipherlast	$in2,$out2,$in2
-	 vadduqm	$out2,$ivec,$two
-	 vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
-	vcipherlast	$in3,$out3,$in3
-	 vadduqm	$out3,$out1,$two
-	 vxor		$out1,$out1,$rndkey0
-	vcipherlast	$in4,$out4,$in4
-	 vadduqm	$out4,$out2,$two
-	 vxor		$out2,$out2,$rndkey0
-	vcipherlast	$in5,$out5,$in5
-	 vadduqm	$out5,$out3,$two
-	 vxor		$out3,$out3,$rndkey0
-	vcipherlast	$in6,$out6,$in6
-	 vadduqm	$out6,$out4,$two
-	 vxor		$out4,$out4,$rndkey0
-	vcipherlast	$in7,$out7,$in7
-	 vadduqm	$out7,$out5,$two
-	 vxor		$out5,$out5,$rndkey0
-	le?vperm	$in0,$in0,$in0,$inpperm
-	 vadduqm	$ivec,$out6,$two	# next counter value
-	 vxor		$out6,$out6,$rndkey0
-	le?vperm	$in1,$in1,$in1,$inpperm
-	 vxor		$out7,$out7,$rndkey0
-	mtctr		$rounds
-
-	 vcipher	$out0,$out0,v24
-	stvx_u		$in0,$x00,$out
-	le?vperm	$in2,$in2,$in2,$inpperm
-	 vcipher	$out1,$out1,v24
-	stvx_u		$in1,$x10,$out
-	le?vperm	$in3,$in3,$in3,$inpperm
-	 vcipher	$out2,$out2,v24
-	stvx_u		$in2,$x20,$out
-	le?vperm	$in4,$in4,$in4,$inpperm
-	 vcipher	$out3,$out3,v24
-	stvx_u		$in3,$x30,$out
-	le?vperm	$in5,$in5,$in5,$inpperm
-	 vcipher	$out4,$out4,v24
-	stvx_u		$in4,$x40,$out
-	le?vperm	$in6,$in6,$in6,$inpperm
-	 vcipher	$out5,$out5,v24
-	stvx_u		$in5,$x50,$out
-	le?vperm	$in7,$in7,$in7,$inpperm
-	 vcipher	$out6,$out6,v24
-	stvx_u		$in6,$x60,$out
-	 vcipher	$out7,$out7,v24
-	stvx_u		$in7,$x70,$out
-	addi		$out,$out,0x80
-
-	b		Loop_ctr32_enc8x_middle
-
-.align	5
-Lctr32_enc8x_break:
-	cmpwi		$len,-0x60
-	blt		Lctr32_enc8x_one
-	nop
-	beq		Lctr32_enc8x_two
-	cmpwi		$len,-0x40
-	blt		Lctr32_enc8x_three
-	nop
-	beq		Lctr32_enc8x_four
-	cmpwi		$len,-0x20
-	blt		Lctr32_enc8x_five
-	nop
-	beq		Lctr32_enc8x_six
-	cmpwi		$len,0x00
-	blt		Lctr32_enc8x_seven
-
-Lctr32_enc8x_eight:
-	vcipherlast	$out0,$out0,$in0
-	vcipherlast	$out1,$out1,$in1
-	vcipherlast	$out2,$out2,$in2
-	vcipherlast	$out3,$out3,$in3
-	vcipherlast	$out4,$out4,$in4
-	vcipherlast	$out5,$out5,$in5
-	vcipherlast	$out6,$out6,$in6
-	vcipherlast	$out7,$out7,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x60,$out
-	stvx_u		$out7,$x70,$out
-	addi		$out,$out,0x80
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_seven:
-	vcipherlast	$out0,$out0,$in1
-	vcipherlast	$out1,$out1,$in2
-	vcipherlast	$out2,$out2,$in3
-	vcipherlast	$out3,$out3,$in4
-	vcipherlast	$out4,$out4,$in5
-	vcipherlast	$out5,$out5,$in6
-	vcipherlast	$out6,$out6,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	stvx_u		$out6,$x60,$out
-	addi		$out,$out,0x70
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_six:
-	vcipherlast	$out0,$out0,$in2
-	vcipherlast	$out1,$out1,$in3
-	vcipherlast	$out2,$out2,$in4
-	vcipherlast	$out3,$out3,$in5
-	vcipherlast	$out4,$out4,$in6
-	vcipherlast	$out5,$out5,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	stvx_u		$out5,$x50,$out
-	addi		$out,$out,0x60
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_five:
-	vcipherlast	$out0,$out0,$in3
-	vcipherlast	$out1,$out1,$in4
-	vcipherlast	$out2,$out2,$in5
-	vcipherlast	$out3,$out3,$in6
-	vcipherlast	$out4,$out4,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_four:
-	vcipherlast	$out0,$out0,$in4
-	vcipherlast	$out1,$out1,$in5
-	vcipherlast	$out2,$out2,$in6
-	vcipherlast	$out3,$out3,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_three:
-	vcipherlast	$out0,$out0,$in5
-	vcipherlast	$out1,$out1,$in6
-	vcipherlast	$out2,$out2,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_two:
-	vcipherlast	$out0,$out0,$in6
-	vcipherlast	$out1,$out1,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_one:
-	vcipherlast	$out0,$out0,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	stvx_u		$out0,0,$out
-	addi		$out,$out,0x10
-
-Lctr32_enc8x_done:
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$inpperm,r10,$sp	# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
-___
-}}	}}}
-
-#########################################################################
-{{{	# XTS procedures						#
-# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#
-#                             const AES_KEY *key1, const AES_KEY *key2,	#
-#                             [const] unsigned char iv[16]);		#
-# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#
-# input tweak value is assumed to be encrypted already, and last tweak	#
-# value, one suitable for consecutive call on same chunk of data, is	#
-# written back to original buffer. In addition, in "tweak chaining"	#
-# mode only complete input blocks are processed.			#
-
-my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));
-my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));
-my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));
-my $taillen = $key2;
-
-   ($inp,$idx) = ($idx,$inp);				# reassign
-
-$code.=<<___;
-.globl	.${prefix}_xts_encrypt
-	mr		$inp,r3				# reassign
-	li		r3,-1
-	${UCMP}i	$len,16
-	bltlr-
-
-	lis		r0,0xfff0
-	mfspr		r12,256				# save vrsave
-	li		r11,0
-	mtspr		256,r0
-
-	vspltisb	$seven,0x07			# 0x070707..07
-	le?lvsl		$leperm,r11,r11
-	le?vspltisb	$tmp,0x0f
-	le?vxor		$leperm,$leperm,$seven
-
-	li		$idx,15
-	lvx		$tweak,0,$ivp			# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$tweak,$tweak,$inptail,$inpperm
-
-	neg		r11,$inp
-	lvsr		$inpperm,0,r11			# prepare for unaligned load
-	lvx		$inout,0,$inp
-	addi		$inp,$inp,15			# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	${UCMP}i	$key2,0				# key2==NULL?
-	beq		Lxts_enc_no_key2
-
-	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
-	lwz		$rounds,240($key2)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	lvx		$rndkey0,0,$key2
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Ltweak_xts_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	bdnz		Ltweak_xts_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$tweak,$tweak,$rndkey0
-
-	li		$ivp,0				# don't chain the tweak
-	b		Lxts_enc
-
-Lxts_enc_no_key2:
-	li		$idx,-16
-	and		$len,$len,$idx			# in "tweak chaining"
-							# mode only complete
-							# blocks are processed
-Lxts_enc:
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-
-	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
-	lwz		$rounds,240($key1)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	vslb		$eighty7,$seven,$seven		# 0x808080..80
-	vor		$eighty7,$eighty7,$seven	# 0x878787..87
-	vspltisb	$tmp,1				# 0x010101..01
-	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
-
-	${UCMP}i	$len,96
-	bge		_aesp8_xts_encrypt6x
-
-	andi.		$taillen,$len,15
-	subic		r0,$len,32
-	subi		$taillen,$taillen,16
-	subfe		r0,r0,r0
-	and		r0,r0,$taillen
-	add		$inp,$inp,r0
-
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	mtctr		$rounds
-	b		Loop_xts_enc
-
-.align	5
-Loop_xts_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak
-	vcipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-	addi		$out,$out,16
-
-	subic.		$len,$len,16
-	beq		Lxts_enc_done
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-
-	subic		r0,$len,32
-	subfe		r0,r0,r0
-	and		r0,r0,$taillen
-	add		$inp,$inp,r0
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$output,$output,$rndkey0	# just in case $len<16
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	mtctr		$rounds
-	${UCMP}i	$len,16
-	bge		Loop_xts_enc
-
-	vxor		$output,$output,$tweak
-	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
-	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
-	vspltisb	$tmp,-1
-	vperm		$inptail,$inptail,$tmp,$inpperm
-	vsel		$inout,$inout,$output,$inptail
-
-	subi		r11,$out,17
-	subi		$out,$out,16
-	mtctr		$len
-	li		$len,16
-Loop_xts_enc_steal:
-	lbzu		r0,1(r11)
-	stb		r0,16(r11)
-	bdnz		Loop_xts_enc_steal
-
-	mtctr		$rounds
-	b		Loop_xts_enc			# one more time...
-
-Lxts_enc_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_enc_ret
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_enc_ret:
-	mtspr		256,r12				# restore vrsave
-	li		r3,0
-	blr
-	.long		0
-	.byte		0,12,0x04,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
-
-.globl	.${prefix}_xts_decrypt
-	mr		$inp,r3				# reassign
-	li		r3,-1
-	${UCMP}i	$len,16
-	bltlr-
-
-	lis		r0,0xfff8
-	mfspr		r12,256				# save vrsave
-	li		r11,0
-	mtspr		256,r0
-
-	andi.		r0,$len,15
-	neg		r0,r0
-	andi.		r0,r0,16
-	sub		$len,$len,r0
-
-	vspltisb	$seven,0x07			# 0x070707..07
-	le?lvsl		$leperm,r11,r11
-	le?vspltisb	$tmp,0x0f
-	le?vxor		$leperm,$leperm,$seven
-
-	li		$idx,15
-	lvx		$tweak,0,$ivp			# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$tweak,$tweak,$inptail,$inpperm
-
-	neg		r11,$inp
-	lvsr		$inpperm,0,r11			# prepare for unaligned load
-	lvx		$inout,0,$inp
-	addi		$inp,$inp,15			# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	${UCMP}i	$key2,0				# key2==NULL?
-	beq		Lxts_dec_no_key2
-
-	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
-	lwz		$rounds,240($key2)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	lvx		$rndkey0,0,$key2
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Ltweak_xts_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	bdnz		Ltweak_xts_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$tweak,$tweak,$rndkey0
-
-	li		$ivp,0				# don't chain the tweak
-	b		Lxts_dec
-
-Lxts_dec_no_key2:
-	neg		$idx,$len
-	andi.		$idx,$idx,15
-	add		$len,$len,$idx			# in "tweak chaining"
-							# mode only complete
-							# blocks are processed
-Lxts_dec:
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-
-	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
-	lwz		$rounds,240($key1)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	vslb		$eighty7,$seven,$seven		# 0x808080..80
-	vor		$eighty7,$eighty7,$seven	# 0x878787..87
-	vspltisb	$tmp,1				# 0x010101..01
-	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
-
-	${UCMP}i	$len,96
-	bge		_aesp8_xts_decrypt6x
-
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-	${UCMP}i	$len,16
-	blt		Ltail_xts_dec
-	be?b		Loop_xts_dec
-
-.align	5
-Loop_xts_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak
-	vncipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-	addi		$out,$out,16
-
-	subic.		$len,$len,16
-	beq		Lxts_dec_done
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	mtctr		$rounds
-	${UCMP}i	$len,16
-	bge		Loop_xts_dec
-
-Ltail_xts_dec:
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak1,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak1,$tweak1,$tmp
-
-	subi		$inp,$inp,16
-	add		$inp,$inp,$len
-
-	vxor		$inout,$inout,$tweak		# :-(
-	vxor		$inout,$inout,$tweak1		# :-)
-
-Loop_xts_dec_short:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_dec_short
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak1
-	vncipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	#addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-
-	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
-	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
-	vspltisb	$tmp,-1
-	vperm		$inptail,$inptail,$tmp,$inpperm
-	vsel		$inout,$inout,$output,$inptail
-
-	vxor		$rndkey0,$rndkey0,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	subi		r11,$out,1
-	mtctr		$len
-	li		$len,16
-Loop_xts_dec_steal:
-	lbzu		r0,1(r11)
-	stb		r0,16(r11)
-	bdnz		Loop_xts_dec_steal
-
-	mtctr		$rounds
-	b		Loop_xts_dec			# one more time...
-
-Lxts_dec_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_dec_ret
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_dec_ret:
-	mtspr		256,r12				# restore vrsave
-	li		r3,0
-	blr
-	.long		0
-	.byte		0,12,0x04,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
-___
-#########################################################################
-{{	# Optimized XTS procedures					#
-my $key_=$key2;
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
-    $x00=0 if ($flavour =~ /osx/);
-my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
-my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
-my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($keyperm)=($out0);	# aliases with "caller", redundant assignment
-my $taillen=$x70;
-
-$code.=<<___;
-.align	5
-_aesp8_xts_encrypt6x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	mflr		r11
-	li		r7,`$FRAME+8*16+15`
-	li		r3,`$FRAME+8*16+31`
-	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
-	stvx		v20,r7,$sp		# ABI says so
-	addi		r7,r7,32
-	stvx		v21,r3,$sp
-	addi		r3,r3,32
-	stvx		v22,r7,$sp
-	addi		r7,r7,32
-	stvx		v23,r3,$sp
-	addi		r3,r3,32
-	stvx		v24,r7,$sp
-	addi		r7,r7,32
-	stvx		v25,r3,$sp
-	addi		r3,r3,32
-	stvx		v26,r7,$sp
-	addi		r7,r7,32
-	stvx		v27,r3,$sp
-	addi		r3,r3,32
-	stvx		v28,r7,$sp
-	addi		r7,r7,32
-	stvx		v29,r3,$sp
-	addi		r3,r3,32
-	stvx		v30,r7,$sp
-	stvx		v31,r3,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	xxlor		2, 32+$eighty7, 32+$eighty7
-	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
-	xxlor		1, 32+$eighty7, 32+$eighty7
-
-	# Load XOR Lconsts.
-	mr		$x70, r6
-	bl		Lconsts
-	lxvw4x		0, $x40, r6		# load XOR contents
-	mr		r6, $x70
-	li		$x70,0x70
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key1	# load key schedule
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	lvx		v31,$x00,$key1
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_xts_enc_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key1
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_xts_enc_key
-
-	lvx		v26,$x10,$key1
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key1
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key1
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key1
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key1
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key1
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$twk5,$x70,$key1	# borrow $twk5
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$twk5,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	# Switch to use the following codes with 0x010101..87 to generate tweak.
-	#     eighty7 = 0x010101..87
-	# vsrab         tmp, tweak, seven       # next tweak value, right shift 7 bits
-	# vand          tmp, tmp, eighty7       # last byte with carry
-	# vaddubm       tweak, tweak, tweak     # left shift 1 bit (x2)
-	# xxlor         vsx, 0, 0
-	# vpermxor      tweak, tweak, tmp, vsx
-
-	 vperm		$in0,$inout,$inptail,$inpperm
-	 subi		$inp,$inp,31		# undo "caller"
-	vxor		$twk0,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out0,$in0,$twk0
-	xxlor		32+$in1, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in1
-
-	 lvx_u		$in1,$x10,$inp
-	vxor		$twk1,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in1,$in1,$in1,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out1,$in1,$twk1
-	xxlor		32+$in2, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in2
-
-	 lvx_u		$in2,$x20,$inp
-	 andi.		$taillen,$len,15
-	vxor		$twk2,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in2,$in2,$in2,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out2,$in2,$twk2
-	xxlor		32+$in3, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in3
-
-	 lvx_u		$in3,$x30,$inp
-	 sub		$len,$len,$taillen
-	vxor		$twk3,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in3,$in3,$in3,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out3,$in3,$twk3
-	xxlor		32+$in4, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in4
-
-	 lvx_u		$in4,$x40,$inp
-	 subi		$len,$len,0x60
-	vxor		$twk4,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in4,$in4,$in4,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out4,$in4,$twk4
-	xxlor		32+$in5, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in5
-
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	vxor		$twk5,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in5,$in5,$in5,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out5,$in5,$twk5
-	xxlor		32+$in0, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in0
-
-	vxor		v31,v31,$rndkey0
-	mtctr		$rounds
-	b		Loop_xts_enc6x
-
-.align	5
-Loop_xts_enc6x:
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	vcipher		$out5,$out5,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_enc6x
-
-	xxlor		32+$eighty7, 1, 1	# 0x010101..87
-
-	subic		$len,$len,96		# $len-=96
-	 vxor		$in0,$twk0,v31		# xor with last round key
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk0,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	vcipher		$out5,$out5,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	 xxlor		32+$in1, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in1
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	 vxor		$in1,$twk1,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk1,$tweak,$rndkey0
-	vcipher		$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-
-	and		r0,r0,$len
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out0,$out0,v26
-	vcipher		$out1,$out1,v26
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	 xxlor		32+$in2, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in2
-	vcipher		$out4,$out4,v26
-	vcipher		$out5,$out5,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in5 are loaded
-						# with last "words"
-	 vxor		$in2,$twk2,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk2,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out0,$out0,v27
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out4,$out4,v27
-	vcipher		$out5,$out5,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	 xxlor		32+$in3, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in3
-	vcipher		$out0,$out0,v28
-	vcipher		$out1,$out1,v28
-	 vxor		$in3,$twk3,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk3,$tweak,$rndkey0
-	vcipher		$out2,$out2,v28
-	vcipher		$out3,$out3,v28
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out4,$out4,v28
-	vcipher		$out5,$out5,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vand		$tmp,$tmp,$eighty7
-
-	vcipher		$out0,$out0,v29
-	vcipher		$out1,$out1,v29
-	 xxlor		32+$in4, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in4
-	vcipher		$out2,$out2,v29
-	vcipher		$out3,$out3,v29
-	 vxor		$in4,$twk4,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk4,$tweak,$rndkey0
-	vcipher		$out4,$out4,v29
-	vcipher		$out5,$out5,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vaddubm	$tweak,$tweak,$tweak
-
-	vcipher		$out0,$out0,v30
-	vcipher		$out1,$out1,v30
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out2,$out2,v30
-	vcipher		$out3,$out3,v30
-	 xxlor		32+$in5, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in5
-	vcipher		$out4,$out4,v30
-	vcipher		$out5,$out5,v30
-	 vxor		$in5,$twk5,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk5,$tweak,$rndkey0
-
-	vcipherlast	$out0,$out0,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipherlast	$out1,$out1,$in1
-	 lvx_u		$in1,$x10,$inp
-	vcipherlast	$out2,$out2,$in2
-	 le?vperm	$in0,$in0,$in0,$leperm
-	 lvx_u		$in2,$x20,$inp
-	 vand		$tmp,$tmp,$eighty7
-	vcipherlast	$out3,$out3,$in3
-	 le?vperm	$in1,$in1,$in1,$leperm
-	 lvx_u		$in3,$x30,$inp
-	vcipherlast	$out4,$out4,$in4
-	 le?vperm	$in2,$in2,$in2,$leperm
-	 lvx_u		$in4,$x40,$inp
-	 xxlor		10, 32+$in0, 32+$in0
-	 xxlor		32+$in0, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in0
-	 xxlor		32+$in0, 10, 10
-	vcipherlast	$tmp,$out5,$in5		# last block might be needed
-						# in stealing mode
-	 le?vperm	$in3,$in3,$in3,$leperm
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	 le?vperm	$in4,$in4,$in4,$leperm
-	 le?vperm	$in5,$in5,$in5,$leperm
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	 vxor		$out0,$in0,$twk0
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	 vxor		$out1,$in1,$twk1
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$twk2
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$twk3
-	le?vperm	$out5,$tmp,$tmp,$leperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$twk4
-	le?stvx_u	$out5,$x50,$out
-	be?stvx_u	$tmp, $x50,$out
-	 vxor		$out5,$in5,$twk5
-	addi		$out,$out,0x60
-
-	mtctr		$rounds
-	beq		Loop_xts_enc6x		# did $len-=96 borrow?
-
-	xxlor		32+$eighty7, 2, 2	# 0x010101..87
-
-	addic.		$len,$len,0x60
-	beq		Lxts_enc6x_zero
-	cmpwi		$len,0x20
-	blt		Lxts_enc6x_one
-	nop
-	beq		Lxts_enc6x_two
-	cmpwi		$len,0x40
-	blt		Lxts_enc6x_three
-	nop
-	beq		Lxts_enc6x_four
-
-Lxts_enc6x_five:
-	vxor		$out0,$in1,$twk0
-	vxor		$out1,$in2,$twk1
-	vxor		$out2,$in3,$twk2
-	vxor		$out3,$in4,$twk3
-	vxor		$out4,$in5,$twk4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk5		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	vxor		$tmp,$out4,$twk5	# last block prep for stealing
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_four:
-	vxor		$out0,$in2,$twk0
-	vxor		$out1,$in3,$twk1
-	vxor		$out2,$in4,$twk2
-	vxor		$out3,$in5,$twk3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk4		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	vxor		$tmp,$out3,$twk4	# last block prep for stealing
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_three:
-	vxor		$out0,$in3,$twk0
-	vxor		$out1,$in4,$twk1
-	vxor		$out2,$in5,$twk2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk3		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$tmp,$out2,$twk3	# last block prep for stealing
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_two:
-	vxor		$out0,$in4,$twk0
-	vxor		$out1,$in5,$twk1
-	vxor		$out2,$out2,$out2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk2		# unused tweak
-	vxor		$tmp,$out1,$twk2	# last block prep for stealing
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_one:
-	vxor		$out0,$in5,$twk0
-	nop
-Loop_xts_enc1x:
-	vcipher		$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_enc1x
-
-	add		$inp,$inp,$taillen
-	cmpwi		$taillen,0
-	vcipher		$out0,$out0,v24
-
-	subi		$inp,$inp,16
-	vcipher		$out0,$out0,v25
-
-	lvsr		$inpperm,0,$taillen
-	vcipher		$out0,$out0,v26
-
-	lvx_u		$in0,0,$inp
-	vcipher		$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vcipher		$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk0,$twk0,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vcipher		$out0,$out0,v30
-
-	vperm		$in0,$in0,$in0,$inpperm
-	vcipherlast	$out0,$out0,$twk0
-
-	vmr		$twk0,$twk1		# unused tweak
-	vxor		$tmp,$out0,$twk1	# last block prep for stealing
-	le?vperm	$out0,$out0,$out0,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	addi		$out,$out,0x10
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_zero:
-	cmpwi		$taillen,0
-	beq		Lxts_enc6x_done
-
-	add		$inp,$inp,$taillen
-	subi		$inp,$inp,16
-	lvx_u		$in0,0,$inp
-	lvsr		$inpperm,0,$taillen	# $in5 is no more
-	le?vperm	$in0,$in0,$in0,$leperm
-	vperm		$in0,$in0,$in0,$inpperm
-	vxor		$tmp,$tmp,$twk0
-Lxts_enc6x_steal:
-	vxor		$in0,$in0,$twk0
-	vxor		$out0,$out0,$out0
-	vspltisb	$out1,-1
-	vperm		$out0,$out0,$out1,$inpperm
-	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?
-
-	subi		r30,$out,17
-	subi		$out,$out,16
-	mtctr		$taillen
-Loop_xts_enc6x_steal:
-	lbzu		r0,1(r30)
-	stb		r0,16(r30)
-	bdnz		Loop_xts_enc6x_steal
-
-	li		$taillen,0
-	mtctr		$rounds
-	b		Loop_xts_enc1x		# one more time...
-
-.align	4
-Lxts_enc6x_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_enc6x_ret
-
-	vxor		$tweak,$twk0,$rndkey0
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_enc6x_ret:
-	mtlr		r11
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$seven,r10,$sp		# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x04,1,0x80,6,6,0
-	.long		0
-
-.align	5
-_aesp8_xts_enc5x:
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		_aesp8_xts_enc5x
-
-	add		$inp,$inp,$taillen
-	cmpwi		$taillen,0
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-
-	subi		$inp,$inp,16
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	 vxor		$twk0,$twk0,v31
-
-	vcipher		$out0,$out0,v26
-	lvsr		$inpperm,r0,$taillen	# $in5 is no more
-	vcipher		$out1,$out1,v26
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	vcipher		$out4,$out4,v26
-	 vxor		$in1,$twk1,v31
-
-	vcipher		$out0,$out0,v27
-	lvx_u		$in0,0,$inp
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	vcipher		$out4,$out4,v27
-	 vxor		$in2,$twk2,v31
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v28
-	vcipher		$out1,$out1,v28
-	vcipher		$out2,$out2,v28
-	vcipher		$out3,$out3,v28
-	vcipher		$out4,$out4,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vxor		$in3,$twk3,v31
-
-	vcipher		$out0,$out0,v29
-	le?vperm	$in0,$in0,$in0,$leperm
-	vcipher		$out1,$out1,v29
-	vcipher		$out2,$out2,v29
-	vcipher		$out3,$out3,v29
-	vcipher		$out4,$out4,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$in4,$twk4,v31
-
-	vcipher		$out0,$out0,v30
-	vperm		$in0,$in0,$in0,$inpperm
-	vcipher		$out1,$out1,v30
-	vcipher		$out2,$out2,v30
-	vcipher		$out3,$out3,v30
-	vcipher		$out4,$out4,v30
-
-	vcipherlast	$out0,$out0,$twk0
-	vcipherlast	$out1,$out1,$in1
-	vcipherlast	$out2,$out2,$in2
-	vcipherlast	$out3,$out3,$in3
-	vcipherlast	$out4,$out4,$in4
-	blr
-        .long   	0
-        .byte   	0,12,0x14,0,0,0,0,0
-
-.align	5
-_aesp8_xts_decrypt6x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	mflr		r11
-	li		r7,`$FRAME+8*16+15`
-	li		r3,`$FRAME+8*16+31`
-	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
-	stvx		v20,r7,$sp		# ABI says so
-	addi		r7,r7,32
-	stvx		v21,r3,$sp
-	addi		r3,r3,32
-	stvx		v22,r7,$sp
-	addi		r7,r7,32
-	stvx		v23,r3,$sp
-	addi		r3,r3,32
-	stvx		v24,r7,$sp
-	addi		r7,r7,32
-	stvx		v25,r3,$sp
-	addi		r3,r3,32
-	stvx		v26,r7,$sp
-	addi		r7,r7,32
-	stvx		v27,r3,$sp
-	addi		r3,r3,32
-	stvx		v28,r7,$sp
-	addi		r7,r7,32
-	stvx		v29,r3,$sp
-	addi		r3,r3,32
-	stvx		v30,r7,$sp
-	stvx		v31,r3,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	xxlor		2, 32+$eighty7, 32+$eighty7
-	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
-	xxlor		1, 32+$eighty7, 32+$eighty7
-
-	# Load XOR Lconsts.
-	mr		$x70, r6
-	bl		Lconsts
-	lxvw4x		0, $x40, r6		# load XOR contents
-	mr		r6, $x70
-	li		$x70,0x70
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key1	# load key schedule
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	lvx		v31,$x00,$key1
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_xts_dec_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key1
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_xts_dec_key
-
-	lvx		v26,$x10,$key1
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key1
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key1
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key1
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key1
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key1
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$twk5,$x70,$key1	# borrow $twk5
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$twk5,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	 vperm		$in0,$inout,$inptail,$inpperm
-	 subi		$inp,$inp,31		# undo "caller"
-	vxor		$twk0,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out0,$in0,$twk0
-	xxlor		32+$in1, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in1
-
-	 lvx_u		$in1,$x10,$inp
-	vxor		$twk1,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in1,$in1,$in1,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out1,$in1,$twk1
-	xxlor		32+$in2, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in2
-
-	 lvx_u		$in2,$x20,$inp
-	 andi.		$taillen,$len,15
-	vxor		$twk2,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in2,$in2,$in2,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out2,$in2,$twk2
-	xxlor		32+$in3, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in3
-
-	 lvx_u		$in3,$x30,$inp
-	 sub		$len,$len,$taillen
-	vxor		$twk3,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in3,$in3,$in3,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out3,$in3,$twk3
-	xxlor		32+$in4, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in4
-
-	 lvx_u		$in4,$x40,$inp
-	 subi		$len,$len,0x60
-	vxor		$twk4,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in4,$in4,$in4,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out4,$in4,$twk4
-	xxlor		32+$in5, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in5
-
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	vxor		$twk5,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in5,$in5,$in5,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out5,$in5,$twk5
-	xxlor		32+$in0, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in0
-
-	vxor		v31,v31,$rndkey0
-	mtctr		$rounds
-	b		Loop_xts_dec6x
-
-.align	5
-Loop_xts_dec6x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_dec6x
-
-	xxlor		32+$eighty7, 1, 1	# 0x010101..87
-
-	subic		$len,$len,96		# $len-=96
-	 vxor		$in0,$twk0,v31		# xor with last round key
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk0,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	 xxlor		32+$in1, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in1
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	 vxor		$in1,$twk1,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk1,$tweak,$rndkey0
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-
-	and		r0,r0,$len
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	 xxlor		32+$in2, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in2
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in5 are loaded
-						# with last "words"
-	 vxor		$in2,$twk2,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk2,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out0,$out0,v27
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	 xxlor		32+$in3, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in3
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	 vxor		$in3,$twk3,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk3,$tweak,$rndkey0
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vand		$tmp,$tmp,$eighty7
-
-	vncipher	$out0,$out0,v29
-	vncipher	$out1,$out1,v29
-	 xxlor		32+$in4, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in4
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	 vxor		$in4,$twk4,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk4,$tweak,$rndkey0
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vaddubm	$tweak,$tweak,$tweak
-
-	vncipher	$out0,$out0,v30
-	vncipher	$out1,$out1,v30
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out2,$out2,v30
-	vncipher	$out3,$out3,v30
-	 xxlor		32+$in5, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in5
-	vncipher	$out4,$out4,v30
-	vncipher	$out5,$out5,v30
-	 vxor		$in5,$twk5,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk5,$tweak,$rndkey0
-
-	vncipherlast	$out0,$out0,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipherlast	$out1,$out1,$in1
-	 lvx_u		$in1,$x10,$inp
-	vncipherlast	$out2,$out2,$in2
-	 le?vperm	$in0,$in0,$in0,$leperm
-	 lvx_u		$in2,$x20,$inp
-	 vand		$tmp,$tmp,$eighty7
-	vncipherlast	$out3,$out3,$in3
-	 le?vperm	$in1,$in1,$in1,$leperm
-	 lvx_u		$in3,$x30,$inp
-	vncipherlast	$out4,$out4,$in4
-	 le?vperm	$in2,$in2,$in2,$leperm
-	 lvx_u		$in4,$x40,$inp
-	 xxlor		10, 32+$in0, 32+$in0
-	 xxlor		32+$in0, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in0
-	 xxlor		32+$in0, 10, 10
-	vncipherlast	$out5,$out5,$in5
-	 le?vperm	$in3,$in3,$in3,$leperm
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	 le?vperm	$in4,$in4,$in4,$leperm
-	 le?vperm	$in5,$in5,$in5,$leperm
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	 vxor		$out0,$in0,$twk0
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	 vxor		$out1,$in1,$twk1
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$twk2
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$twk3
-	le?vperm	$out5,$out5,$out5,$leperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$twk4
-	stvx_u		$out5,$x50,$out
-	 vxor		$out5,$in5,$twk5
-	addi		$out,$out,0x60
-
-	mtctr		$rounds
-	beq		Loop_xts_dec6x		# did $len-=96 borrow?
-
-	xxlor		32+$eighty7, 2, 2	# 0x010101..87
-
-	addic.		$len,$len,0x60
-	beq		Lxts_dec6x_zero
-	cmpwi		$len,0x20
-	blt		Lxts_dec6x_one
-	nop
-	beq		Lxts_dec6x_two
-	cmpwi		$len,0x40
-	blt		Lxts_dec6x_three
-	nop
-	beq		Lxts_dec6x_four
-
-Lxts_dec6x_five:
-	vxor		$out0,$in1,$twk0
-	vxor		$out1,$in2,$twk1
-	vxor		$out2,$in3,$twk2
-	vxor		$out3,$in4,$twk3
-	vxor		$out4,$in5,$twk4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk5		# unused tweak
-	vxor		$twk1,$tweak,$rndkey0
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk1
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_four:
-	vxor		$out0,$in2,$twk0
-	vxor		$out1,$in3,$twk1
-	vxor		$out2,$in4,$twk2
-	vxor		$out3,$in5,$twk3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk4		# unused tweak
-	vmr		$twk1,$twk5
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk5
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_three:
-	vxor		$out0,$in3,$twk0
-	vxor		$out1,$in4,$twk1
-	vxor		$out2,$in5,$twk2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk3		# unused tweak
-	vmr		$twk1,$twk4
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk4
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_two:
-	vxor		$out0,$in4,$twk0
-	vxor		$out1,$in5,$twk1
-	vxor		$out2,$out2,$out2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk2		# unused tweak
-	vmr		$twk1,$twk3
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk3
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_one:
-	vxor		$out0,$in5,$twk0
-	nop
-Loop_xts_dec1x:
-	vncipher	$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_dec1x
-
-	subi		r0,$taillen,1
-	vncipher	$out0,$out0,v24
-
-	andi.		r0,r0,16
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-
-	sub		$inp,$inp,r0
-	vncipher	$out0,$out0,v26
-
-	lvx_u		$in0,0,$inp
-	vncipher	$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk0,$twk0,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out0,$out0,v30
-
-	mtctr		$rounds
-	vncipherlast	$out0,$out0,$twk0
-
-	vmr		$twk0,$twk1		# unused tweak
-	vmr		$twk1,$twk2
-	le?vperm	$out0,$out0,$out0,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	addi		$out,$out,0x10
-	vxor		$out0,$in0,$twk2
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_zero:
-	cmpwi		$taillen,0
-	beq		Lxts_dec6x_done
-
-	lvx_u		$in0,0,$inp
-	le?vperm	$in0,$in0,$in0,$leperm
-	vxor		$out0,$in0,$twk1
-Lxts_dec6x_steal:
-	vncipher	$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Lxts_dec6x_steal
-
-	add		$inp,$inp,$taillen
-	vncipher	$out0,$out0,v24
-
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-
-	lvx_u		$in0,0,$inp
-	vncipher	$out0,$out0,v26
-
-	lvsr		$inpperm,0,$taillen	# $in5 is no more
-	vncipher	$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk1,$twk1,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out0,$out0,v30
-
-	vperm		$in0,$in0,$in0,$inpperm
-	vncipherlast	$tmp,$out0,$twk1
-
-	le?vperm	$out0,$tmp,$tmp,$leperm
-	le?stvx_u	$out0,0,$out
-	be?stvx_u	$tmp,0,$out
-
-	vxor		$out0,$out0,$out0
-	vspltisb	$out1,-1
-	vperm		$out0,$out0,$out1,$inpperm
-	vsel		$out0,$in0,$tmp,$out0
-	vxor		$out0,$out0,$twk0
-
-	subi		r30,$out,1
-	mtctr		$taillen
-Loop_xts_dec6x_steal:
-	lbzu		r0,1(r30)
-	stb		r0,16(r30)
-	bdnz		Loop_xts_dec6x_steal
-
-	li		$taillen,0
-	mtctr		$rounds
-	b		Loop_xts_dec1x		# one more time...
-
-.align	4
-Lxts_dec6x_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_dec6x_ret
-
-	vxor		$tweak,$twk0,$rndkey0
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_dec6x_ret:
-	mtlr		r11
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$seven,r10,$sp		# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x04,1,0x80,6,6,0
-	.long		0
-
-.align	5
-_aesp8_xts_dec5x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		_aesp8_xts_dec5x
-
-	subi		r0,$taillen,1
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-
-	andi.		r0,r0,16
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	 vxor		$twk0,$twk0,v31
-
-	sub		$inp,$inp,r0
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	 vxor		$in1,$twk1,v31
-
-	vncipher	$out0,$out0,v27
-	lvx_u		$in0,0,$inp
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	 vxor		$in2,$twk2,v31
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vxor		$in3,$twk3,v31
-
-	vncipher	$out0,$out0,v29
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$in4,$twk4,v31
-
-	vncipher	$out0,$out0,v30
-	vncipher	$out1,$out1,v30
-	vncipher	$out2,$out2,v30
-	vncipher	$out3,$out3,v30
-	vncipher	$out4,$out4,v30
-
-	vncipherlast	$out0,$out0,$twk0
-	vncipherlast	$out1,$out1,$in1
-	vncipherlast	$out2,$out2,$in2
-	vncipherlast	$out3,$out3,$in3
-	vncipherlast	$out4,$out4,$in4
-	mtctr		$rounds
-	blr
-        .long   	0
-        .byte   	0,12,0x14,0,0,0,0,0
-___
-}}	}}}
-
-my $consts=1;
-foreach(split("\n",$code)) {
-        s/\`([^\`]*)\`/eval($1)/geo;
-
-	# constants table endian-specific conversion
-	if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
-	    my $conv=$3;
-	    my @bytes=();
-
-	    # convert to endian-agnostic format
-	    if ($1 eq "long") {
-	      foreach (split(/,\s*/,$2)) {
-		my $l = /^0/?oct:int;
-		push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
-	      }
-	    } else {
-		@bytes = map(/^0/?oct:int,split(/,\s*/,$2));
-	    }
-
-	    # little-endian conversion
-	    if ($flavour =~ /le$/o) {
-		SWITCH: for($conv)  {
-		    /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
-		    /\?rev/ && do   { @bytes=reverse(@bytes);    last; };
-		}
-	    }
-
-	    #emit
-	    print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
-	    next;
-	}
-	$consts=0 if (m/Lconsts:/o);	# end of table
-
-	# instructions prefixed with '?' are endian-specific and need
-	# to be adjusted accordingly...
-	if ($flavour =~ /le$/o) {	# little-endian
-	    s/le\?//o		or
-	    s/be\?/#be#/o	or
-	    s/\?lvsr/lvsl/o	or
-	    s/\?lvsl/lvsr/o	or
-	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
-	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
-	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
-	} else {			# big-endian
-	    s/le\?/#le#/o	or
-	    s/be\?//o		or
-	    s/\?([a-z]+)/$1/o;
-	}
-
-        print $_,"\n";
-}
-
-close STDOUT;
diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c
index 0b725e826388..7d2beb774f99 100644
--- a/arch/powerpc/crypto/vmx.c
+++ b/arch/powerpc/crypto/vmx.c
@@ -27,13 +27,9 @@ static int __init p8_init(void)
 	if (ret)
 		goto err;
 
-	ret = crypto_register_alg(&p8_aes_alg);
-	if (ret)
-		goto err_unregister_ghash;
-
 	ret = crypto_register_skcipher(&p8_aes_cbc_alg);
 	if (ret)
-		goto err_unregister_aes;
+		goto err_unregister_ghash;
 
 	ret = crypto_register_skcipher(&p8_aes_ctr_alg);
 	if (ret)
@@ -49,8 +45,6 @@ err_unregister_aes_ctr:
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 err_unregister_aes_cbc:
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-err_unregister_aes:
-	crypto_unregister_alg(&p8_aes_alg);
 err_unregister_ghash:
 	crypto_unregister_shash(&p8_ghash_alg);
 err:
@@ -62,7 +56,6 @@ static void __exit p8_exit(void)
 	crypto_unregister_skcipher(&p8_aes_xts_alg);
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-	crypto_unregister_alg(&p8_aes_alg);
 	crypto_unregister_shash(&p8_ghash_alg);
 }
 
@@ -74,4 +67,3 @@ MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions "
 		   "support on Power 8");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("1.0.0");
-MODULE_IMPORT_NS("CRYPTO_INTERNAL");