Add scalar AArch64 Keccak-f1600 ASM (#133)

* Add first AArch64 Keccak-f1600 ASM Signed-off-by: Hanno Becker <[email protected]> * Update scalar Keccak ASM with A55-optimized version This should perform decent on most microarchitectures. Signed-off-by: Hanno Becker <[email protected]> * Minor cleanup of auto-generated scalar Keccak-f1600 assembly Signed-off-by: Hanno Becker <[email protected]> --------- Signed-off-by: Hanno Becker <[email protected]>
pq-code-package · Sep 17, 2024 · 861095b · 861095b
1 parent 640005d
commit 861095b
Show file tree

Hide file tree

Showing 7 changed files with 527 additions and 19 deletions.
diff --git a/fips202/asm/aarch64/common.i b/fips202/asm/aarch64/common.i
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: MIT
+
+#if __APPLE__
+#define ASM_LOAD(dst, symbol)                                                  \
+  adrp dst, symbol @PAGE %% add dst, dst, symbol @PAGEOFF
+#else
+#define ASM_LOAD(dst, symbol)                                                  \
+  adrp dst, symbol;                                                            \
+  add dst, dst, : lo12 : symbol;
+.endm
+
+#endif
diff --git a/fips202/asm/aarch64/keccak_f1600_x1_scalar_opt_a55.S b/fips202/asm/aarch64/keccak_f1600_x1_scalar_opt_a55.S
diff --git a/fips202/asm/asm.h b/fips202/asm/asm.h
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+#ifndef ASM_H
+#define ASM_H
+
+#include <stdint.h>
+#include "params.h"
+#include "config.h"
+
+#ifdef MLKEM_USE_AARCH64_ASM
+void keccak_f1600_x1_scalar_slothy_opt_a55(uint64_t *state);
+
+#define keccak_f1600_x1_asm keccak_f1600_x1_scalar_slothy_opt_a55
+#endif /* MLKEM_USE_AARCH64_ASM */
+
+
+#endif
diff --git a/fips202/keccakf1600.c b/fips202/keccakf1600.c
@@ -11,9 +11,31 @@
 #include <assert.h>
 #include "keccakf1600.h"
 
+#include "asm/asm.h"
+#include "config.h"
+
 #define NROUNDS 24
 #define ROL(a, offset) ((a << offset) ^ (a >> (64-offset)))
 
+void KeccakF1600_StateExtractBytes(uint64_t *state, unsigned char *data, unsigned int offset, unsigned int length)
+{
+    unsigned int i;
+    for (i = 0; i < length; i++)
+    {
+        data[i] = state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07));
+    }
+}
+
+void KeccakF1600_StateXORBytes(uint64_t *state, const unsigned char *data, unsigned int offset, unsigned int length)
+{
+    unsigned int i;
+    for (i = 0; i < length; i++)
+    {
+        state[(offset + i) >> 3] ^= (uint64_t)data[i] << (8 * ((offset + i) & 0x07));
+    }
+}
+
+#if !defined(MLKEM_USE_AARCH64_ASM)
 static const uint64_t KeccakF_RoundConstants[NROUNDS] =
 {
     (uint64_t)0x0000000000000001ULL,
@@ -42,24 +64,6 @@ static const uint64_t KeccakF_RoundConstants[NROUNDS] =
     (uint64_t)0x8000000080008008ULL
 };
 
-void KeccakF1600_StateExtractBytes(uint64_t *state, unsigned char *data, unsigned int offset, unsigned int length)
-{
-    unsigned int i;
-    for (i = 0; i < length; i++)
-    {
-        data[i] = state[(offset + i) >> 3] >> (8 * ((offset + i) & 0x07));
-    }
-}
-
-void KeccakF1600_StateXORBytes(uint64_t *state, const unsigned char *data, unsigned int offset, unsigned int length)
-{
-    unsigned int i;
-    for (i = 0; i < length; i++)
-    {
-        state[(offset + i) >> 3] ^= (uint64_t)data[i] << (8 * ((offset + i) & 0x07));
-    }
-}
-
 void KeccakF1600_StatePermute(uint64_t *state)
 {
     int round;
@@ -326,3 +330,4 @@ void KeccakF1600_StatePermute(uint64_t *state)
 
 #undef    round
 }
+#endif /* !MLKEM_USE_AARCH64_ASM */
diff --git a/fips202/keccakf1600.h b/fips202/keccakf1600.h
@@ -2,10 +2,16 @@
 #ifndef KECCAKF1600_H
 #define KECCAKF1600_H
 
+#include "asm/asm.h"
 #include <stdint.h>
 
 void KeccakF1600_StateExtractBytes(uint64_t *state, unsigned char *data, unsigned int offset, unsigned int length);
 void KeccakF1600_StateXORBytes(uint64_t *state, const unsigned char *data, unsigned int offset, unsigned int length);
+
+#if !defined(MLKEM_USE_AARCH64_ASM)
 void KeccakF1600_StatePermute(uint64_t *state);
+#else
+#define KeccakF1600_StatePermute keccak_f1600_x1_asm
+#endif
 
 #endif
diff --git a/mk/crypto.mk b/mk/crypto.mk
@@ -15,9 +15,15 @@ else
 	CPPFLAGS += -Irandombytes
 endif
 
+FIPS202_SRCS = $(wildcard fips202/*.c)
+ifeq ($(OPT),1)
+	FIPS202_SRCS += $(wildcard fips202/asm/aarch64/*.S)
+	CPPFLAGS += -DMLKEM_USE_ASM
+endif
+
 $(LIB_DIR)/librng.a: $(call OBJS,$(wildcard randombytes/*.c))
 
 $(LIB_DIR)/libnistrng.a: CFLAGS += -Wno-unused-result -O3 -fomit-frame-pointer
 $(LIB_DIR)/libnistrng.a: $(call OBJS,$(wildcard test/nistrng/*.c))
 
-$(LIB_DIR)/libfips202.a: $(call OBJS,$(wildcard fips202/*.c))
+$(LIB_DIR)/libfips202.a: $(call OBJS, $(FIPS202_SRCS))
diff --git a/mk/rules.mk b/mk/rules.mk
@@ -15,6 +15,11 @@ $(BUILD_DIR)/%.c.o: %.c $(CONFIG)
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
 	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
 
+$(BUILD_DIR)/%.S.o: %.S $(CONFIG)
+	$(Q)echo "  AS      $@"
+	$(Q)[ -d $(@D) ] || mkdir -p $(@D)
+	$(Q)$(CC) -c -o $@ $(CFLAGS) $<
+
 $(BUILD_DIR)/mlkem512/%.c.o: %.c $(CONFIG)
 	$(Q)echo "  CC      $@"
 	$(Q)[ -d $(@D) ] || mkdir -p $(@D)