Unify assembler files between Linux and Windows.

CMakeLists.txt

@@ -48,7 +48,7 @@ add_compile_options(
 	-Wno-microsoft-enum-forward-reference
 	-Wno-visibility
 	-Wno-microsoft-anon-tag
-	-Wno-ignored-attributes
+	# -Wno-ignored-attributes
 	-Wno-unused-command-line-argument
 	-Wno-unused-local-typedef
 	-Wno-int-to-void-pointer-cast

include/os/windows/spl/sys/debug.h

@@ -62,12 +62,12 @@
 
 #define	unlikely
 #define	likely
-#define	__attribute__(X)
 #define	__maybe_unused
 #define	__printflike(X, Y)
 #define	__unused
 #define	always_inline __forceinline
-#define	_Noreturn
+#define	_Noreturn __declspec(noreturn)
+
 
 #else
 
@@ -88,12 +88,11 @@
 #define	__unused  __attribute__((unused))
 #define	_Noreturn	__attribute__((__noreturn__))
 
-
 #endif
 
+extern void _Noreturn panic(const char *fmt, ...);
 
 
-extern void panic(const char *fmt, ...) __attribute__((__noreturn__));
 
 extern void printBuffer(const char *fmt, ...);
 

lib/libspl/include/os/windows/win/asm_linkage.h → include/os/windows/spl/sys/ia32/asm_linkage.h

@@ -27,18 +27,23 @@
 #ifndef _IA32_SYS_ASM_LINKAGE_H
 #define	_IA32_SYS_ASM_LINKAGE_H
 
-#if defined(__linux__) && defined(CONFIG_SLS)
-#define	RET	ret; int3
-#else
 #define	RET	ret
-#endif
+
+/* Tell compiler to call assembler like Unix */
+#define	ASMABI	__attribute__((sysv_abi))
+
+#define	ENDBR
+
+#define	SECTION_TEXT .text
+#define	SECTION_STATIC .data
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 #ifdef _ASM	/* The remainder of this file is only for assembly files */
 
+
 /*
  * make annoying differences in assembler syntax go away
  */
@@ -154,6 +159,9 @@ x:; \
  */
 #define	SET_SIZE(x)
 
+#define	SET_OBJ(x)
+
+
 #endif /* _ASM */
 
 #ifdef	__cplusplus

include/os/windows/spl/sys/linker_set.h

@@ -47,7 +47,7 @@
 #define	__GLOBL(sym)	__asm__(".globl " __XSTRING(sym))
 #define	__WEAK(sym)	__asm__(".weak " __XSTRING(sym))
 
-#define	__weak_symbol	__attribute__((__weak__))
+#define	__weak_symbol	/* __attribute__((__weak__)) */
 
 #if __has_attribute(no_sanitize) && defined(__clang__)
 #ifdef _KERNEL

include/os/windows/spl/sys/mod_os.h

@@ -87,7 +87,6 @@ extern "C" {
 		fn();		\
 	}
 
-#define	module_param_call(a, b, c, d, e)
 #define	module_param_named(a, b, c, d)
 
 #define	module_init_early(fn)	\
@@ -304,10 +303,10 @@ ZT_GET_VALUE(ztunable_t *zt, void **ptr, ULONG *len, ULONG *type)
 
 
 #define	ZFS_MODULE_PARAM_CALL_IMPL( \
-    scope_prefix, name_prefix, name, perm, args, desc) \
+    scope_prefix, name_prefix, name, perm, func, args, desc) \
 	static ztunable_t zt_ ## name_prefix ## name = { \
-		.zt_ptr = &name_prefix ## name, \
-		.zt_func = args, \
+		.zt_ptr = args, \
+		.zt_func = func, \
 		.zt_name = #name_prefix #name, \
 		.zt_prefix = #scope_prefix, \
 		.zt_desc = #desc, \
@@ -320,9 +319,17 @@ ZT_GET_VALUE(ztunable_t *zt, void **ptr, ULONG *len, ULONG *type)
 #define	ZFS_MODULE_PARAM_CALL( \
     scope_prefix, name_prefix, name, func, _, perm, desc) \
 	ZFS_MODULE_PARAM_CALL_IMPL(scope_prefix, name_prefix, name, perm, \
-	    func, desc)
+	    func, &name_prefix ## name, desc)
+
+#define	ZFS_MODULE_VIRTUAL_PARAM_CALL( \
+    scope_prefix, name_prefix, name, func, _, perm, desc) \
+	ZFS_MODULE_PARAM_CALL_IMPL(scope_prefix, name_prefix, name, perm, \
+	    win32_ ## func, NULL, desc)
 
-#define	ZFS_MODULE_VIRTUAL_PARAM_CALL ZFS_MODULE_PARAM_CALL
+#define	module_param_call(name, _set, _get, var, mode) \
+	extern int win32_ ## _set(ZFS_MODULE_PARAM_ARGS);	\
+	ZFS_MODULE_PARAM_CALL_IMPL(zfs, /* */, name, ZMOD_RW,	\
+		win32_ ## _set, var, "xxx")
 
 struct zfs_kernel_param_s;
 typedef struct zfs_kernel_param_s zfs_kernel_param_t;

include/os/windows/spl/sys/processor.h

@@ -8,4 +8,21 @@ extern uint32_t getcpuid();
 
 typedef int	processorid_t;
 
+#define	CPUID_FEATURE_PCLMULQDQ		(1<<1)
+#define	CPUID_FEATURE_MOVBE		(1<<22)
+#define	CPUID_FEATURE_AES		(1<<25)
+#define	CPUID_FEATURE_XSAVE		(1<<26)
+#define	CPUID_FEATURE_OSXSAVE		(1<<27)
+#define	CPUID_FEATURE_AVX1_0		(1<<28)
+
+#define	CPUID_FEATURE_SSE		(1<<25)
+#define	CPUID_FEATURE_SSE2		(1<<26)
+#define	CPUID_FEATURE_SSE3		(1<<0)
+#define	CPUID_FEATURE_SSSE3		(1<<9)
+#define	CPUID_FEATURE_SSE4_2		(1<<20)
+#define	CPUID_FEATURE_SSE4_1		(1<<19)
+
+#define	CPUID_LEAF7_FEATURE_AVX2    (1<<5)
+#define	CPUID_LEAF7_FEATURE_AVX512F    (1<<16)
+
 #endif /* _SPL_PROCESSOR_H */

include/os/windows/spl/sys/simd.h

@@ -68,6 +68,7 @@
 #define	_SIMD_X86_H
 
 #include <sys/isa_defs.h>
+#include <sys/processor.h>
 
 /* only for __x86 */
 #if defined(__x86)
@@ -92,11 +93,6 @@ xgetbv(uint32_t c)
 
 #endif
 
-#define	CPUID_FEATURE_PCLMULQDQ		(1<<1)
-#define	CPUID_FEATURE_AES		(1<<25)
-#define	CPUID_FEATURE_XSAVE		(1<<26)
-// #define   CPUID_FEATURE_AVX		(1<<28)
-
 extern uint64_t spl_cpuid_features(void);
 extern uint64_t spl_cpuid_leaf7_features(void);
 
@@ -109,8 +105,16 @@ extern uint64_t spl_cpuid_leaf7_features(void);
 #define	kfpu_init()		(0)
 #define	kfpu_fini()		do {} while (0)
 
-#define	kfpu_begin()  ((void)0)
-#define	kfpu_end()    ((void)0)
+extern uint32_t kfpu_state;
+
+#define	kfpu_begin() \
+	NTSTATUS saveStatus = STATUS_INVALID_PARAMETER; \
+	XSTATE_SAVE SaveState; \
+	saveStatus = KeSaveExtendedProcessorState(kfpu_state, &SaveState);
+
+#define	kfpu_end() \
+	if (NT_SUCCESS(saveStatus)) \
+		KeRestoreExtendedProcessorState(&SaveState);
 
 /*
  * CPUID feature tests for user-space. Linux kernel provides an interface for
@@ -284,19 +288,6 @@ CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
 
 #endif /* !defined(_KERNEL) */
 
-// WIN fix me, no asm for now
-#define	CPUID_FEATURE_SSE 0
-#define	CPUID_FEATURE_SSE2 0
-#define	CPUID_FEATURE_SSE3 0
-#define	CPUID_FEATURE_SSSE3 0
-#define	CPUID_FEATURE_SSE4_1 0
-#define	CPUID_FEATURE_SSE4_2 0
-#define	CPUID_FEATURE_OSXSAVE 0
-#define	CPUID_FEATURE_AVX1_0 0
-#define	CPUID_FEATURE_SSE 0
-#define	CPUID_FEATURE_SSE 0
-#define	CPUID_FEATURE_SSE 0
-
 
 /*
  * Detect register set support
@@ -727,6 +718,20 @@ zfs_avx512vbmi_available(void)
 	return (has_avx512 && __zmm_enabled());
 }
 
+static inline boolean_t
+zfs_movbe_available(void)
+{
+#if defined(_KERNEL)
+#if defined(HAVE_MOVBE)
+	return (!!(spl_cpuid_features() & CPUID_FEATURE_MOVBE));
+#else
+	return (B_FALSE);
+#endif
+	return (B_FALSE);
+#endif
+}
+
+
 #endif /* defined(__x86) */
 
 #endif /* _SIMD_X86_H */

include/os/windows/spl/sys/thread.h

@@ -83,7 +83,7 @@ extern kthread_t *spl_thread_create(caddr_t stk, size_t stksize,
 #endif
 
 #define	thread_exit spl_thread_exit
-extern void spl_thread_exit(void);
+extern void __declspec(noreturn) spl_thread_exit(void);
 
 extern kthread_t *spl_current_thread(void);
 

include/os/windows/zfs/zfs_config.h

@@ -64,9 +64,24 @@
 #define	HAVE_USLEEP 1
 
 /* These control which assembler files to use */
-//#define	HAVE_AVX 1
+#define	HAVE_SSE2 1
+#define	HAVE_SSSE3 1
+#define	HAVE_SSE4_1
+#define	HAVE_AVX 1
+#define	HAVE_AVX2 1
 #define	HAVE_PCLMULQDQ 1
+#define	HAVE_MOVBE 1
 #define	HAVE_AES 1
+#define	HAVE_AVX512F 1
+#define	HAVE_AVX512CD 1
+#define	HAVE_AVX512ER 1
+#define	HAVE_AVX512BW 1
+#define	HAVE_AVX512DQ 1
+#define	HAVE_AVX512VL 1
+#define	HAVE_AVX512IFMA 1
+#define	HAVE_AVX512VBMI 1
+#define	HAVE_AVX512PF 1
+
 
 /* Path where the kernel module is installed. */
 #define	KERNEL_MODPREFIX "/Library/Extensions"

module/icp/include/sys/asm_linkage.h → include/sys/asm_linkage.h

@@ -28,13 +28,8 @@
 #define	_SYS_ASM_LINKAGE_H
 
 #if defined(__i386) || defined(__amd64)
-
-#ifdef _WIN32
-#include <win/asm_linkage.h>
-#else
 #include <sys/ia32/asm_linkage.h>	/* XX64	x86/sys/asm_linkage.h */
 #endif
-#endif
 
 #if defined(_KERNEL) && defined(HAVE_KERNEL_OBJTOOL)
 

lib/libicp/CMakeLists.txt

@@ -41,11 +41,18 @@ add_library(libicp
     "${ICP_MODULE_DIR}/spi/kcf_spi.c"
 
     "${ICP_MODULE_DIR}/asm-x86_64/aes/aeskey.c"
-    "${ICP_MODULE_DIR}/asm-x86_64/os/windows/aes/aes_aesni.S"
-    "${ICP_MODULE_DIR}/asm-x86_64/os/windows/aes/aes_amd64.S"
-    "${ICP_MODULE_DIR}/asm-x86_64/os/windows/sha2/sha256_impl.S"
-    "${ICP_MODULE_DIR}/asm-x86_64/os/windows/sha2/sha512_impl.S"
-    "${ICP_MODULE_DIR}/asm-x86_64/os/windows/modes/gcm_pclmulqdq.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/aes/aes_aesni.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/aes/aes_amd64.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/modes/aesni-gcm-x86_64.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/modes/gcm_pclmulqdq.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/modes/ghash-x86_64.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/sha2/sha256_impl.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/sha2/sha512_impl.S"
+
+    "${ICP_MODULE_DIR}/asm-x86_64/blake3/blake3_avx2.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/blake3/blake3_avx512.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/blake3/blake3_sse2.S"
+    "${ICP_MODULE_DIR}/asm-x86_64/blake3/blake3_sse41.S"
 )
 
 # Add windows/assembler sources here too.

module/icp/include/sys/ia32/asm_linkage.h → lib/libspl/include/os/linux/sys/ia32/asm_linkage.h

@@ -56,6 +56,12 @@
 #define	RET	ret
 #endif
 
+/* You can set to nothing on Unix platforms */
+#define	ASMABI	__attribute__((sysv_abi))
+
+#define	SECTION_TEXT .text
+#define	SECTION_STATIC .section .rodata
+
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -185,6 +191,8 @@ x:; \
 #define	SET_SIZE(x) \
 	.size	x, [.-x]
 
+#define	SET_OBJ(x) .type	x, @object
+
 #endif /* _ASM */
 
 #ifdef	__cplusplus

include/os/windows/spl/win/asm_linkage.h → lib/libspl/include/os/windows/sys/ia32/asm_linkage.h

@@ -27,15 +27,20 @@
 #ifndef _IA32_SYS_ASM_LINKAGE_H
 #define	_IA32_SYS_ASM_LINKAGE_H
 
-#include <sys/stack.h>
-#include <sys/trap.h>
-
 #if defined(__linux__) && defined(CONFIG_SLS)
 #define	RET	ret; int3
 #else
 #define	RET	ret
 #endif
 
+/* Tell compiler to call assembler like Unix */
+#define	ASMABI	__attribute__((sysv_abi))
+
+#define	ENDBR
+
+#define	SECTION_TEXT .text
+#define	SECTION_STATIC .data
+
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -157,6 +162,8 @@ x:; \
  */
 #define	SET_SIZE(x)
 
+#define	SET_OBJ(x)
+
 #endif /* _ASM */
 
 #ifdef	__cplusplus

lib/libzpool/CMakeLists.txt

@@ -38,6 +38,9 @@ add_library(libzpool
     "${MODULE_DIR}/zcommon/zfs_comutil.c"
     "${MODULE_DIR}/zcommon/zfs_deleg.c"
     "${MODULE_DIR}/zcommon/zfs_fletcher.c"
+    "${MODULE_DIR}/zcommon/zfs_fletcher_avx512.c"
+    "${MODULE_DIR}/zcommon/zfs_fletcher_intel.c"
+    "${MODULE_DIR}/zcommon/zfs_fletcher_sse.c"
     "${MODULE_DIR}/zcommon/zfs_fletcher_superscalar.c"
     "${MODULE_DIR}/zcommon/zfs_fletcher_superscalar4.c"
     "${MODULE_DIR}/zcommon/zfs_namecheck.c"
@@ -127,7 +130,12 @@ add_library(libzpool
     "${MODULE_DIR}/zfs/vdev_queue.c"
     "${MODULE_DIR}/zfs/vdev_raidz.c"
     "${MODULE_DIR}/zfs/vdev_raidz_math.c"
+    "${MODULE_DIR}/zfs/vdev_raidz_math_avx2.c"
+    "${MODULE_DIR}/zfs/vdev_raidz_math_avx512bw.c"
+    "${MODULE_DIR}/zfs/vdev_raidz_math_avx512f.c"
     "${MODULE_DIR}/zfs/vdev_raidz_math_scalar.c"
+    "${MODULE_DIR}/zfs/vdev_raidz_math_sse2.c"
+    "${MODULE_DIR}/zfs/vdev_raidz_math_ssse3.c"
     "${MODULE_DIR}/zfs/vdev_rebuild.c"
     "${MODULE_DIR}/zfs/vdev_removal.c"
     "${MODULE_DIR}/zfs/vdev_root.c"