From a62c21733ea37c4aca5dbb1f2b44d0296d436a86 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 1 Apr 2023 22:07:11 +0300 Subject: [PATCH] SSE2 equal.singeli --- build/src/build.bqn | 2 +- src/core/stuff.c | 4 ++-- src/singeli/src/base.singeli | 1 + src/singeli/src/equal.singeli | 40 ++++++++++++++++++----------------- src/singeli/src/sse.singeli | 3 ++- src/singeli/src/sse2.singeli | 4 +++- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index d88e6a65..a4269346 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -590,7 +590,7 @@ cachedBin‿linkerCache ← { ⟩ cbqnSrc ↩ cbqnSrc clangd.Files "src" singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨ - "xa."‿"src/builtins/arithm.c"‿"monarith", "2a."‿"src/core/stuff.c"‿"equal", + "xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal", "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "2a."‿"src/builtins/squeeze.c"‿"squeeze", "2a."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", diff --git a/src/core/stuff.c b/src/core/stuff.c index a5bfc534..dc757fa5 100644 --- a/src/core/stuff.c +++ b/src/core/stuff.c @@ -420,8 +420,8 @@ u8 eqFnData[] = { // for the main diagonal, amount to shift length by; otherwise n,n,n,n,n,1,1,2, }; -#if SINGELI_SIMD - #define F(X) avx2_equal_##X +#if SINGELI_SIMD || SINGELI_X86_64 + #define F(X) simd_equal_##X #define SINGELI_FILE equal #include "../utils/includeSingeli.h" #else diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index f6cda783..eb47fa78 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -164,6 +164,7 @@ def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}} def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}} def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}} def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}} +def widen{...x} = assert{'widen not supported', show{...x}} def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index fddb0457..420483ee 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -1,10 +1,12 @@ include './base' include './cbqnDefs' include './f64' -if (hasarch{'X86_64'}) { +if (hasarch{'AVX2'}) { include './sse' include './avx' include './avx2' +} else if (hasarch{'X86_64'}) { + include './sse2' } else if (hasarch{'AARCH64'}) { include './neon' } @@ -36,9 +38,9 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { f0:= TF**0.0 f1:= TF**1.0 maskedLoopPositive{bulk, l, {i, M} => { - wu:= (if (hasarch{'X86_64'}) { + wu:= (if (hasarch{'AVX2'}) { cw:= b_getBatchLo{bulk, *u64~~w, i} - topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}} + topBlend{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}} } else { cw:= loadBatchBit{TU, *u64~~w, i} homBlend{f0, f1, cw} @@ -77,23 +79,23 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { 1 } -export{'avx2_equal_1_1', equal{u1, u1}} -export{'avx2_equal_1_8', equal{u1, u8}} -export{'avx2_equal_1_16', equal{u1, u16}} -export{'avx2_equal_1_32', equal{u1, u32}} -export{'avx2_equal_1_f64', equal{u1, f64}} +export{'simd_equal_1_1', equal{u1, u1}} +export{'simd_equal_1_8', equal{u1, u8}} +export{'simd_equal_1_16', equal{u1, u16}} +export{'simd_equal_1_32', equal{u1, u32}} +export{'simd_equal_1_f64', equal{u1, f64}} -export{'avx2_equal_8_8', equal{i8, i8}} +export{'simd_equal_8_8', equal{i8, i8}} -export{'avx2_equal_s8_16', equal{i8, i16}} -export{'avx2_equal_s8_32', equal{i8, i32}} -export{'avx2_equal_s16_32', equal{i16, i32}} +export{'simd_equal_s8_16', equal{i8, i16}} +export{'simd_equal_s8_32', equal{i8, i32}} +export{'simd_equal_s16_32', equal{i16, i32}} -export{'avx2_equal_s8_f64', equal{i8, f64}} -export{'avx2_equal_s16_f64', equal{i16, f64}} -export{'avx2_equal_s32_f64', equal{i32, f64}} -export{'avx2_equal_f64_f64', equal{f64, f64}} +export{'simd_equal_s8_f64', equal{i8, f64}} +export{'simd_equal_s16_f64', equal{i16, f64}} +export{'simd_equal_s32_f64', equal{i32, f64}} +export{'simd_equal_f64_f64', equal{f64, f64}} -export{'avx2_equal_u8_16', equal{u8, u16}} -export{'avx2_equal_u8_32', equal{u8, u32}} -export{'avx2_equal_u16_32', equal{u16, u32}} +export{'simd_equal_u8_16', equal{u8, u16}} +export{'simd_equal_u8_32', equal{u8, u32}} +export{'simd_equal_u16_32', equal{u16, u32}} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index d31db747..fc40c888 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -41,7 +41,8 @@ def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x} def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} -# mask +# mask stuff +def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 627705ec..0c51c81b 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -159,6 +159,7 @@ def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}} def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}} # mask stuff +def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0} def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} @@ -196,7 +197,8 @@ def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n} def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X} -def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24} +def widen{T, x:X & w128i{T,32} & w128i{X,8} & w128s{T}==w128s{X}} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24} +def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m)) def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}