From b3f31900283980f6622dabf90c35f25477bbcfa9 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 6 Apr 2023 19:49:08 +0300 Subject: [PATCH] SSE2 fold.singeli --- build/src/build.bqn | 5 ++-- src/builtins/fold.c | 12 ++++---- src/singeli/src/fold.singeli | 54 ++++++++++++++++++++++-------------- src/singeli/src/sse2.singeli | 5 ++-- 4 files changed, 44 insertions(+), 32 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index b354f478..889cb44f 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -599,12 +599,11 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xa."‿"src/builtins/search.c"‿"search", + "xa."‿"src/builtins/search.c"‿"search", "x.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", - "2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", - "2.."‿"src/builtins/slash.c"‿"count" + "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 8c1fe1a8..4026d695 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -15,7 +15,7 @@ #include "../builtins.h" #include "../utils/mut.h" -#if SINGELI_AVX2 +#if SINGELI_X86_64 #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif @@ -92,8 +92,8 @@ B sum_c1(B t, B x) { } r += s; } else { - #if SINGELI_AVX2 - r = avx2_sum_f64(xv, ia); + #if SINGELI_X86_64 + r = simd_sum_f64(xv, ia); #else r=0; for (usz i=0; i) } DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32) -#if SINGELI_AVX2 - static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); } - static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); } +#if SINGELI_X86_64 + static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); } + static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); } #else DEF_MIN_MAX(f64) #endif diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 8de34d67..c1548020 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,11 +1,19 @@ include './base' -include './sse' -include './avx' -include './avx2' +if (hasarch{'AVX2'}) { + include './sse' + include './avx' + include './avx2' +} else if (hasarch{'X86_64'}) { + include './sse2' +} else if (hasarch{'AARCH64'}) { + include './neon' +} include './mask' -def opsh{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}} -def mix{op, v:[4]f64} = { def sh=opsh{op}; sh{sh{v, 4b2301}, 4b1032} } +def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}} +def opsh32{op}{v:[2]f64, perm} = op{v, shuf{[4]u32, v, perm}} +def mix{op, v:[4]f64 & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} } +def mix{op, v:[2]f64 & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032} def reduce_pairwise{op, plog, x:*T, len, init:T} = { # Pairwise combination to shorten dependency chains @@ -22,35 +30,39 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = { } fn fold_idem{T==f64, op}(x:*T, len:u64) : T = { - def step = 256/width{T} - def V = [step]T + def bulk = arch_defvw/width{T} + def V = [bulk]T xv:= *V ~~ x - r:V = [4]f64**0 - if (len < step) { + r:V = [bulk]f64**0 + if (len < bulk) { # Can't overlap like the long case assert{len > 0} r = load{xv} - if (len > 1) { - if (len > 2) r = opsh{op}{r, 4b2222} - r = opsh{op}{r, 4b1111} + if (bulk==4 and hasarch{'AVX'}) { + if (len > 1) { + if (len > 2) r = opsh64{op}{r, 4b2222} + r = opsh64{op}{r, 4b1111} + } + } else { + assert{bulk==2} } } else { - i:= load{*V ~~ (x+len-step)} - r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/step, i}} + i:= load{*V ~~ (x+len-bulk)} + r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/bulk, i}} } extract{r, 0} } -export{'avx2_fold_min_f64', fold_idem{f64,min}} -export{'avx2_fold_max_f64', fold_idem{f64,max}} +export{'simd_fold_min_f64', fold_idem{f64,min}} +export{'simd_fold_max_f64', fold_idem{f64,max}} fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = { - def step = 256/width{T} - def V = [step]T + def bulk = arch_defvw/width{T} + def V = [bulk]T xv:= *V ~~ x - e:= len / step - i:= load{xv, e} & (V~~maskOf{V, len % step}) + e:= len / bulk + i:= load{xv, e} & (V~~maskOf{V, len % bulk}) r:= reduce_pairwise{op, 2, xv, e, i} extract{mix{op, r}, 0} } -export{'avx2_sum_f64', fold_assoc_0{f64,+}} +export{'simd_sum_f64', fold_assoc_0{f64,+}} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 746cb1b1..7c8fef3f 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -194,8 +194,9 @@ def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}} def unpackQ{a:T, b:T & w128{T}} = unpack{a, b} def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} -def extract{x:T, i==0 & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si32', x} -def extract{x:T, i==0 & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si64', x} +def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x} +def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x} +def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x} def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}