SSE2 fold.singeli

This commit is contained in:
dzaima 2023-04-06 19:49:08 +03:00
parent 1d6b33d528
commit b3f3190028
4 changed files with 44 additions and 32 deletions

View File

@ -599,12 +599,11 @@ cachedBin‿linkerCache ← {
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp",
"xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose",
"xa.""src/builtins/search.c""search",
"xa.""src/builtins/search.c""search", "x..""src/builtins/fold.c""fold",
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",
"2..""src/builtins/fold.c""fold", "2..""src/builtins/slash.c""slash",
"2..""src/builtins/slash.c""constrep", "2..""src/builtins/scan.c""neq",
"2..""src/builtins/slash.c""count"
"2..""src/builtins/slash.c""slash", "2..""src/builtins/slash.c""count"
objs

View File

@ -15,7 +15,7 @@
#include "../builtins.h"
#include "../utils/mut.h"
#if SINGELI_AVX2
#if SINGELI_X86_64
#define SINGELI_FILE fold
#include "../utils/includeSingeli.h"
#endif
@ -92,8 +92,8 @@ B sum_c1(B t, B x) {
}
r += s;
} else {
#if SINGELI_AVX2
r = avx2_sum_f64(xv, ia);
#if SINGELI_X86_64
r = simd_sum_f64(xv, ia);
#else
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
#endif
@ -133,9 +133,9 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
#if SINGELI_AVX2
static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); }
static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); }
#if SINGELI_X86_64
static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); }
static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); }
#else
DEF_MIN_MAX(f64)
#endif

View File

@ -1,11 +1,19 @@
include './base'
include './sse'
include './avx'
include './avx2'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './mask'
def opsh{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
def mix{op, v:[4]f64} = { def sh=opsh{op}; sh{sh{v, 4b2301}, 4b1032} }
def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
def opsh32{op}{v:[2]f64, perm} = op{v, shuf{[4]u32, v, perm}}
def mix{op, v:[4]f64 & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} }
def mix{op, v:[2]f64 & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032}
def reduce_pairwise{op, plog, x:*T, len, init:T} = {
# Pairwise combination to shorten dependency chains
@ -22,35 +30,39 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = {
}
fn fold_idem{T==f64, op}(x:*T, len:u64) : T = {
def step = 256/width{T}
def V = [step]T
def bulk = arch_defvw/width{T}
def V = [bulk]T
xv:= *V ~~ x
r:V = [4]f64**0
if (len < step) {
r:V = [bulk]f64**0
if (len < bulk) {
# Can't overlap like the long case
assert{len > 0}
r = load{xv}
if (len > 1) {
if (len > 2) r = opsh{op}{r, 4b2222}
r = opsh{op}{r, 4b1111}
if (bulk==4 and hasarch{'AVX'}) {
if (len > 1) {
if (len > 2) r = opsh64{op}{r, 4b2222}
r = opsh64{op}{r, 4b1111}
}
} else {
assert{bulk==2}
}
} else {
i:= load{*V ~~ (x+len-step)}
r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/step, i}}
i:= load{*V ~~ (x+len-bulk)}
r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/bulk, i}}
}
extract{r, 0}
}
export{'avx2_fold_min_f64', fold_idem{f64,min}}
export{'avx2_fold_max_f64', fold_idem{f64,max}}
export{'simd_fold_min_f64', fold_idem{f64,min}}
export{'simd_fold_max_f64', fold_idem{f64,max}}
fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
def step = 256/width{T}
def V = [step]T
def bulk = arch_defvw/width{T}
def V = [bulk]T
xv:= *V ~~ x
e:= len / step
i:= load{xv, e} & (V~~maskOf{V, len % step})
e:= len / bulk
i:= load{xv, e} & (V~~maskOf{V, len % bulk})
r:= reduce_pairwise{op, 2, xv, e, i}
extract{mix{op, r}, 0}
}
export{'avx2_sum_f64', fold_assoc_0{f64,+}}
export{'simd_sum_f64', fold_assoc_0{f64,+}}

View File

@ -194,8 +194,9 @@ def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
def extract{x:T, i==0 & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
def extract{x:T, i==0 & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x}
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}