SSE2 fold.singeli
This commit is contained in:
parent
1d6b33d528
commit
b3f3190028
@ -599,12 +599,11 @@ cachedBin‿linkerCache ← {
|
||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
||||
"xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy",
|
||||
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
||||
"xa."‿"src/builtins/search.c"‿"search",
|
||||
"xa."‿"src/builtins/search.c"‿"search", "x.."‿"src/builtins/fold.c"‿"fold",
|
||||
|
||||
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
||||
"2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash",
|
||||
"2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq",
|
||||
"2.."‿"src/builtins/slash.c"‿"count"
|
||||
"2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
#include "../builtins.h"
|
||||
#include "../utils/mut.h"
|
||||
|
||||
#if SINGELI_AVX2
|
||||
#if SINGELI_X86_64
|
||||
#define SINGELI_FILE fold
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
@ -92,8 +92,8 @@ B sum_c1(B t, B x) {
|
||||
}
|
||||
r += s;
|
||||
} else {
|
||||
#if SINGELI_AVX2
|
||||
r = avx2_sum_f64(xv, ia);
|
||||
#if SINGELI_X86_64
|
||||
r = simd_sum_f64(xv, ia);
|
||||
#else
|
||||
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
|
||||
#endif
|
||||
@ -133,9 +133,9 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
|
||||
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
|
||||
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
|
||||
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
|
||||
#if SINGELI_AVX2
|
||||
static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); }
|
||||
static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); }
|
||||
#if SINGELI_X86_64
|
||||
static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); }
|
||||
static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); }
|
||||
#else
|
||||
DEF_MIN_MAX(f64)
|
||||
#endif
|
||||
|
||||
@ -1,11 +1,19 @@
|
||||
include './base'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
|
||||
def opsh{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
|
||||
def mix{op, v:[4]f64} = { def sh=opsh{op}; sh{sh{v, 4b2301}, 4b1032} }
|
||||
def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
|
||||
def opsh32{op}{v:[2]f64, perm} = op{v, shuf{[4]u32, v, perm}}
|
||||
def mix{op, v:[4]f64 & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} }
|
||||
def mix{op, v:[2]f64 & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032}
|
||||
|
||||
def reduce_pairwise{op, plog, x:*T, len, init:T} = {
|
||||
# Pairwise combination to shorten dependency chains
|
||||
@ -22,35 +30,39 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = {
|
||||
}
|
||||
|
||||
fn fold_idem{T==f64, op}(x:*T, len:u64) : T = {
|
||||
def step = 256/width{T}
|
||||
def V = [step]T
|
||||
def bulk = arch_defvw/width{T}
|
||||
def V = [bulk]T
|
||||
xv:= *V ~~ x
|
||||
r:V = [4]f64**0
|
||||
if (len < step) {
|
||||
r:V = [bulk]f64**0
|
||||
if (len < bulk) {
|
||||
# Can't overlap like the long case
|
||||
assert{len > 0}
|
||||
r = load{xv}
|
||||
if (len > 1) {
|
||||
if (len > 2) r = opsh{op}{r, 4b2222}
|
||||
r = opsh{op}{r, 4b1111}
|
||||
if (bulk==4 and hasarch{'AVX'}) {
|
||||
if (len > 1) {
|
||||
if (len > 2) r = opsh64{op}{r, 4b2222}
|
||||
r = opsh64{op}{r, 4b1111}
|
||||
}
|
||||
} else {
|
||||
assert{bulk==2}
|
||||
}
|
||||
} else {
|
||||
i:= load{*V ~~ (x+len-step)}
|
||||
r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/step, i}}
|
||||
i:= load{*V ~~ (x+len-bulk)}
|
||||
r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/bulk, i}}
|
||||
}
|
||||
extract{r, 0}
|
||||
}
|
||||
|
||||
export{'avx2_fold_min_f64', fold_idem{f64,min}}
|
||||
export{'avx2_fold_max_f64', fold_idem{f64,max}}
|
||||
export{'simd_fold_min_f64', fold_idem{f64,min}}
|
||||
export{'simd_fold_max_f64', fold_idem{f64,max}}
|
||||
|
||||
fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
|
||||
def step = 256/width{T}
|
||||
def V = [step]T
|
||||
def bulk = arch_defvw/width{T}
|
||||
def V = [bulk]T
|
||||
xv:= *V ~~ x
|
||||
e:= len / step
|
||||
i:= load{xv, e} & (V~~maskOf{V, len % step})
|
||||
e:= len / bulk
|
||||
i:= load{xv, e} & (V~~maskOf{V, len % bulk})
|
||||
r:= reduce_pairwise{op, 2, xv, e, i}
|
||||
extract{mix{op, r}, 0}
|
||||
}
|
||||
export{'avx2_sum_f64', fold_assoc_0{f64,+}}
|
||||
export{'simd_sum_f64', fold_assoc_0{f64,+}}
|
||||
|
||||
@ -194,8 +194,9 @@ def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
|
||||
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
|
||||
|
||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||
def extract{x:T, i==0 & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
||||
def extract{x:T, i==0 & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
|
||||
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
||||
def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
|
||||
def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x}
|
||||
|
||||
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user