aarch64 fold.singeli

This commit is contained in:
dzaima 2023-04-06 20:09:04 +03:00
parent b3f3190028
commit 897c46df76
7 changed files with 23 additions and 28 deletions

View File

@ -599,7 +599,7 @@ cachedBin‿linkerCache ← {
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp",
"xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose",
"xa.""src/builtins/search.c""search", "x..""src/builtins/fold.c""fold",
"xa.""src/builtins/search.c""search", "xa.""src/builtins/fold.c""fold",
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",
"2..""src/builtins/slash.c""constrep", "2..""src/builtins/scan.c""neq",

View File

@ -15,7 +15,7 @@
#include "../builtins.h"
#include "../utils/mut.h"
#if SINGELI_X86_64
#if SINGELI_SIMD
#define SINGELI_FILE fold
#include "../utils/includeSingeli.h"
#endif
@ -92,7 +92,7 @@ B sum_c1(B t, B x) {
}
r += s;
} else {
#if SINGELI_X86_64
#if SINGELI_SIMD
r = simd_sum_f64(xv, ia);
#else
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
@ -133,7 +133,7 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
#if SINGELI_X86_64
#if SINGELI_SIMD
static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); }
static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); }
#else

View File

@ -149,10 +149,8 @@ def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
def packQ{...x} = assert{'packQ not supported', show{...x}}
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
def fold_max {...x} = assert{'fold_max not supported', show{...x}}
def fold_min {...x} = assert{'fold_min not supported', show{...x}}
def fold_add {...x} = assert{'fold_add not supported', show{...x}}
def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
def vfold{...x} = assert{'vfold not supported', show{...x}}
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
def pair{...x} = assert{'pair not supported', show{...x}}
def cvt{...x} = assert{'cvt not supported', show{...x}}

View File

@ -34,21 +34,22 @@ fn fold_idem{T==f64, op}(x:*T, len:u64) : T = {
def V = [bulk]T
xv:= *V ~~ x
r:V = [bulk]f64**0
if (len < bulk) {
assert{len > 0}
if (len<bulk) {
# Can't overlap like the long case
assert{len > 0}
r = load{xv}
if (bulk==4 and hasarch{'AVX'}) {
if (bulk>2) {
assert{(bulk==4) & hasarch{'AVX'}}
if (len > 1) {
if (len > 2) r = opsh64{op}{r, 4b2222}
r = opsh64{op}{r, 4b1111}
}
} else {
assert{bulk==2}
}
} else {
i:= load{*V ~~ (x+len-bulk)}
r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/bulk, i}}
r0:= reduce_pairwise{op, 2, xv, (len-1)/bulk, i}
if (hasarch{'AARCH64'}) return{vfold{op, r0}}
else r = mix{op, r0}
}
extract{r, 0}
}
@ -63,6 +64,7 @@ fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
e:= len / bulk
i:= load{xv, e} & (V~~maskOf{V, len % bulk})
r:= reduce_pairwise{op, 2, xv, e, i}
extract{mix{op, r}, 0}
if (hasarch{'AARCH64'}) vfold{op, r}
else extract{mix{op, r}, 0}
}
export{'simd_sum_f64', fold_assoc_0{f64,+}}

View File

@ -29,6 +29,9 @@ def mul1{a:T,b:T & w64i{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b}
def mul2{a:T,b:T & w128i{T}} = emit{ty_dbl{T}, ntyp0{'vmull_high', T}, a, b}
def mul12{a:T,b:T & w128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}}
def __or{a:T,b:T & nvecf{T}} = T~~ __or{ty_u{a}, ty_u{b}}
def __and{a:T,b:T & nvecf{T}} = T~~__and{ty_u{a}, ty_u{b}}
def __xor{a:T,b:T & nvecf{T}} = T~~__xor{ty_u{a}, ty_u{b}}
def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b}
def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b}
def __mul{a:T,b:T & nvec {T}} = emit{T, ntyp{'vmul', T}, a, b}
@ -69,10 +72,10 @@ def __ne{a:T,b:T & nvec{T}} = ~(a==b)
def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
def fold_addw{a:T & nveci{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a}
def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x}
def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x}
def fold_min {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def vfold{F, x:T & nvec{T} & ~nveci{T,64} & match{F, min}} = fold_min{x}
def vfold{F, x:T & nvec{T} & ~nveci{T,64} & match{F, max}} = fold_max{x}
def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x}
# TODO don't rely on regular stores being unaligned

View File

@ -73,9 +73,9 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
if (elwidth{T}==64) {
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
else fold_max{narrow{TE, x}}
else vfold{max, narrow{TE, x}}
} else {
fold_max{x}
vfold{max, x}
}
}

View File

@ -1,14 +1,6 @@
# Fold associative/commutative operation across a register
# Used by squeeze.singeli, count.singeli
# Has to be included after util/tup because of name conflict
def vfold{F, x:T} = {
show{'WARNING: using fallback fold for ', F, T}
def E = eltype{T}
r:E = 0
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
r
}
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}