From 897c46df76b7428d61b9edf630ad8a8d59493515 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 6 Apr 2023 20:09:04 +0300 Subject: [PATCH] aarch64 fold.singeli --- build/src/build.bqn | 2 +- src/builtins/fold.c | 6 +++--- src/singeli/src/base.singeli | 4 +--- src/singeli/src/fold.singeli | 16 +++++++++------- src/singeli/src/neon.singeli | 11 +++++++---- src/singeli/src/squeeze.singeli | 4 ++-- src/singeli/src/vecfold.singeli | 8 -------- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index 889cb44f..13e92fc8 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -599,7 +599,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xa."‿"src/builtins/search.c"‿"search", "x.."‿"src/builtins/fold.c"‿"fold", + "xa."‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 4026d695..c5af2770 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -15,7 +15,7 @@ #include "../builtins.h" #include "../utils/mut.h" -#if SINGELI_X86_64 +#if SINGELI_SIMD #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif @@ -92,7 +92,7 @@ B sum_c1(B t, B x) { } r += s; } else { - #if SINGELI_X86_64 + #if SINGELI_SIMD r = simd_sum_f64(xv, ia); #else r=0; for (usz i=0; i) } DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32) -#if SINGELI_X86_64 +#if SINGELI_SIMD static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); } static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); } #else diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index af264639..58f90119 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -149,10 +149,8 @@ def unpackHi{...x} = assert{'unpackHi not supported', show{...x}} def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} def packQ{...x} = assert{'packQ not supported', show{...x}} def __mulhi{...x} = assert{'__mulhi not supported', show{...x}} -def fold_max {...x} = assert{'fold_max not supported', show{...x}} -def fold_min {...x} = assert{'fold_min not supported', show{...x}} -def fold_add {...x} = assert{'fold_add not supported', show{...x}} def fold_addw{...x} = assert{'fold_addw not supported', show{...x}} +def vfold{...x} = assert{'vfold not supported', show{...x}} def narrowPair{...x} = assert{'narrowPair not supported', show{...x}} def pair{...x} = assert{'pair not supported', show{...x}} def cvt{...x} = assert{'cvt not supported', show{...x}} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index c1548020..79f70053 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -34,21 +34,22 @@ fn fold_idem{T==f64, op}(x:*T, len:u64) : T = { def V = [bulk]T xv:= *V ~~ x r:V = [bulk]f64**0 - if (len < bulk) { + assert{len > 0} + if (len 0} r = load{xv} - if (bulk==4 and hasarch{'AVX'}) { + if (bulk>2) { + assert{(bulk==4) & hasarch{'AVX'}} if (len > 1) { if (len > 2) r = opsh64{op}{r, 4b2222} r = opsh64{op}{r, 4b1111} } - } else { - assert{bulk==2} } } else { i:= load{*V ~~ (x+len-bulk)} - r = mix{op, reduce_pairwise{op, 2, xv, (len-1)/bulk, i}} + r0:= reduce_pairwise{op, 2, xv, (len-1)/bulk, i} + if (hasarch{'AARCH64'}) return{vfold{op, r0}} + else r = mix{op, r0} } extract{r, 0} } @@ -63,6 +64,7 @@ fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = { e:= len / bulk i:= load{xv, e} & (V~~maskOf{V, len % bulk}) r:= reduce_pairwise{op, 2, xv, e, i} - extract{mix{op, r}, 0} + if (hasarch{'AARCH64'}) vfold{op, r} + else extract{mix{op, r}, 0} } export{'simd_sum_f64', fold_assoc_0{f64,+}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index dd653ad4..a0b84018 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -29,6 +29,9 @@ def mul1{a:T,b:T & w64i{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b} def mul2{a:T,b:T & w128i{T}} = emit{ty_dbl{T}, ntyp0{'vmull_high', T}, a, b} def mul12{a:T,b:T & w128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}} +def __or{a:T,b:T & nvecf{T}} = T~~ __or{ty_u{a}, ty_u{b}} +def __and{a:T,b:T & nvecf{T}} = T~~__and{ty_u{a}, ty_u{b}} +def __xor{a:T,b:T & nvecf{T}} = T~~__xor{ty_u{a}, ty_u{b}} def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b} def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b} def __mul{a:T,b:T & nvec {T}} = emit{T, ntyp{'vmul', T}, a, b} @@ -69,10 +72,10 @@ def __ne{a:T,b:T & nvec{T}} = ~(a==b) def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a} def fold_addw{a:T & nveci{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} -def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a} -def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a} -def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x} -def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x} +def fold_min {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a} +def fold_max {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a} +def vfold{F, x:T & nvec{T} & ~nveci{T,64} & match{F, min}} = fold_min{x} +def vfold{F, x:T & nvec{T} & ~nveci{T,64} & match{F, max}} = fold_max{x} def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x} # TODO don't rely on regular stores being unaligned diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 35348be3..1d22a8e2 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -73,9 +73,9 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { def foldTotal{TE, x:T & hasarch{'AARCH64'}} = { if (elwidth{T}==64) { if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} - else fold_max{narrow{TE, x}} + else vfold{max, narrow{TE, x}} } else { - fold_max{x} + vfold{max, x} } } diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index 330122eb..f8b85b26 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -1,14 +1,6 @@ # Fold associative/commutative operation across a register # Used by squeeze.singeli, count.singeli -# Has to be included after util/tup because of name conflict -def vfold{F, x:T} = { - show{'WARNING: using fallback fold for ', F, T} - def E = eltype{T} - r:E = 0 - each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} - r -} def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { c:= x def EW = elwidth{T}