From afe67076547f1de99de4e6daae8c8ef53aa7a61b Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 22 Jan 2023 03:23:43 +0200 Subject: [PATCH] dyarith NEON support --- build/src/build.bqn | 2 +- src/builtins/arithd.c | 6 +++--- src/singeli/src/base.singeli | 9 +++++++-- src/singeli/src/cmp.singeli | 2 +- src/singeli/src/dyarith.singeli | 33 ++++++++++++++++++++++----------- src/singeli/src/neon.singeli | 12 +++++++++--- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index 7636d197..e1e71292 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -514,7 +514,7 @@ cachedBin‿linkerCache ← { ⟨"src/jit/", "nvm.c"⟩ ⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩ ⟩ - singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"; 𝕩} ⟨ + singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"; 𝕩} ⟨ "src/core/stuff.c"‿"equal", "src/utils/mut.c"‿"copy", "src/utils/bits.c"‿"bits" "src/builtins/arithd.c"‿"dyarith", "src/builtins/cmp.c"‿"cmp", "src/builtins/squeeze.c"‿"squeeze" "src/builtins/select.c"‿"select", "src/builtins/fold.c"‿"fold", "src/builtins/scan.c"‿"scan" diff --git a/src/builtins/arithd.c b/src/builtins/arithd.c index d20decd6..ccae09fd 100644 --- a/src/builtins/arithd.c +++ b/src/builtins/arithd.c @@ -21,9 +21,9 @@ B atan2_c2(B, B, B); typedef void (*AndBytesFn)(u8*, u8*, u64, u64); -#if SINGELI_X86_64 +#if SINGELI #include "../singeli/c/arithdDispatch.c" - static AndBytesFn andBytes_fn = avx2_andBytes; + static AndBytesFn andBytes_fn = simd_andBytes; #else static void base_andBytes(u8* r, u8* x, u64 repeatedMask, u64 numBytes) { u64* x64 = (u64*)x; usz i; @@ -138,7 +138,7 @@ typedef void (*AndBytesFn)(u8*, u8*, u64, u64); #undef GC2f - #if SINGELI_X86_64 + #if SINGELI #define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); } AA_DISPATCH(add) AA_DISPATCH(or) AA_DISPATCH(sub) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 3943f8ed..2585e89e 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -111,8 +111,13 @@ def narrow{T, x:X & T==eltype{X}} = x def undefPromote{T, x:X & T==X} = T~~x def andnz{...x} = assert{'andnz not supported', show{...x}} def topBlend{...x} = assert{'topBlend not supported', show{...x}} -def topMask{...x} = assert{'topBlend not supported', show{...x}} -def pack{...x} = assert{'pack not supported', show{...x}} +def topMask{...x} = assert{'topMask not supported', show{...x}} +def packLo{...x} = assert{'packLo not supported', show{...x}} +def packHi{...x} = assert{'packHi not supported', show{...x}} +def mul12{...x} = assert{'mul12 not supported', show{...x}} +def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} +def packQ{...x} = assert{'packQ not supported', show{...x}} +def __mulhi{...x} = assert{'__mulhi not supported', show{...x}} def min{a, b & knum{a} & knum{b}} = tern{ab, a, b} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 63f00cf5..f916c169 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = { r:u64 = 0 if (hasarch{'AARCH64'}) { def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}} - def step{j, n & n>=2} = pack{step{j, n/2}, step{j+n/2, n/2}} + def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}} r = promote{u64, homMask{step{0, unr}}} } else { @unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 53b1d434..d8c533f1 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -1,9 +1,13 @@ include './base' include './f64' include './cbqnDefs' -include './sse3' -include './avx' -include './avx2' +if (hasarch{'X86_64'}) { + include './sse3' + include './avx' + include './avx2' +} else if (hasarch{'AARCH64'}) { + include './neon' +} include './bitops' include './mask' @@ -24,8 +28,8 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - def arithChk1{F==__add, M, w:T, x:T, r:T} = topAny{M{(w^r) & (x^r)}} def arithChk1{F==__sub, M, w:T, x:T, r:T} = topAny{M{(w^x) & (w^r)}} -def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M} -def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M} +def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__adds{w,x}, r, M} +def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__subs{w,x}, r, M} @@ -35,7 +39,7 @@ def arithChk2{F, M, w:T, x:T & is_s{T} & (match{F,__add} | match{F,__sub})} = { } # × -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = { +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasarch{'X86_64'}} = { def wp = unpackQ{w, T ~~ (T**0 > w)} def xp = unpackQ{x, T ~~ (T**0 > x)} def rp = each{__mul, wp, xp} @@ -46,12 +50,12 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = { tup{packQ{rp}, homAny{tupsel{0,bad}|tupsel{1,bad}}} } } -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = { +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasarch{'X86_64'}} = { rl:= __mul {w,x} rh:= __mulhi{w,x} tup{rl, anyne{rh, rl>>15, M}} } -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = { max:= [8]f32 ~~ [8]u32**0x4efffffe def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} f32mul:= cf32{w} * cf32{x} @@ -68,6 +72,13 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, homAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M } +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = { + def r12 = mul12{w, x} + rl:= packLo{r12} + rh:= packHi{r12} + tup{rl, homAny{M{rh != (rl >> (elwidth{T}-1))}}} +} + def runner{u, R, F} = { @@ -147,7 +158,7 @@ arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = { } def arithAA{mode, F, W, X, R} = { - def vw = 256 + def vw = arch_defvw if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R} else arithAAu{vw, mode, F, W, X, R} } @@ -174,7 +185,7 @@ arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = { else len } -def arithSA{mode, F, swap, W, X, R} = arithSAf{256, mode, F, swap, W, X, R} +def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R} andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { assert{vw>=64} @@ -187,6 +198,6 @@ andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { }} } -'avx2_andBytes'=andBytes{256} +'simd_andBytes'=andBytes{arch_defvw} 'orSAc_f64_f64_f64'=arithSA{2,bqn_or,0,f64,f64,f64} include 'gen/arDefs' diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 11fd2a34..d5f7b4eb 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -28,6 +28,9 @@ def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a def __adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} def __subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} +def mul1{a:T,b:T & n64{T} & nveci{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b} +def mul2{a:T,b:T & n128{T} & nveci{T}} = emit{ty_dbl{T}, ntyp{'vmull_high', v_half{T}}, a, b} +def mul12{a:T,b:T & n128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}} def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b} def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b} @@ -39,8 +42,8 @@ def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b} def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b} def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b} def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b} -def min{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmin', T}, a, b} # TODO float - there are multiple options -def max{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmax', T}, a, b} +def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} +def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} def __shl{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a @@ -92,7 +95,10 @@ def half{x:T, n==0 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}} def half{x:T, n==1 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x} def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n} -def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} } +def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} } +def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} } +def packLo{x} = packLo{tupsel{0,x}, tupsel{1,x}} +def packHi{x} = packHi{tupsel{0,x}, tupsel{1,x}} def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, to_el{eltype{I},x}, i}}