dyarith NEON support
This commit is contained in:
parent
20daceaae5
commit
afe6707654
@ -514,7 +514,7 @@ cachedBin‿linkerCache ← {
|
||||
⟨"src/jit/", "nvm.c"⟩
|
||||
⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩
|
||||
⟩
|
||||
singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"; 𝕩} ⟨
|
||||
singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"; 𝕩} ⟨
|
||||
"src/core/stuff.c"‿"equal", "src/utils/mut.c"‿"copy", "src/utils/bits.c"‿"bits"
|
||||
"src/builtins/arithd.c"‿"dyarith", "src/builtins/cmp.c"‿"cmp", "src/builtins/squeeze.c"‿"squeeze"
|
||||
"src/builtins/select.c"‿"select", "src/builtins/fold.c"‿"fold", "src/builtins/scan.c"‿"scan"
|
||||
|
||||
@ -21,9 +21,9 @@ B atan2_c2(B, B, B);
|
||||
|
||||
typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
#include "../singeli/c/arithdDispatch.c"
|
||||
static AndBytesFn andBytes_fn = avx2_andBytes;
|
||||
static AndBytesFn andBytes_fn = simd_andBytes;
|
||||
#else
|
||||
static void base_andBytes(u8* r, u8* x, u64 repeatedMask, u64 numBytes) {
|
||||
u64* x64 = (u64*)x; usz i;
|
||||
@ -138,7 +138,7 @@ typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
|
||||
#undef GC2f
|
||||
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
#define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); }
|
||||
AA_DISPATCH(add) AA_DISPATCH(or)
|
||||
AA_DISPATCH(sub)
|
||||
|
||||
@ -111,8 +111,13 @@ def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def andnz{...x} = assert{'andnz not supported', show{...x}}
|
||||
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
|
||||
def topMask{...x} = assert{'topBlend not supported', show{...x}}
|
||||
def pack{...x} = assert{'pack not supported', show{...x}}
|
||||
def topMask{...x} = assert{'topMask not supported', show{...x}}
|
||||
def packLo{...x} = assert{'packLo not supported', show{...x}}
|
||||
def packHi{...x} = assert{'packHi not supported', show{...x}}
|
||||
def mul12{...x} = assert{'mul12 not supported', show{...x}}
|
||||
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
|
||||
def packQ{...x} = assert{'packQ not supported', show{...x}}
|
||||
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
|
||||
|
||||
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
|
||||
|
||||
@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
|
||||
r:u64 = 0
|
||||
if (hasarch{'AARCH64'}) {
|
||||
def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}}
|
||||
def step{j, n & n>=2} = pack{step{j, n/2}, step{j+n/2, n/2}}
|
||||
def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}}
|
||||
r = promote{u64, homMask{step{0, unr}}}
|
||||
} else {
|
||||
@unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './bitops'
|
||||
include './mask'
|
||||
|
||||
@ -24,8 +28,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
|
||||
# + & -
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = topAny{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = topAny{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__subs{w,x}, r, M}
|
||||
|
||||
|
||||
|
||||
@ -35,7 +39,7 @@ def arithChk2{F, M, w:T, x:T & is_s{T} & (match{F,__add} | match{F,__sub})} = {
|
||||
}
|
||||
|
||||
# ×
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasarch{'X86_64'}} = {
|
||||
def wp = unpackQ{w, T ~~ (T**0 > w)}
|
||||
def xp = unpackQ{x, T ~~ (T**0 > x)}
|
||||
def rp = each{__mul, wp, xp}
|
||||
@ -46,12 +50,12 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
|
||||
tup{packQ{rp}, homAny{tupsel{0,bad}|tupsel{1,bad}}}
|
||||
}
|
||||
}
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasarch{'X86_64'}} = {
|
||||
rl:= __mul {w,x}
|
||||
rh:= __mulhi{w,x}
|
||||
tup{rl, anyne{rh, rl>>15, M}}
|
||||
}
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
|
||||
max:= [8]f32 ~~ [8]u32**0x4efffffe
|
||||
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
@ -68,6 +72,13 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, homAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
}
|
||||
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = {
|
||||
def r12 = mul12{w, x}
|
||||
rl:= packLo{r12}
|
||||
rh:= packHi{r12}
|
||||
tup{rl, homAny{M{rh != (rl >> (elwidth{T}-1))}}}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def runner{u, R, F} = {
|
||||
@ -147,7 +158,7 @@ arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
|
||||
}
|
||||
|
||||
def arithAA{mode, F, W, X, R} = {
|
||||
def vw = 256
|
||||
def vw = arch_defvw
|
||||
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
|
||||
else arithAAu{vw, mode, F, W, X, R}
|
||||
}
|
||||
@ -174,7 +185,7 @@ arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
|
||||
else len
|
||||
}
|
||||
|
||||
def arithSA{mode, F, swap, W, X, R} = arithSAf{256, mode, F, swap, W, X, R}
|
||||
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R}
|
||||
|
||||
andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
assert{vw>=64}
|
||||
@ -187,6 +198,6 @@ andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
}}
|
||||
}
|
||||
|
||||
'avx2_andBytes'=andBytes{256}
|
||||
'simd_andBytes'=andBytes{arch_defvw}
|
||||
'orSAc_f64_f64_f64'=arithSA{2,bqn_or,0,f64,f64,f64}
|
||||
include 'gen/arDefs'
|
||||
|
||||
@ -28,6 +28,9 @@ def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a
|
||||
|
||||
def __adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b}
|
||||
def __subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b}
|
||||
def mul1{a:T,b:T & n64{T} & nveci{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b}
|
||||
def mul2{a:T,b:T & n128{T} & nveci{T}} = emit{ty_dbl{T}, ntyp{'vmull_high', v_half{T}}, a, b}
|
||||
def mul12{a:T,b:T & n128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}}
|
||||
|
||||
def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b}
|
||||
def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b}
|
||||
@ -39,8 +42,8 @@ def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b}
|
||||
def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b}
|
||||
def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b}
|
||||
def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
|
||||
def min{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmin', T}, a, b} # TODO float - there are multiple options
|
||||
def max{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmax', T}, a, b}
|
||||
def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b}
|
||||
def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b}
|
||||
def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
|
||||
|
||||
def __shl{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a
|
||||
@ -92,7 +95,10 @@ def half{x:T, n==0 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}}
|
||||
def half{x:T, n==1 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x}
|
||||
def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n}
|
||||
|
||||
def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
||||
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
||||
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
|
||||
def packLo{x} = packLo{tupsel{0,x}, tupsel{1,x}}
|
||||
def packHi{x} = packHi{tupsel{0,x}, tupsel{1,x}}
|
||||
def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, to_el{eltype{I},x}, i}}
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user