dyarith NEON support

This commit is contained in:
dzaima 2023-01-22 03:23:43 +02:00
parent 20daceaae5
commit afe6707654
6 changed files with 43 additions and 21 deletions

View File

@ -514,7 +514,7 @@ cachedBin‿linkerCache ← {
"src/jit/", "nvm.c"
"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"
singeliMap {po.arch"aarch64"? 𝕩/˜(1¨𝕩)"cmp""bits""equal"; 𝕩}
singeliMap {po.arch"aarch64"? 𝕩/˜(1¨𝕩)"cmp""bits""equal""dyarith"; 𝕩}
"src/core/stuff.c""equal", "src/utils/mut.c""copy", "src/utils/bits.c""bits"
"src/builtins/arithd.c""dyarith", "src/builtins/cmp.c""cmp", "src/builtins/squeeze.c""squeeze"
"src/builtins/select.c""select", "src/builtins/fold.c""fold", "src/builtins/scan.c""scan"

View File

@ -21,9 +21,9 @@ B atan2_c2(B, B, B);
typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
#if SINGELI_X86_64
#if SINGELI
#include "../singeli/c/arithdDispatch.c"
static AndBytesFn andBytes_fn = avx2_andBytes;
static AndBytesFn andBytes_fn = simd_andBytes;
#else
static void base_andBytes(u8* r, u8* x, u64 repeatedMask, u64 numBytes) {
u64* x64 = (u64*)x; usz i;
@ -138,7 +138,7 @@ typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
#undef GC2f
#if SINGELI_X86_64
#if SINGELI
#define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); }
AA_DISPATCH(add) AA_DISPATCH(or)
AA_DISPATCH(sub)

View File

@ -111,8 +111,13 @@ def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def andnz{...x} = assert{'andnz not supported', show{...x}}
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
def topMask{...x} = assert{'topBlend not supported', show{...x}}
def pack{...x} = assert{'pack not supported', show{...x}}
def topMask{...x} = assert{'topMask not supported', show{...x}}
def packLo{...x} = assert{'packLo not supported', show{...x}}
def packHi{...x} = assert{'packHi not supported', show{...x}}
def mul12{...x} = assert{'mul12 not supported', show{...x}}
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
def packQ{...x} = assert{'packQ not supported', show{...x}}
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}

View File

@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
r:u64 = 0
if (hasarch{'AARCH64'}) {
def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}}
def step{j, n & n>=2} = pack{step{j, n/2}, step{j+n/2, n/2}}
def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}}
r = promote{u64, homMask{step{0, unr}}}
} else {
@unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})

View File

@ -1,9 +1,13 @@
include './base'
include './f64'
include './cbqnDefs'
include './sse3'
include './avx'
include './avx2'
if (hasarch{'X86_64'}) {
include './sse3'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './bitops'
include './mask'
@ -24,8 +28,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
# + & -
def arithChk1{F==__add, M, w:T, x:T, r:T} = topAny{M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T} = topAny{M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = anyne{__subs{w,x}, r, M}
@ -35,7 +39,7 @@ def arithChk2{F, M, w:T, x:T & is_s{T} & (match{F,__add} | match{F,__sub})} = {
}
# ×
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasarch{'X86_64'}} = {
def wp = unpackQ{w, T ~~ (T**0 > w)}
def xp = unpackQ{x, T ~~ (T**0 > x)}
def rp = each{__mul, wp, xp}
@ -46,12 +50,12 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
tup{packQ{rp}, homAny{tupsel{0,bad}|tupsel{1,bad}}}
}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasarch{'X86_64'}} = {
rl:= __mul {w,x}
rh:= __mulhi{w,x}
tup{rl, anyne{rh, rl>>15, M}}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
max:= [8]f32 ~~ [8]u32**0x4efffffe
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
f32mul:= cf32{w} * cf32{x}
@ -68,6 +72,13 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, homAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = {
def r12 = mul12{w, x}
rl:= packLo{r12}
rh:= packHi{r12}
tup{rl, homAny{M{rh != (rl >> (elwidth{T}-1))}}}
}
def runner{u, R, F} = {
@ -147,7 +158,7 @@ arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
}
def arithAA{mode, F, W, X, R} = {
def vw = 256
def vw = arch_defvw
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
else arithAAu{vw, mode, F, W, X, R}
}
@ -174,7 +185,7 @@ arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
else len
}
def arithSA{mode, F, swap, W, X, R} = arithSAf{256, mode, F, swap, W, X, R}
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R}
andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
assert{vw>=64}
@ -187,6 +198,6 @@ andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
}}
}
'avx2_andBytes'=andBytes{256}
'simd_andBytes'=andBytes{arch_defvw}
'orSAc_f64_f64_f64'=arithSA{2,bqn_or,0,f64,f64,f64}
include 'gen/arDefs'

View File

@ -28,6 +28,9 @@ def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a
def __adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b}
def __subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b}
def mul1{a:T,b:T & n64{T} & nveci{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b}
def mul2{a:T,b:T & n128{T} & nveci{T}} = emit{ty_dbl{T}, ntyp{'vmull_high', v_half{T}}, a, b}
def mul12{a:T,b:T & n128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}}
def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b}
def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b}
@ -39,8 +42,8 @@ def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b}
def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b}
def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b}
def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
def min{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmin', T}, a, b} # TODO float - there are multiple options
def max{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmax', T}, a, b}
def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b}
def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b}
def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
def __shl{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a
@ -92,7 +95,10 @@ def half{x:T, n==0 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}}
def half{x:T, n==1 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x}
def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n}
def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
def packLo{x} = packLo{tupsel{0,x}, tupsel{1,x}}
def packHi{x} = packHi{tupsel{0,x}, tupsel{1,x}}
def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, to_el{eltype{I},x}, i}}