new Singeli syntax
This commit is contained in:
parent
8194f51b30
commit
1178e5df56
@ -1 +1 @@
|
||||
Subproject commit e6d3b11e59eea7614307a37b27e7f10f4d3d04b8
|
||||
Subproject commit 2b4e3b8cdf6f3a5f8ddf383a8421f012550b8c27
|
||||
@ -6,6 +6,8 @@ oper ~~ reinterpret infix right 55
|
||||
oper ** broadcast infix right 55
|
||||
def Size = u64
|
||||
def load{x} = *x
|
||||
def exportN{f, ...ns} = each{{n} => export{n, f}, ns}
|
||||
def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} }
|
||||
|
||||
def elwidth{T} = width{eltype{T}}
|
||||
|
||||
|
||||
@ -24,10 +24,10 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
|
||||
}}
|
||||
}
|
||||
|
||||
bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
||||
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
||||
bitsel{VL, T, *T~~r, bits, trunc{T,e0}, trunc{T,e1}, len}
|
||||
}
|
||||
|
||||
def table{w} = each{{T} => bitsel_i{w, T}, tup{u8, u16, u32, u64}}
|
||||
|
||||
def tup_bitsel = table{arch_defvw}; v_bitsel:*type{tupsel{0,tup_bitsel}} = tup_bitsel; 'simd_bitsel'=v_bitsel
|
||||
exportT{'simd_bitsel', table{arch_defvw}}
|
||||
@ -15,7 +15,7 @@ def fillbits{dst:*u64, len:(Size), v } = { emit{void, 'fillBits', dst, len,
|
||||
def fillbits{dst:*u64, len:(Size), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
|
||||
def cmp_err{x} = { emit{void, 'cmp_err'}; return{}; }
|
||||
|
||||
cmpIX(dst:*u64, len:Size, x:u64, v:u1) : void = {
|
||||
fn cmpIX(dst:*u64, len:Size, x:u64, v:u1) : void = {
|
||||
nan:u1 = q_f64{x}
|
||||
if (~(nan | q_chr{x})) cmp_err{x}
|
||||
fillbits{dst, len, v & ~nan, x}
|
||||
@ -87,26 +87,26 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
|
||||
ri+= 1
|
||||
}
|
||||
}
|
||||
aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
|
||||
fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
|
||||
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
|
||||
xv:= *VT~~xr; xs:= *eltype{VT}~~xr
|
||||
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>load{xs,i}, {i}=>load{xv,i}, dst, len}
|
||||
}
|
||||
|
||||
as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:Size) : void = {
|
||||
fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:Size) : void = {
|
||||
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
|
||||
xv:= VT**pathAS{dst, len, eltype{VT}, op, x}
|
||||
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>x, {i}=>xv, dst, len}
|
||||
}
|
||||
|
||||
bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
|
||||
fn bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
|
||||
@forNZ (dst, w in *u64~~wr, x in *u64~~xr over _ to cdiv{len,64}) dst = bitop{w,x}
|
||||
}
|
||||
|
||||
not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
|
||||
cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
|
||||
fn not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
|
||||
fn cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
|
||||
|
||||
bitAS{op}(dst:*u64, wr:*void, x:u64, len:Size) : void = { # show{'bitAS'}
|
||||
fn bitAS{op}(dst:*u64, wr:*void, x:u64, len:Size) : void = { # show{'bitAS'}
|
||||
xf:f64 = interp_f64{x}
|
||||
r0:u1 = op{0,xf}
|
||||
r1:u1 = op{1,xf}
|
||||
@ -135,14 +135,10 @@ def table{aa, F, w, G_bit, G_vec} = {
|
||||
def tableAA{w, F, F_bit} = table{1, F, w, {} => bitAA{F_bit}, {V, u} => aa2bit{V, u, F}}
|
||||
def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}}
|
||||
|
||||
def tup_eqAA = tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}; v_eqAA:*type{tupsel{0,tup_eqAA}} = tup_eqAA; 'simd_eqAA'=v_eqAA
|
||||
def tup_neAA = tableAA{arch_defvw, __ne, {a,b}=>a ^ b}; v_neAA:*type{tupsel{0,tup_neAA}} = tup_neAA; 'simd_neAA'=v_neAA
|
||||
def tup_gtAA = tableAA{arch_defvw, __gt, {a,b}=>a & ~b}; v_gtAA:*type{tupsel{0,tup_gtAA}} = tup_gtAA; 'simd_gtAA'=v_gtAA
|
||||
def tup_geAA = tableAA{arch_defvw, __ge, {a,b}=>a | ~b}; v_geAA:*type{tupsel{0,tup_geAA}} = tup_geAA; 'simd_geAA'=v_geAA
|
||||
|
||||
def tup_eqAS = tableAS{arch_defvw, __eq}; v_eqAS:*type{tupsel{0,tup_eqAS}} = tup_eqAS; 'simd_eqAS'=v_eqAS
|
||||
def tup_neAS = tableAS{arch_defvw, __ne}; v_neAS:*type{tupsel{0,tup_neAS}} = tup_neAS; 'simd_neAS'=v_neAS
|
||||
def tup_gtAS = tableAS{arch_defvw, __gt}; v_gtAS:*type{tupsel{0,tup_gtAS}} = tup_gtAS; 'simd_gtAS'=v_gtAS
|
||||
def tup_geAS = tableAS{arch_defvw, __ge}; v_geAS:*type{tupsel{0,tup_geAS}} = tup_geAS; 'simd_geAS'=v_geAS
|
||||
def tup_ltAS = tableAS{arch_defvw, __lt}; v_ltAS:*type{tupsel{0,tup_ltAS}} = tup_ltAS; 'simd_ltAS'=v_ltAS
|
||||
def tup_leAS = tableAS{arch_defvw, __le}; v_leAS:*type{tupsel{0,tup_leAS}} = tup_leAS; 'simd_leAS'=v_leAS
|
||||
exportT{'simd_eqAS', tableAS{arch_defvw, __eq}}; exportT{'simd_eqAA', tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}}
|
||||
exportT{'simd_neAS', tableAS{arch_defvw, __ne}}; exportT{'simd_neAA', tableAA{arch_defvw, __ne, {a,b}=>a ^ b}}
|
||||
exportT{'simd_gtAS', tableAS{arch_defvw, __gt}}; exportT{'simd_gtAA', tableAA{arch_defvw, __gt, {a,b}=>a & ~b}}
|
||||
exportT{'simd_geAS', tableAS{arch_defvw, __ge}}; exportT{'simd_geAA', tableAA{arch_defvw, __ge, {a,b}=>a | ~b}}
|
||||
exportT{'simd_ltAS', tableAS{arch_defvw, __lt}}
|
||||
exportT{'simd_leAS', tableAS{arch_defvw, __le}}
|
||||
|
||||
@ -86,7 +86,7 @@ def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get
|
||||
def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially
|
||||
rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}}
|
||||
rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}}
|
||||
rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
fn rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
def V = [32]i8
|
||||
def st = read_shuf_vecs{wv, elbytes, sh}
|
||||
rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
|
||||
@ -95,7 +95,7 @@ rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
|
||||
rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}}
|
||||
rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
|
||||
rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
fn rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
def h = 4
|
||||
def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}}
|
||||
def V = [32]i8
|
||||
@ -125,7 +125,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
|
||||
}
|
||||
|
||||
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
if (wv > tupsel{-1,rcsh_vals}) {
|
||||
return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)}
|
||||
}
|
||||
@ -146,10 +146,10 @@ def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = {
|
||||
store{*V~~r, -1, v}
|
||||
}
|
||||
}
|
||||
rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n}
|
||||
rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n}
|
||||
fn rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n}
|
||||
fn rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n}
|
||||
|
||||
rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
assert{wv>=2}
|
||||
if (wv>=8 and wv<=fact_size) {
|
||||
fa := promote{u64, load{factors,wv-8}}
|
||||
@ -180,5 +180,5 @@ rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
'constrep_u8' = rep_const{i8 }; 'constrep_u16' = rep_const{i16}
|
||||
'constrep_u32' = rep_const{i32}; 'constrep_u64' = rep_const{u64}
|
||||
export{'constrep_u8', rep_const{i8 }}; export{'constrep_u16', rep_const{i16}}
|
||||
export{'constrep_u32', rep_const{i32}}; export{'constrep_u64', rep_const{u64}}
|
||||
|
||||
@ -23,7 +23,7 @@ def copyFromBits{T, loadFn, rp, l:u64} = {
|
||||
}
|
||||
|
||||
|
||||
copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
def vw = arch_defvw
|
||||
assert{l!=0}
|
||||
|
||||
@ -75,7 +75,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
fn copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
assert{l!=0}
|
||||
x0:= (*u8~~xRaw) + cbqn_tyArrOffset{}
|
||||
xs:= u64~~((*u8~~x) - x0)
|
||||
@ -91,60 +91,26 @@ copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
# }
|
||||
}
|
||||
|
||||
# simd_copy_src_dst
|
||||
# x→int & equal-width copies
|
||||
'simd_copy_1_1' = copy{u1, u1}
|
||||
'simd_copy_1_i8' = copy{u1, i8}
|
||||
'simd_copy_1_i16' = copy{u1, i16}
|
||||
'simd_copy_1_i32' = copy{u1, i32}
|
||||
|
||||
'simd_copy_i8_1' = copy{i8, u1}
|
||||
'simd_copy_i8_i8', 'simd_copy_c8_c8' = copy{i8, i8}
|
||||
'simd_copy_i8_i16' = copy{i8, i16}
|
||||
'simd_copy_i8_i32' = copy{i8, i32}
|
||||
def gen{p} = {
|
||||
def ts = tup{u1, i8, i16, i32, f64, u8, u16, u32, u64}
|
||||
def tn = tup{'1','i8','i16','i32','f64','c8','c16','c32','B'}
|
||||
def tm = tup{0, 0, 0, 0, 0, 1, 1, 1, 2}
|
||||
each{{tx0,nx,mx} => {
|
||||
each{{tr0,nr,mr} => {
|
||||
if ((mx==mr or mx==2 or mr==2) and (if (mx==2) mr==1; else 1)) {
|
||||
def tr = if (mx==0 and mr==2) f64; else if (tx0==tr0 and mx==1) ty_s{tx0}; else tr0
|
||||
def tx = if (mr==0 and mx==2) f64; else if (tx0==tr0 and mx==1) ty_s{tx0}; else tx0
|
||||
export{merge{p, nx, '_', nr}, copy{tx, tr}}
|
||||
}
|
||||
}, ts, tn, tm}
|
||||
}, ts, tn, tm}
|
||||
}
|
||||
|
||||
'simd_copy_i16_1' = copy{i16, u1}
|
||||
'simd_copy_i16_i8' = copy{i16, i8}
|
||||
'simd_copy_i16_i16', 'simd_copy_c16_c16' = copy{i16, i16}
|
||||
'simd_copy_i16_i32' = copy{i16, i32}
|
||||
|
||||
'simd_copy_i32_1' = copy{i32, u1}
|
||||
'simd_copy_i32_i8' = copy{i32, i8}
|
||||
'simd_copy_i32_i16' = copy{i32, i16}
|
||||
'simd_copy_i32_i32', 'simd_copy_c32_c32'= copy{i32, i32}
|
||||
|
||||
'simd_copy_f64_1' = copy{f64, u1}
|
||||
'simd_copy_f64_i8' = copy{f64, i8}
|
||||
'simd_copy_f64_i16' = copy{f64, i16}
|
||||
'simd_copy_f64_i32' = copy{f64, i32}
|
||||
|
||||
# x→f64, x→B (no simd_copy_B_B because that may possibly need refcounting)
|
||||
'simd_copy_1_f64', 'simd_copy_1_B' = copy{u1, f64}
|
||||
'simd_copy_i8_f64', 'simd_copy_i8_B' = copy{i8, f64}
|
||||
'simd_copy_i16_f64', 'simd_copy_i16_B' = copy{i16, f64}
|
||||
'simd_copy_i32_f64', 'simd_copy_i32_B' = copy{i32, f64}
|
||||
'simd_copy_f64_f64', 'simd_copy_f64_B' = copy{f64, f64}
|
||||
|
||||
# chr→x
|
||||
'simd_copy_c8_c16' = copy{u8, u16}
|
||||
'simd_copy_c8_c32' = copy{u8, u32}
|
||||
'simd_copy_c8_B' = copy{u8, u64}
|
||||
|
||||
'simd_copy_c16_c8' = copy{u16, u8}
|
||||
'simd_copy_c16_c32' = copy{u16, u32}
|
||||
'simd_copy_c16_B' = copy{u16, u64}
|
||||
|
||||
'simd_copy_c32_c8' = copy{u32, u8}
|
||||
'simd_copy_c32_c16' = copy{u32, u16}
|
||||
'simd_copy_c32_B' = copy{u32, u64}
|
||||
|
||||
# B→chr
|
||||
'simd_copy_B_c8' = copy{u64, u8}
|
||||
'simd_copy_B_c16' = copy{u64, u16}
|
||||
'simd_copy_B_c32' = copy{u64, u32}
|
||||
gen{'simd_copy_'}
|
||||
|
||||
# unaligned bitarr widening
|
||||
'simd_copy_1u_i8' = copy_ubit{i8}
|
||||
'simd_copy_1u_i16' = copy_ubit{i16}
|
||||
'simd_copy_1u_i32' = copy_ubit{i32}
|
||||
'simd_copy_1u_f64' = copy_ubit{f64}
|
||||
export{'simd_copy_1u_i8', copy_ubit{i8}}
|
||||
export{'simd_copy_1u_i16', copy_ubit{i16}}
|
||||
export{'simd_copy_1u_i32', copy_ubit{i32}}
|
||||
export{'simd_copy_1u_f64', copy_ubit{f64}}
|
||||
@ -173,12 +173,12 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
|
||||
}
|
||||
}
|
||||
|
||||
arithAAc{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : u64 = {
|
||||
fn arithAAc{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : u64 = {
|
||||
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
|
||||
if (mode==1) len
|
||||
else 0
|
||||
}
|
||||
arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
|
||||
fn arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
|
||||
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
|
||||
}
|
||||
|
||||
@ -189,7 +189,7 @@ def arithAA{mode, F, W, X, R} = {
|
||||
}
|
||||
|
||||
# mode: 0:overflow-checked, needed; 1:overflow-erroring; 2: overflow-checked, not needed
|
||||
arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
|
||||
fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
|
||||
# show{F, swap, mode, W, X, R}
|
||||
assert{len>0}
|
||||
def bulk = vw / max{width{W}, width{R}}
|
||||
@ -214,7 +214,7 @@ arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
|
||||
|
||||
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R}
|
||||
|
||||
andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
assert{vw>=64}
|
||||
def bulk = vw / width{u8}
|
||||
def T8 = [bulk]u8
|
||||
@ -225,6 +225,6 @@ andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
}}
|
||||
}
|
||||
|
||||
'simd_andBytes'=andBytes{arch_defvw}
|
||||
'orSAc_f64_f64_f64'=arithSA{2,bqn_or,0,f64,f64,f64}
|
||||
export{'simd_andBytes', andBytes{arch_defvw}}
|
||||
export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}}
|
||||
include 'gen/arDefs'
|
||||
|
||||
@ -18,7 +18,7 @@ def swap{w,x} = {
|
||||
}
|
||||
|
||||
# width{W} ≤ width{X}
|
||||
equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
def vw = arch_defvw
|
||||
def bulk = vw / width{X}
|
||||
if (W!=X) if (d!=0) swap{w,x}
|
||||
@ -77,23 +77,23 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
1
|
||||
}
|
||||
|
||||
'avx2_equal_1_1' = equal{u1, u1}
|
||||
'avx2_equal_1_8' = equal{u1, u8}
|
||||
'avx2_equal_1_16' = equal{u1, u16}
|
||||
'avx2_equal_1_32' = equal{u1, u32}
|
||||
'avx2_equal_1_f64' = equal{u1, f64}
|
||||
export{'avx2_equal_1_1', equal{u1, u1}}
|
||||
export{'avx2_equal_1_8', equal{u1, u8}}
|
||||
export{'avx2_equal_1_16', equal{u1, u16}}
|
||||
export{'avx2_equal_1_32', equal{u1, u32}}
|
||||
export{'avx2_equal_1_f64', equal{u1, f64}}
|
||||
|
||||
'avx2_equal_8_8' = equal{i8, i8}
|
||||
export{'avx2_equal_8_8', equal{i8, i8}}
|
||||
|
||||
'avx2_equal_s8_16' = equal{i8, i16}
|
||||
'avx2_equal_s8_32' = equal{i8, i32}
|
||||
'avx2_equal_s16_32' = equal{i16, i32}
|
||||
export{'avx2_equal_s8_16', equal{i8, i16}}
|
||||
export{'avx2_equal_s8_32', equal{i8, i32}}
|
||||
export{'avx2_equal_s16_32', equal{i16, i32}}
|
||||
|
||||
'avx2_equal_s8_f64' = equal{i8, f64}
|
||||
'avx2_equal_s16_f64'= equal{i16, f64}
|
||||
'avx2_equal_s32_f64'= equal{i32, f64}
|
||||
'avx2_equal_f64_f64'= equal{f64, f64}
|
||||
export{'avx2_equal_s8_f64', equal{i8, f64}}
|
||||
export{'avx2_equal_s16_f64', equal{i16, f64}}
|
||||
export{'avx2_equal_s32_f64', equal{i32, f64}}
|
||||
export{'avx2_equal_f64_f64', equal{f64, f64}}
|
||||
|
||||
'avx2_equal_u8_16' = equal{u8, u16}
|
||||
'avx2_equal_u8_32' = equal{u8, u32}
|
||||
'avx2_equal_u16_32' = equal{u16, u32}
|
||||
export{'avx2_equal_u8_16', equal{u8, u16}}
|
||||
export{'avx2_equal_u8_32', equal{u8, u32}}
|
||||
export{'avx2_equal_u16_32', equal{u16, u32}}
|
||||
|
||||
@ -21,7 +21,7 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = {
|
||||
r
|
||||
}
|
||||
|
||||
fold_idem{T==f64, op}(x:*T, len:u64) : T = {
|
||||
fn fold_idem{T==f64, op}(x:*T, len:u64) : T = {
|
||||
def step = 256/width{T}
|
||||
def V = [step]T
|
||||
xv:= *V ~~ x
|
||||
@ -41,10 +41,10 @@ fold_idem{T==f64, op}(x:*T, len:u64) : T = {
|
||||
extract{r, 0}
|
||||
}
|
||||
|
||||
'avx2_fold_min_f64' = fold_idem{f64,min}
|
||||
'avx2_fold_max_f64' = fold_idem{f64,max}
|
||||
export{'avx2_fold_min_f64', fold_idem{f64,min}}
|
||||
export{'avx2_fold_max_f64', fold_idem{f64,max}}
|
||||
|
||||
fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
|
||||
fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
|
||||
def step = 256/width{T}
|
||||
def V = [step]T
|
||||
xv:= *V ~~ x
|
||||
@ -53,4 +53,4 @@ fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
|
||||
r:= reduce_pairwise{op, 2, xv, e, i}
|
||||
extract{mix{op, r}, 0}
|
||||
}
|
||||
'avx2_sum_f64' = fold_assoc_0{f64,+}
|
||||
export{'avx2_sum_f64', fold_assoc_0{f64,+}}
|
||||
|
||||
@ -163,7 +163,7 @@ Make ← {𝕊ns:
|
||||
tr≢@?
|
||||
siName ← ∾⟨id, argarr, 𝕨⊑"?ce", ∾'_'⊸∾¨ tw‿tx‿tr⟩
|
||||
𝕨 { 𝕊:
|
||||
siSrc∾↩ ∾⟨"'", siName, "'=arithSA{", '0'+ 𝕨⊑⟨@, onlyFirst⊑0‿2, 1⟩, ',', si, ',', '0'+swap, ∾(',' ∾ 'u'⌾⊑⍟('c'≡⊑))¨ tw‿tx‿tr, '}', nl⟩
|
||||
siSrc∾↩ ∾⟨"export{'", siName, "', arithSA{", '0'+ 𝕨⊑⟨@, onlyFirst⊑0‿2, 1⟩, ',', si, ',', '0'+swap, ∾(',' ∾ 'u'⌾⊑⍟('c'≡⊑))¨ tw‿tx‿tr, "}}", nl⟩
|
||||
singeliASFns∾↩ <siName
|
||||
}⍟⊢ ¬⊑(<siName)∊singeliASFns
|
||||
siName;
|
||||
@ -242,7 +242,7 @@ Make ← {𝕊ns:
|
||||
|
||||
GenSingeli ← { 𝕊name‿w‿x‿r‿tm‿si:
|
||||
si≢@?
|
||||
siSrc ∾↩ ∾⟨"'", name, "'=arithAA{", '0'+tm, ',', si, ',', w⊑tySiName, ',', x⊑tySiName, ',', r⊑tySiName, '}', nl⟩;
|
||||
siSrc ∾↩ ∾⟨"export{'", name, "', arithAA{", '0'+tm, ',', si, ',', w⊑tySiName, ',', x⊑tySiName, ',', r⊑tySiName, "}}", nl⟩;
|
||||
# else
|
||||
@
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@ include './bitops'
|
||||
include './mask'
|
||||
|
||||
|
||||
absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
def bulk = arch_defvw/width{T}
|
||||
def VT = [bulk]T
|
||||
muLoop{bulk, tern{T==f64, 2, 1}, len, {is, M} => {
|
||||
@ -23,6 +23,4 @@ absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
len
|
||||
}
|
||||
|
||||
def tup_abs = each{{T}=>absFn{T}, tup{i8, i16, i32, f64}}
|
||||
t_abs:*type{tupsel{0,tup_abs}} = tup_abs
|
||||
'simd_abs' = t_abs
|
||||
exportT{'simd_abs', each{{T}=>absFn{T}, tup{i8, i16, i32, f64}}}
|
||||
@ -4,7 +4,7 @@ include './sse3'
|
||||
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
|
||||
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}
|
||||
|
||||
clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||
fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||
def V = [2]u64
|
||||
m := V**mark
|
||||
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
|
||||
@ -25,7 +25,7 @@ clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
|
||||
}
|
||||
}
|
||||
clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = {
|
||||
fn clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = {
|
||||
clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1))
|
||||
}
|
||||
'clmul_scan_ne' = clmul_scan_ne_bit
|
||||
export{'clmul_scan_ne', clmul_scan_ne_bit}
|
||||
|
||||
@ -39,7 +39,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||
}
|
||||
|
||||
# Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈
|
||||
avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
fn avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
# Within each lane, scan using shifts by powers of 2. First k elements
|
||||
# when shifting by k don't need to change, so leave them alone.
|
||||
def w = width{T}
|
||||
@ -55,26 +55,26 @@ avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
|
||||
scan_post{T, init, x, r, len, op, pre}
|
||||
}
|
||||
avx2_scan_idem{T==f64, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
fn avx2_scan_idem{T==f64, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
def sh{s, a} = op{a, shuf{[4]u64, a, s}}
|
||||
scan_post{T, init, x, r, len, op, {a}=>sh{4b1110,sh{4b2200,a}}}
|
||||
}
|
||||
'avx2_scan_min_init_i8' = avx2_scan_idem{i8 , min}; 'avx2_scan_max_init_i8' = avx2_scan_idem{i8 , max}
|
||||
'avx2_scan_min_init_i16' = avx2_scan_idem{i16, min}; 'avx2_scan_max_init_i16' = avx2_scan_idem{i16, max}
|
||||
'avx2_scan_min_init_i32' = avx2_scan_idem{i32, min}; 'avx2_scan_max_init_i32' = avx2_scan_idem{i32, max}
|
||||
'avx2_scan_min_init_f64' = avx2_scan_idem{f64, min}; 'avx2_scan_max_init_f64' = avx2_scan_idem{f64, max}
|
||||
export{'avx2_scan_min_init_i8', avx2_scan_idem{i8 , min}}; export{'avx2_scan_max_init_i8', avx2_scan_idem{i8 , max}}
|
||||
export{'avx2_scan_min_init_i16', avx2_scan_idem{i16, min}}; export{'avx2_scan_max_init_i16', avx2_scan_idem{i16, max}}
|
||||
export{'avx2_scan_min_init_i32', avx2_scan_idem{i32, min}}; export{'avx2_scan_max_init_i32', avx2_scan_idem{i32, max}}
|
||||
export{'avx2_scan_min_init_f64', avx2_scan_idem{f64, min}}; export{'avx2_scan_max_init_f64', avx2_scan_idem{f64, max}}
|
||||
|
||||
avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
|
||||
fn avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
|
||||
def m = 1 << (width{T}-1)
|
||||
def id = (if (match{op,min}) m-1; else -m)
|
||||
avx2_scan_idem{T, op}(x, r, len, id)
|
||||
}
|
||||
'avx2_scan_min_i8' = avx2_scan_idem_id{i8 , min}; 'avx2_scan_max_i8' = avx2_scan_idem_id{i8 , max}
|
||||
'avx2_scan_min_i16' = avx2_scan_idem_id{i16, min}; 'avx2_scan_max_i16' = avx2_scan_idem_id{i16, max}
|
||||
'avx2_scan_min_i32' = avx2_scan_idem_id{i32, min}; 'avx2_scan_max_i32' = avx2_scan_idem_id{i32, max}
|
||||
export{'avx2_scan_min_i8', avx2_scan_idem_id{i8 , min}}; export{'avx2_scan_max_i8', avx2_scan_idem_id{i8 , max}}
|
||||
export{'avx2_scan_min_i16', avx2_scan_idem_id{i16, min}}; export{'avx2_scan_max_i16', avx2_scan_idem_id{i16, max}}
|
||||
export{'avx2_scan_min_i32', avx2_scan_idem_id{i32, min}}; export{'avx2_scan_max_i32', avx2_scan_idem_id{i32, max}}
|
||||
|
||||
# Associative scan
|
||||
avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
fn avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
# Prefix op on entire AVX register
|
||||
def pre{a} = {
|
||||
# Within each lane, scan using shifts by powers of 2.
|
||||
@ -88,12 +88,12 @@ avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
}
|
||||
scan_post{T, init, x, r, len, op, pre}
|
||||
}
|
||||
'avx2_scan_pluswrap_u8' = avx2_scan_assoc_0{u8 , +}
|
||||
'avx2_scan_pluswrap_u16' = avx2_scan_assoc_0{u16, +}
|
||||
'avx2_scan_pluswrap_u32' = avx2_scan_assoc_0{u32, +}
|
||||
export{'avx2_scan_pluswrap_u8', avx2_scan_assoc_0{u8 , +}}
|
||||
export{'avx2_scan_pluswrap_u16', avx2_scan_assoc_0{u16, +}}
|
||||
export{'avx2_scan_pluswrap_u32', avx2_scan_assoc_0{u32, +}}
|
||||
|
||||
# Boolean cumulative sum
|
||||
avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
def U = ty_u{T}
|
||||
def w = width{T}
|
||||
def vl= 256 / w
|
||||
@ -148,6 +148,6 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
step{load{xv, e}, e, st}
|
||||
}
|
||||
}
|
||||
'avx2_bcs8' = avx2_bcs{i8}
|
||||
'avx2_bcs16' = avx2_bcs{i16}
|
||||
'avx2_bcs32' = avx2_bcs{i32}
|
||||
export{'avx2_bcs8', avx2_bcs{i8}}
|
||||
export{'avx2_bcs16', avx2_bcs{i16}}
|
||||
export{'avx2_bcs32', avx2_bcs{i32}}
|
||||
|
||||
@ -97,7 +97,7 @@ def makeperm{VI, VD, x0, logv} = {
|
||||
makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}}
|
||||
}
|
||||
|
||||
select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
fn select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
w:= *TI ~~ w0
|
||||
x:= *TD ~~ x0
|
||||
r:= *TD ~~ r0
|
||||
@ -134,13 +134,12 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
}
|
||||
def select{TI, TD} = select{256, TI, TD}
|
||||
|
||||
def selects = join{table{select, tup{i8, i16, i32}, # indices
|
||||
tup{u8, u16, u32, u64}}} # values
|
||||
avx2_select_tab:*type{tupsel{0,selects}} = selects
|
||||
'avx2_select_tab' = avx2_select_tab
|
||||
exportT{'avx2_select_tab', join{table{select,
|
||||
tup{i8, i16, i32}, # indices
|
||||
tup{u8, u16, u32, u64}}}} # values
|
||||
|
||||
|
||||
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
fn avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
def TI = i8
|
||||
def VI = [32]TI
|
||||
w:= *VI ~~ w0
|
||||
@ -166,4 +165,4 @@ avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
}
|
||||
1
|
||||
}
|
||||
'avx2_select_bool128' = avx2_select_bool128
|
||||
export{'avx2_select_bool128', avx2_select_bool128}
|
||||
|
||||
@ -34,18 +34,18 @@ def comp16{w:*u64, X, r:*i16, l:u64} = {
|
||||
}
|
||||
}
|
||||
|
||||
slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
xv:= *u64~~x
|
||||
F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l}
|
||||
}
|
||||
|
||||
slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
|
||||
fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
|
||||
x:u64 = iota
|
||||
F{w, {} => {c:= x; x+= add; c}, r, l}
|
||||
}
|
||||
|
||||
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
|
||||
'bmipopc_2slash8' = slash2{comp8, i8}
|
||||
'bmipopc_2slash16' = slash2{comp16, i16}
|
||||
'bmipopc_1slash8' = slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}
|
||||
'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}
|
||||
export{'bmipopc_2slash8', slash2{comp8, i8}}
|
||||
export{'bmipopc_2slash16', slash2{comp16, i16}}
|
||||
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
|
||||
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
|
||||
|
||||
@ -62,7 +62,7 @@ def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
|
||||
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
|
||||
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
|
||||
|
||||
squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
assert{len>0}
|
||||
def bulk = vw / width{X}
|
||||
def XV = [bulk]X
|
||||
@ -177,12 +177,12 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
}
|
||||
}
|
||||
|
||||
'avx2_squeeze_i8' = squeeze{arch_defvw, i8, 0, 0}
|
||||
'avx2_squeeze_i16' = squeeze{arch_defvw, i16, 0, 0}
|
||||
'avx2_squeeze_i32' = squeeze{arch_defvw, i32, 0, 0}
|
||||
'avx2_squeeze_f64' = squeeze{arch_defvw, f64, 0, 0}
|
||||
'avx2_squeeze_numB'= squeeze{arch_defvw, f64, 0, 1}
|
||||
export{'avx2_squeeze_i8', squeeze{arch_defvw, i8, 0, 0}}
|
||||
export{'avx2_squeeze_i16', squeeze{arch_defvw, i16, 0, 0}}
|
||||
export{'avx2_squeeze_i32', squeeze{arch_defvw, i32, 0, 0}}
|
||||
export{'avx2_squeeze_f64', squeeze{arch_defvw, f64, 0, 0}}
|
||||
export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}}
|
||||
|
||||
'avx2_squeeze_c16' = squeeze{arch_defvw, u16, 1, 0}
|
||||
'avx2_squeeze_c32' = squeeze{arch_defvw, u32, 1, 0}
|
||||
'avx2_squeeze_chrB'= squeeze{arch_defvw, u64, 1, 1}
|
||||
export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}}
|
||||
export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}}
|
||||
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}
|
||||
@ -19,7 +19,7 @@ def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}}
|
||||
|
||||
|
||||
|
||||
transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
|
||||
fn transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
|
||||
if (wm<=0 or hm<=0) return{}
|
||||
|
||||
if (scl==1) {
|
||||
@ -37,7 +37,7 @@ transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void =
|
||||
}
|
||||
}
|
||||
|
||||
transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
rp:*T = *T~~r0
|
||||
xp:*T = *T~~x0
|
||||
|
||||
@ -60,4 +60,4 @@ transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h}
|
||||
}
|
||||
|
||||
'simd_transpose_i32' = transpose{u32}
|
||||
export{'simd_transpose_i32', transpose{u32}}
|
||||
Loading…
Reference in New Issue
Block a user