new Singeli syntax

This commit is contained in:
dzaima 2023-03-11 23:17:30 +02:00
parent 8194f51b30
commit 1178e5df56
17 changed files with 124 additions and 163 deletions

@ -1 +1 @@
Subproject commit e6d3b11e59eea7614307a37b27e7f10f4d3d04b8
Subproject commit 2b4e3b8cdf6f3a5f8ddf383a8421f012550b8c27

View File

@ -6,6 +6,8 @@ oper ~~ reinterpret infix right 55
oper ** broadcast infix right 55
def Size = u64
def load{x} = *x
def exportN{f, ...ns} = each{{n} => export{n, f}, ns}
def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} }
def elwidth{T} = width{eltype{T}}

View File

@ -24,10 +24,10 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
}}
}
bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
bitsel{VL, T, *T~~r, bits, trunc{T,e0}, trunc{T,e1}, len}
}
def table{w} = each{{T} => bitsel_i{w, T}, tup{u8, u16, u32, u64}}
def tup_bitsel = table{arch_defvw}; v_bitsel:*type{tupsel{0,tup_bitsel}} = tup_bitsel; 'simd_bitsel'=v_bitsel
exportT{'simd_bitsel', table{arch_defvw}}

View File

@ -15,7 +15,7 @@ def fillbits{dst:*u64, len:(Size), v } = { emit{void, 'fillBits', dst, len,
def fillbits{dst:*u64, len:(Size), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
def cmp_err{x} = { emit{void, 'cmp_err'}; return{}; }
cmpIX(dst:*u64, len:Size, x:u64, v:u1) : void = {
fn cmpIX(dst:*u64, len:Size, x:u64, v:u1) : void = {
nan:u1 = q_f64{x}
if (~(nan | q_chr{x})) cmp_err{x}
fillbits{dst, len, v & ~nan, x}
@ -87,26 +87,26 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
ri+= 1
}
}
aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= *VT~~xr; xs:= *eltype{VT}~~xr
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>load{xs,i}, {i}=>load{xv,i}, dst, len}
}
as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:Size) : void = {
fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:Size) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= VT**pathAS{dst, len, eltype{VT}, op, x}
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>x, {i}=>xv, dst, len}
}
bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
fn bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
@forNZ (dst, w in *u64~~wr, x in *u64~~xr over _ to cdiv{len,64}) dst = bitop{w,x}
}
not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
fn not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
fn cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
bitAS{op}(dst:*u64, wr:*void, x:u64, len:Size) : void = { # show{'bitAS'}
fn bitAS{op}(dst:*u64, wr:*void, x:u64, len:Size) : void = { # show{'bitAS'}
xf:f64 = interp_f64{x}
r0:u1 = op{0,xf}
r1:u1 = op{1,xf}
@ -135,14 +135,10 @@ def table{aa, F, w, G_bit, G_vec} = {
def tableAA{w, F, F_bit} = table{1, F, w, {} => bitAA{F_bit}, {V, u} => aa2bit{V, u, F}}
def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}}
def tup_eqAA = tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}; v_eqAA:*type{tupsel{0,tup_eqAA}} = tup_eqAA; 'simd_eqAA'=v_eqAA
def tup_neAA = tableAA{arch_defvw, __ne, {a,b}=>a ^ b}; v_neAA:*type{tupsel{0,tup_neAA}} = tup_neAA; 'simd_neAA'=v_neAA
def tup_gtAA = tableAA{arch_defvw, __gt, {a,b}=>a & ~b}; v_gtAA:*type{tupsel{0,tup_gtAA}} = tup_gtAA; 'simd_gtAA'=v_gtAA
def tup_geAA = tableAA{arch_defvw, __ge, {a,b}=>a | ~b}; v_geAA:*type{tupsel{0,tup_geAA}} = tup_geAA; 'simd_geAA'=v_geAA
def tup_eqAS = tableAS{arch_defvw, __eq}; v_eqAS:*type{tupsel{0,tup_eqAS}} = tup_eqAS; 'simd_eqAS'=v_eqAS
def tup_neAS = tableAS{arch_defvw, __ne}; v_neAS:*type{tupsel{0,tup_neAS}} = tup_neAS; 'simd_neAS'=v_neAS
def tup_gtAS = tableAS{arch_defvw, __gt}; v_gtAS:*type{tupsel{0,tup_gtAS}} = tup_gtAS; 'simd_gtAS'=v_gtAS
def tup_geAS = tableAS{arch_defvw, __ge}; v_geAS:*type{tupsel{0,tup_geAS}} = tup_geAS; 'simd_geAS'=v_geAS
def tup_ltAS = tableAS{arch_defvw, __lt}; v_ltAS:*type{tupsel{0,tup_ltAS}} = tup_ltAS; 'simd_ltAS'=v_ltAS
def tup_leAS = tableAS{arch_defvw, __le}; v_leAS:*type{tupsel{0,tup_leAS}} = tup_leAS; 'simd_leAS'=v_leAS
exportT{'simd_eqAS', tableAS{arch_defvw, __eq}}; exportT{'simd_eqAA', tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}}
exportT{'simd_neAS', tableAS{arch_defvw, __ne}}; exportT{'simd_neAA', tableAA{arch_defvw, __ne, {a,b}=>a ^ b}}
exportT{'simd_gtAS', tableAS{arch_defvw, __gt}}; exportT{'simd_gtAA', tableAA{arch_defvw, __gt, {a,b}=>a & ~b}}
exportT{'simd_geAS', tableAS{arch_defvw, __ge}}; exportT{'simd_geAA', tableAA{arch_defvw, __ge, {a,b}=>a | ~b}}
exportT{'simd_ltAS', tableAS{arch_defvw, __lt}}
exportT{'simd_leAS', tableAS{arch_defvw, __le}}

View File

@ -86,7 +86,7 @@ def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get
def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially
rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}}
rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}}
rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
fn rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
def V = [32]i8
def st = read_shuf_vecs{wv, elbytes, sh}
rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
@ -95,7 +95,7 @@ rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}}
rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
fn rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
def h = 4
def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}}
def V = [32]i8
@ -125,7 +125,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
}
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
if (wv > tupsel{-1,rcsh_vals}) {
return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)}
}
@ -146,10 +146,10 @@ def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = {
store{*V~~r, -1, v}
}
}
rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n}
rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n}
fn rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n}
fn rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n}
rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
assert{wv>=2}
if (wv>=8 and wv<=fact_size) {
fa := promote{u64, load{factors,wv-8}}
@ -180,5 +180,5 @@ rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
}
}
'constrep_u8' = rep_const{i8 }; 'constrep_u16' = rep_const{i16}
'constrep_u32' = rep_const{i32}; 'constrep_u64' = rep_const{u64}
export{'constrep_u8', rep_const{i8 }}; export{'constrep_u16', rep_const{i16}}
export{'constrep_u32', rep_const{i32}}; export{'constrep_u64', rep_const{u64}}

View File

@ -23,7 +23,7 @@ def copyFromBits{T, loadFn, rp, l:u64} = {
}
copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
def vw = arch_defvw
assert{l!=0}
@ -75,7 +75,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
}
}
copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
fn copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
assert{l!=0}
x0:= (*u8~~xRaw) + cbqn_tyArrOffset{}
xs:= u64~~((*u8~~x) - x0)
@ -91,60 +91,26 @@ copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
# }
}
# simd_copy_src_dst
# x→int & equal-width copies
'simd_copy_1_1' = copy{u1, u1}
'simd_copy_1_i8' = copy{u1, i8}
'simd_copy_1_i16' = copy{u1, i16}
'simd_copy_1_i32' = copy{u1, i32}
'simd_copy_i8_1' = copy{i8, u1}
'simd_copy_i8_i8', 'simd_copy_c8_c8' = copy{i8, i8}
'simd_copy_i8_i16' = copy{i8, i16}
'simd_copy_i8_i32' = copy{i8, i32}
def gen{p} = {
def ts = tup{u1, i8, i16, i32, f64, u8, u16, u32, u64}
def tn = tup{'1','i8','i16','i32','f64','c8','c16','c32','B'}
def tm = tup{0, 0, 0, 0, 0, 1, 1, 1, 2}
each{{tx0,nx,mx} => {
each{{tr0,nr,mr} => {
if ((mx==mr or mx==2 or mr==2) and (if (mx==2) mr==1; else 1)) {
def tr = if (mx==0 and mr==2) f64; else if (tx0==tr0 and mx==1) ty_s{tx0}; else tr0
def tx = if (mr==0 and mx==2) f64; else if (tx0==tr0 and mx==1) ty_s{tx0}; else tx0
export{merge{p, nx, '_', nr}, copy{tx, tr}}
}
}, ts, tn, tm}
}, ts, tn, tm}
}
'simd_copy_i16_1' = copy{i16, u1}
'simd_copy_i16_i8' = copy{i16, i8}
'simd_copy_i16_i16', 'simd_copy_c16_c16' = copy{i16, i16}
'simd_copy_i16_i32' = copy{i16, i32}
'simd_copy_i32_1' = copy{i32, u1}
'simd_copy_i32_i8' = copy{i32, i8}
'simd_copy_i32_i16' = copy{i32, i16}
'simd_copy_i32_i32', 'simd_copy_c32_c32'= copy{i32, i32}
'simd_copy_f64_1' = copy{f64, u1}
'simd_copy_f64_i8' = copy{f64, i8}
'simd_copy_f64_i16' = copy{f64, i16}
'simd_copy_f64_i32' = copy{f64, i32}
# x→f64, x→B (no simd_copy_B_B because that may possibly need refcounting)
'simd_copy_1_f64', 'simd_copy_1_B' = copy{u1, f64}
'simd_copy_i8_f64', 'simd_copy_i8_B' = copy{i8, f64}
'simd_copy_i16_f64', 'simd_copy_i16_B' = copy{i16, f64}
'simd_copy_i32_f64', 'simd_copy_i32_B' = copy{i32, f64}
'simd_copy_f64_f64', 'simd_copy_f64_B' = copy{f64, f64}
# chr→x
'simd_copy_c8_c16' = copy{u8, u16}
'simd_copy_c8_c32' = copy{u8, u32}
'simd_copy_c8_B' = copy{u8, u64}
'simd_copy_c16_c8' = copy{u16, u8}
'simd_copy_c16_c32' = copy{u16, u32}
'simd_copy_c16_B' = copy{u16, u64}
'simd_copy_c32_c8' = copy{u32, u8}
'simd_copy_c32_c16' = copy{u32, u16}
'simd_copy_c32_B' = copy{u32, u64}
# B→chr
'simd_copy_B_c8' = copy{u64, u8}
'simd_copy_B_c16' = copy{u64, u16}
'simd_copy_B_c32' = copy{u64, u32}
gen{'simd_copy_'}
# unaligned bitarr widening
'simd_copy_1u_i8' = copy_ubit{i8}
'simd_copy_1u_i16' = copy_ubit{i16}
'simd_copy_1u_i32' = copy_ubit{i32}
'simd_copy_1u_f64' = copy_ubit{f64}
export{'simd_copy_1u_i8', copy_ubit{i8}}
export{'simd_copy_1u_i16', copy_ubit{i16}}
export{'simd_copy_1u_i32', copy_ubit{i32}}
export{'simd_copy_1u_f64', copy_ubit{f64}}

View File

@ -173,12 +173,12 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
}
}
arithAAc{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : u64 = {
fn arithAAc{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : u64 = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
if (mode==1) len
else 0
}
arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
fn arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
}
@ -189,7 +189,7 @@ def arithAA{mode, F, W, X, R} = {
}
# mode: 0:overflow-checked, needed; 1:overflow-erroring; 2: overflow-checked, not needed
arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
# show{F, swap, mode, W, X, R}
assert{len>0}
def bulk = vw / max{width{W}, width{R}}
@ -214,7 +214,7 @@ arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R}
andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
assert{vw>=64}
def bulk = vw / width{u8}
def T8 = [bulk]u8
@ -225,6 +225,6 @@ andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
}}
}
'simd_andBytes'=andBytes{arch_defvw}
'orSAc_f64_f64_f64'=arithSA{2,bqn_or,0,f64,f64,f64}
export{'simd_andBytes', andBytes{arch_defvw}}
export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}}
include 'gen/arDefs'

View File

@ -18,7 +18,7 @@ def swap{w,x} = {
}
# width{W} ≤ width{X}
equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
def vw = arch_defvw
def bulk = vw / width{X}
if (W!=X) if (d!=0) swap{w,x}
@ -77,23 +77,23 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
1
}
'avx2_equal_1_1' = equal{u1, u1}
'avx2_equal_1_8' = equal{u1, u8}
'avx2_equal_1_16' = equal{u1, u16}
'avx2_equal_1_32' = equal{u1, u32}
'avx2_equal_1_f64' = equal{u1, f64}
export{'avx2_equal_1_1', equal{u1, u1}}
export{'avx2_equal_1_8', equal{u1, u8}}
export{'avx2_equal_1_16', equal{u1, u16}}
export{'avx2_equal_1_32', equal{u1, u32}}
export{'avx2_equal_1_f64', equal{u1, f64}}
'avx2_equal_8_8' = equal{i8, i8}
export{'avx2_equal_8_8', equal{i8, i8}}
'avx2_equal_s8_16' = equal{i8, i16}
'avx2_equal_s8_32' = equal{i8, i32}
'avx2_equal_s16_32' = equal{i16, i32}
export{'avx2_equal_s8_16', equal{i8, i16}}
export{'avx2_equal_s8_32', equal{i8, i32}}
export{'avx2_equal_s16_32', equal{i16, i32}}
'avx2_equal_s8_f64' = equal{i8, f64}
'avx2_equal_s16_f64'= equal{i16, f64}
'avx2_equal_s32_f64'= equal{i32, f64}
'avx2_equal_f64_f64'= equal{f64, f64}
export{'avx2_equal_s8_f64', equal{i8, f64}}
export{'avx2_equal_s16_f64', equal{i16, f64}}
export{'avx2_equal_s32_f64', equal{i32, f64}}
export{'avx2_equal_f64_f64', equal{f64, f64}}
'avx2_equal_u8_16' = equal{u8, u16}
'avx2_equal_u8_32' = equal{u8, u32}
'avx2_equal_u16_32' = equal{u16, u32}
export{'avx2_equal_u8_16', equal{u8, u16}}
export{'avx2_equal_u8_32', equal{u8, u32}}
export{'avx2_equal_u16_32', equal{u16, u32}}

View File

@ -21,7 +21,7 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = {
r
}
fold_idem{T==f64, op}(x:*T, len:u64) : T = {
fn fold_idem{T==f64, op}(x:*T, len:u64) : T = {
def step = 256/width{T}
def V = [step]T
xv:= *V ~~ x
@ -41,10 +41,10 @@ fold_idem{T==f64, op}(x:*T, len:u64) : T = {
extract{r, 0}
}
'avx2_fold_min_f64' = fold_idem{f64,min}
'avx2_fold_max_f64' = fold_idem{f64,max}
export{'avx2_fold_min_f64', fold_idem{f64,min}}
export{'avx2_fold_max_f64', fold_idem{f64,max}}
fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
def step = 256/width{T}
def V = [step]T
xv:= *V ~~ x
@ -53,4 +53,4 @@ fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = {
r:= reduce_pairwise{op, 2, xv, e, i}
extract{mix{op, r}, 0}
}
'avx2_sum_f64' = fold_assoc_0{f64,+}
export{'avx2_sum_f64', fold_assoc_0{f64,+}}

View File

@ -163,7 +163,7 @@ Make ← {𝕊ns:
tr@?
siName id, argarr, 𝕨"?ce", '_'¨ twtxtr
𝕨 { 𝕊:
siSrc "'", siName, "'=arithSA{", '0'+ 𝕨@, onlyFirst02, 1, ',', si, ',', '0'+swap, (',' 'u'('c'))¨ twtxtr, '}', nl
siSrc "export{'", siName, "', arithSA{", '0'+ 𝕨@, onlyFirst02, 1, ',', si, ',', '0'+swap, (',' 'u'('c'))¨ twtxtr, "}}", nl
singeliASFns <siName
} ¬(<siName)singeliASFns
siName;
@ -242,7 +242,7 @@ Make ← {𝕊ns:
GenSingeli { 𝕊namewxrtmsi:
si@?
siSrc "'", name, "'=arithAA{", '0'+tm, ',', si, ',', wtySiName, ',', xtySiName, ',', rtySiName, '}', nl;
siSrc "export{'", name, "', arithAA{", '0'+tm, ',', si, ',', wtySiName, ',', xtySiName, ',', rtySiName, "}}", nl;
# else
@
}

View File

@ -12,7 +12,7 @@ include './bitops'
include './mask'
absFn{T}(r:*void, x:*void, len:u64) : u64 = {
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T}
def VT = [bulk]T
muLoop{bulk, tern{T==f64, 2, 1}, len, {is, M} => {
@ -23,6 +23,4 @@ absFn{T}(r:*void, x:*void, len:u64) : u64 = {
len
}
def tup_abs = each{{T}=>absFn{T}, tup{i8, i16, i32, f64}}
t_abs:*type{tupsel{0,tup_abs}} = tup_abs
'simd_abs' = t_abs
exportT{'simd_abs', each{{T}=>absFn{T}, tup{i8, i16, i32, f64}}}

View File

@ -4,7 +4,7 @@ include './sse3'
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}
clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
def V = [2]u64
m := V**mark
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
@ -25,7 +25,7 @@ clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
}
}
clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = {
fn clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = {
clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1))
}
'clmul_scan_ne' = clmul_scan_ne_bit
export{'clmul_scan_ne', clmul_scan_ne_bit}

View File

@ -39,7 +39,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
}
# Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈
avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
fn avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
# Within each lane, scan using shifts by powers of 2. First k elements
# when shifting by k don't need to change, so leave them alone.
def w = width{T}
@ -55,26 +55,26 @@ avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
scan_post{T, init, x, r, len, op, pre}
}
avx2_scan_idem{T==f64, op}(x:*T, r:*T, len:u64, init:T) : void = {
fn avx2_scan_idem{T==f64, op}(x:*T, r:*T, len:u64, init:T) : void = {
def sh{s, a} = op{a, shuf{[4]u64, a, s}}
scan_post{T, init, x, r, len, op, {a}=>sh{4b1110,sh{4b2200,a}}}
}
'avx2_scan_min_init_i8' = avx2_scan_idem{i8 , min}; 'avx2_scan_max_init_i8' = avx2_scan_idem{i8 , max}
'avx2_scan_min_init_i16' = avx2_scan_idem{i16, min}; 'avx2_scan_max_init_i16' = avx2_scan_idem{i16, max}
'avx2_scan_min_init_i32' = avx2_scan_idem{i32, min}; 'avx2_scan_max_init_i32' = avx2_scan_idem{i32, max}
'avx2_scan_min_init_f64' = avx2_scan_idem{f64, min}; 'avx2_scan_max_init_f64' = avx2_scan_idem{f64, max}
export{'avx2_scan_min_init_i8', avx2_scan_idem{i8 , min}}; export{'avx2_scan_max_init_i8', avx2_scan_idem{i8 , max}}
export{'avx2_scan_min_init_i16', avx2_scan_idem{i16, min}}; export{'avx2_scan_max_init_i16', avx2_scan_idem{i16, max}}
export{'avx2_scan_min_init_i32', avx2_scan_idem{i32, min}}; export{'avx2_scan_max_init_i32', avx2_scan_idem{i32, max}}
export{'avx2_scan_min_init_f64', avx2_scan_idem{f64, min}}; export{'avx2_scan_max_init_f64', avx2_scan_idem{f64, max}}
avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
fn avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
def m = 1 << (width{T}-1)
def id = (if (match{op,min}) m-1; else -m)
avx2_scan_idem{T, op}(x, r, len, id)
}
'avx2_scan_min_i8' = avx2_scan_idem_id{i8 , min}; 'avx2_scan_max_i8' = avx2_scan_idem_id{i8 , max}
'avx2_scan_min_i16' = avx2_scan_idem_id{i16, min}; 'avx2_scan_max_i16' = avx2_scan_idem_id{i16, max}
'avx2_scan_min_i32' = avx2_scan_idem_id{i32, min}; 'avx2_scan_max_i32' = avx2_scan_idem_id{i32, max}
export{'avx2_scan_min_i8', avx2_scan_idem_id{i8 , min}}; export{'avx2_scan_max_i8', avx2_scan_idem_id{i8 , max}}
export{'avx2_scan_min_i16', avx2_scan_idem_id{i16, min}}; export{'avx2_scan_max_i16', avx2_scan_idem_id{i16, max}}
export{'avx2_scan_min_i32', avx2_scan_idem_id{i32, min}}; export{'avx2_scan_max_i32', avx2_scan_idem_id{i32, max}}
# Associative scan
avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
fn avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
# Prefix op on entire AVX register
def pre{a} = {
# Within each lane, scan using shifts by powers of 2.
@ -88,12 +88,12 @@ avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
}
scan_post{T, init, x, r, len, op, pre}
}
'avx2_scan_pluswrap_u8' = avx2_scan_assoc_0{u8 , +}
'avx2_scan_pluswrap_u16' = avx2_scan_assoc_0{u16, +}
'avx2_scan_pluswrap_u32' = avx2_scan_assoc_0{u32, +}
export{'avx2_scan_pluswrap_u8', avx2_scan_assoc_0{u8 , +}}
export{'avx2_scan_pluswrap_u16', avx2_scan_assoc_0{u16, +}}
export{'avx2_scan_pluswrap_u32', avx2_scan_assoc_0{u32, +}}
# Boolean cumulative sum
avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
def U = ty_u{T}
def w = width{T}
def vl= 256 / w
@ -148,6 +148,6 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
step{load{xv, e}, e, st}
}
}
'avx2_bcs8' = avx2_bcs{i8}
'avx2_bcs16' = avx2_bcs{i16}
'avx2_bcs32' = avx2_bcs{i32}
export{'avx2_bcs8', avx2_bcs{i8}}
export{'avx2_bcs16', avx2_bcs{i16}}
export{'avx2_bcs32', avx2_bcs{i32}}

View File

@ -97,7 +97,7 @@ def makeperm{VI, VD, x0, logv} = {
makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}}
}
select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
fn select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
w:= *TI ~~ w0
x:= *TD ~~ x0
r:= *TD ~~ r0
@ -134,13 +134,12 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
}
def select{TI, TD} = select{256, TI, TD}
def selects = join{table{select, tup{i8, i16, i32}, # indices
tup{u8, u16, u32, u64}}} # values
avx2_select_tab:*type{tupsel{0,selects}} = selects
'avx2_select_tab' = avx2_select_tab
exportT{'avx2_select_tab', join{table{select,
tup{i8, i16, i32}, # indices
tup{u8, u16, u32, u64}}}} # values
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
fn avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def TI = i8
def VI = [32]TI
w:= *VI ~~ w0
@ -166,4 +165,4 @@ avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
}
1
}
'avx2_select_bool128' = avx2_select_bool128
export{'avx2_select_bool128', avx2_select_bool128}

View File

@ -34,18 +34,18 @@ def comp16{w:*u64, X, r:*i16, l:u64} = {
}
}
slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = {
fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = {
xv:= *u64~~x
F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l}
}
slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
x:u64 = iota
F{w, {} => {c:= x; x+= add; c}, r, l}
}
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
'bmipopc_2slash8' = slash2{comp8, i8}
'bmipopc_2slash16' = slash2{comp16, i16}
'bmipopc_1slash8' = slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}
'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}
export{'bmipopc_2slash8', slash2{comp8, i8}}
export{'bmipopc_2slash16', slash2{comp16, i16}}
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}

View File

@ -62,7 +62,7 @@ def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
assert{len>0}
def bulk = vw / width{X}
def XV = [bulk]X
@ -177,12 +177,12 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
}
}
'avx2_squeeze_i8' = squeeze{arch_defvw, i8, 0, 0}
'avx2_squeeze_i16' = squeeze{arch_defvw, i16, 0, 0}
'avx2_squeeze_i32' = squeeze{arch_defvw, i32, 0, 0}
'avx2_squeeze_f64' = squeeze{arch_defvw, f64, 0, 0}
'avx2_squeeze_numB'= squeeze{arch_defvw, f64, 0, 1}
export{'avx2_squeeze_i8', squeeze{arch_defvw, i8, 0, 0}}
export{'avx2_squeeze_i16', squeeze{arch_defvw, i16, 0, 0}}
export{'avx2_squeeze_i32', squeeze{arch_defvw, i32, 0, 0}}
export{'avx2_squeeze_f64', squeeze{arch_defvw, f64, 0, 0}}
export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}}
'avx2_squeeze_c16' = squeeze{arch_defvw, u16, 1, 0}
'avx2_squeeze_c32' = squeeze{arch_defvw, u32, 1, 0}
'avx2_squeeze_chrB'= squeeze{arch_defvw, u64, 1, 1}
export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}}
export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}}
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}

View File

@ -19,7 +19,7 @@ def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}}
transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
fn transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
if (wm<=0 or hm<=0) return{}
if (scl==1) {
@ -37,7 +37,7 @@ transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void =
}
}
transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
rp:*T = *T~~r0
xp:*T = *T~~x0
@ -60,4 +60,4 @@ transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h}
}
'simd_transpose_i32' = transpose{u32}
export{'simd_transpose_i32', transpose{u32}}