make arch_defvw a generator

This commit is contained in:
dzaima 2025-05-01 16:20:41 +03:00
parent 624c74058c
commit 4e12204c52
18 changed files with 78 additions and 78 deletions

View File

@ -161,7 +161,7 @@ def el_s{V=[_]T} = re_el{w_h{T}, V}
# vector definitions # vector definitions
def arch_defvw = if (hasarch{'AVX2'}) 256 else 128 def arch_defvw{} = if (hasarch{'AVX2'}) 256 else 128
def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'} def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'}
def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'} def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'}
def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP

View File

@ -22,7 +22,7 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}} def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}}
export_tab{'simd_bitsel', table{arch_defvw}} export_tab{'simd_bitsel', table{arch_defvw{}}}
fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : void = { fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : void = {
if (same{E,'!'}) { if (same{E,'!'}) {
@ -30,7 +30,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
} else if (E==u1) { } else if (E==u1) {
emit{void, 'blendArrScalarBits', rp, zero, cast_i{E, one0}, mask, len} emit{void, 'blendArrScalarBits', rp, zero, cast_i{E, one0}, mask, len}
} else { } else {
def bulk = arch_defvw / width{E} def bulk = arch_defvw{} / width{E}
def VT = [bulk]E def VT = [bulk]E
def one = VT**cast_i{E, one0} def one = VT**cast_i{E, one0}
@for_mu{bulk,mask_unr{bulk}}(r in tup{'g',*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r{eachx{blend_hom, zero, one, mask}} @for_mu{bulk,mask_unr{bulk}}(r in tup{'g',*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r{eachx{blend_hom, zero, one, mask}}
@ -43,7 +43,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
fn bitwiden_n_8(src:*void, dst:*void, csz:ux, cam:ux) : void = { fn bitwiden_n_8(src:*void, dst:*void, csz:ux, cam:ux) : void = {
assert{cam>0} assert{cam>0}
assert{(csz>1) & (csz<8)} assert{(csz>1) & (csz<8)}
def bulk = arch_defvw / 8 def bulk = arch_defvw{} / 8
def V = [bulk]u8 def V = [bulk]u8
bitalign{tup{2,8,csz}, 8, {s, align} => { bitalign{tup{2,8,csz}, 8, {s, align} => {
@for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) { @for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) {
@ -59,7 +59,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
fn bitnarrow_8_n(src:*void, dst:*void, csz:ux, cam:ux) : void = { fn bitnarrow_8_n(src:*void, dst:*void, csz:ux, cam:ux) : void = {
assert{cam>0} assert{cam>0}
assert{(csz>1) & (csz<8)} assert{(csz>1) & (csz<8)}
def bulk = arch_defvw / 8 def bulk = arch_defvw{} / 8
def V = [bulk]u8 def V = [bulk]u8
dstC:= *u8~~dst dstC:= *u8~~dst
dstE:= *u8~~dst + cdiv{csz*cam, 8} dstE:= *u8~~dst + cdiv{csz*cam, 8}

View File

@ -130,9 +130,9 @@ def tableAA{w, F, F_bit} = table{1, F, w, {} => bitAA{F_bit}, {V, u} => aa2bit{V
def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}} def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}}
export_tab{'simd_eqAS', tableAS{arch_defvw, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}} export_tab{'simd_eqAS', tableAS{arch_defvw{}, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw{}, __eq, {a,b}=>a ^ ~b}}
export_tab{'simd_neAS', tableAS{arch_defvw, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw, __ne, {a,b}=>a ^ b}} export_tab{'simd_neAS', tableAS{arch_defvw{}, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw{}, __ne, {a,b}=>a ^ b}}
export_tab{'simd_gtAS', tableAS{arch_defvw, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw, __gt, {a,b}=>a & ~b}} export_tab{'simd_gtAS', tableAS{arch_defvw{}, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw{}, __gt, {a,b}=>a & ~b}}
export_tab{'simd_geAS', tableAS{arch_defvw, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw, __ge, {a,b}=>a | ~b}} export_tab{'simd_geAS', tableAS{arch_defvw{}, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw{}, __ge, {a,b}=>a | ~b}}
export_tab{'simd_ltAS', tableAS{arch_defvw, __lt}} export_tab{'simd_ltAS', tableAS{arch_defvw{}, __lt}}
export_tab{'simd_leAS', tableAS{arch_defvw, __le}} export_tab{'simd_leAS', tableAS{arch_defvw{}, __le}}

View File

@ -14,7 +14,7 @@ def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = {
fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
def vw = arch_defvw def vw = arch_defvw{}
assert{l!=0} assert{l!=0}
def bulk = vw/__max{width{X}, width{R}} def bulk = vw/__max{width{X}, width{R}}
@ -68,7 +68,7 @@ fn copy_ubit{R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
# if ((xs&7)==0) { # if ((xs&7)==0) {
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw) # copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
# } else { # } else {
def vw = arch_defvw def vw = arch_defvw{}
def bulk = vw/width{R} def bulk = vw/width{R}
def RV = [bulk]R def RV = [bulk]R
rp:= *R~~r rp:= *R~~r

View File

@ -6,7 +6,7 @@ def inc{ptr, ind} = inc{ptr, ind, 1}
# Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x # Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x
fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
def vbits = arch_defvw def vbits = arch_defvw{}
def vec = vbits/width{T} def vec = vbits/width{T}
def TU = ty_u{T} def TU = ty_u{T}
def V = [vec]T def V = [vec]T
@ -77,7 +77,7 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
} }
fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = { fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
def vl = arch_defvw/16 def vl = arch_defvw{}/16
def V = [vl]u16 def V = [vl]u16
def bot = tail{15} def bot = tail{15}
on:usz = 0 on:usz = 0
@ -147,7 +147,7 @@ def count_adapt_runs{x0, tab, n} = {
} }
} }
def mark_run_ends{x:*T, m:(ux)} = { def mark_run_ends{x:*T, m:(ux)} = {
def vec = arch_defvw/width{T} def vec = arch_defvw{}/width{T}
def V = [vec]T def V = [vec]T
@unroll (j to width{ux} / vec) { @unroll (j to width{ux} / vec) {
def jv = j*vec def jv = j*vec
@ -175,7 +175,7 @@ fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = count_adapt_runs{x, tab, n}
# ov is sorted but may not be unique, and oc contains multiples of 128 # ov is sorted but may not be unique, and oc contains multiples of 128
# Return the shared length of ov and oc # Return the shared length of ov and oc
fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = { fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = {
def V = [arch_defvw/width{T}]T def V = [arch_defvw{}/width{T}]T
def block = 128 def block = 128
i:usz = 0 i:usz = 0
on:usz = 0 on:usz = 0

View File

@ -164,7 +164,7 @@ fn arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
} }
def arithAA{mode, F, W, X, R} = { def arithAA{mode, F, W, X, R} = {
def vw = arch_defvw def vw = arch_defvw{}
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R} if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
else arithAAu{vw, mode, F, W, X, R} else arithAAu{vw, mode, F, W, X, R}
} }
@ -192,7 +192,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
else len else len
} }
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R} def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw{}, mode, F, swap, W, X, R}
fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
assert{vw>=64} assert{vw>=64}
@ -203,6 +203,6 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
@for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull} @for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
} }
export{'simd_andBytes', andBytes{arch_defvw}} export{'simd_andBytes', andBytes{arch_defvw{}}}
export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}} export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}}
include 'gen/arDefs' include 'gen/arDefs'

View File

@ -12,7 +12,7 @@ def swap{w,x} = {
# width{W} ≤ width{X} # width{W} ≤ width{X}
fn equal{W, X}(w:*void, x:*void, l:ux, d:u64) : u1 = { fn equal{W, X}(w:*void, x:*void, l:ux, d:u64) : u1 = {
def vw = arch_defvw def vw = arch_defvw{}
def bulk = vw / width{X} def bulk = vw / width{X}
if (W!=X) if (d!=0) swap{w,x} if (W!=X) if (d!=0) swap{w,x}
assert{l>0} assert{l>0}
@ -83,7 +83,7 @@ def any_ne_reflexive_qnan{M, a:V=[k](f64), b:V} = { # (a==b) | (isQNaN{a} & isQN
fn equal_reflexive{}(w:*void, x:*void, l:ux, d:u64) : u1 = { fn equal_reflexive{}(w:*void, x:*void, l:ux, d:u64) : u1 = {
def w = *f64~~w def w = *f64~~w
def x = *f64~~x def x = *f64~~x
def bulk = arch_defvw / width{f64} def bulk = arch_defvw{} / width{f64}
def unr = if (hasarch{'AARCH64'}) 2 else 1 def unr = if (hasarch{'AARCH64'}) 2 else 1
def V = [bulk]f64 def V = [bulk]f64
@for_mu{bulk, unr}(w in tup{V,w}, x in tup{V,x}, M in 'm' over i to l) { @for_mu{bulk, unr}(w in tup{V,w}, x in tup{V,x}, M in 'm' over i to l) {

View File

@ -29,7 +29,7 @@ fn fold_idem{T, op}(x:*T, len:u64) : T = {
a a
} }
fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = { fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = {
def bulk = arch_defvw/width{T} def bulk = arch_defvw{}/width{T}
def V = [bulk]T def V = [bulk]T
xv:= *V ~~ x xv:= *V ~~ x
r:V = [bulk]f64**0 r:V = [bulk]f64**0
@ -62,7 +62,7 @@ fn fold_assoc_0{T, op}(x:*T, len:u64) : T = {
a a
} }
fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = {
def bulk = arch_defvw/width{T} def bulk = arch_defvw{}/width{T}
def V = [bulk]T def V = [bulk]T
xv:= *V ~~ x xv:= *V ~~ x
e:= len / bulk e:= len / bulk
@ -75,7 +75,7 @@ export{'si_sum_f64', fold_assoc_0{f64,+}}
def extract_column_pow2{T, x0, r0, nv, k} = { def extract_column_pow2{T, x0, r0, nv, k} = {
def V = [arch_defvw / width{T}]T def V = [arch_defvw{} / width{T}]T
xv := *V~~x0 xv := *V~~x0
@for (r in *V~~r0 over i to nv) { @for (r in *V~~r0 over i to nv) {
xs := each{load{xv, .}, iota{k}} xs := each{load{xv, .}, iota{k}}
@ -234,7 +234,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
# Maximum of n result values, return actual number written # Maximum of n result values, return actual number written
fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = { fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = {
n <<= el n <<= el
def vl = arch_defvw / 8 def vl = arch_defvw{} / 8
def thr = __min{vl+2, 20} def thr = __min{vl+2, 20}
if ((not has_simd) or n < vl or l > usz~~thr>>el or l<<el >= thr) return{0} if ((not has_simd) or n < vl or l > usz~~thr>>el or l<<el >= thr) return{0}
nv := n / vl nv := n / vl
@ -529,7 +529,7 @@ fn sum_small{T if has_simd}(xv:*void, ia:usz) : i64 = {
def A = if (width{T}<=16) i32 else i64 def A = if (width{T}<=16) i32 else i64
xv:= *T~~xv xv:= *T~~xv
def unr = 4 def unr = 4
def bulk = arch_defvw / width{T} def bulk = arch_defvw{} / width{T}
def V = [bulk]T def V = [bulk]T
def acc = sum_accumulator{A, unr, V} def acc = sum_accumulator{A, unr, V}

View File

@ -21,7 +21,7 @@ def group_statistics{T} = {
fn group_statistics{T}(w:*void, xn:usz, outs:each{__pnt,types}) : void = { fn group_statistics{T}(w:*void, xn:usz, outs:each{__pnt,types}) : void = {
def w = *T~~w def w = *T~~w
def accs = if (has_simd) { def accs = if (has_simd) {
def bulk = arch_defvw/width{T} def bulk = arch_defvw{}/width{T}
def V = [bulk]T def V = [bulk]T
def VU = ty_u{V} def VU = ty_u{V}

View File

@ -36,7 +36,7 @@ local def loader{G} = {
} }
def unzip_load{E, n, TG} = each{merge, unzip_load{E, n/2, TG}, unzip_load{E, n/2, TG{'offset', E, n}}} def unzip_load{E, n, TG} = each{merge, unzip_load{E, n/2, TG}, unzip_load{E, n/2, TG{'offset', E, n}}}
def unzip_load{E, n, TG if width{E}*n <= arch_defvw} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}} def unzip_load{E, n, TG if width{E}*n <= arch_defvw{}} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}}
def widen_tup{(u32), is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is def widen_tup{(u32), is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is
widen{[8]u32, is}, widen{[8]u32, is},

View File

@ -6,7 +6,7 @@ include './mask'
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T} def bulk = arch_defvw{}/width{T}
def VT = [bulk]T def VT = [bulk]T
@for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { @for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk} if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk}

View File

@ -82,7 +82,7 @@ def fact_inds = slice{iota{fact_size+1},8}
def fact_tab = makefact{basic_rep, fact_inds} def fact_tab = makefact{basic_rep, fact_inds}
factors:*u8 = fact_tab factors:*u8 = fact_tab
def sdtype = [arch_defvw/8]i8 # shuf data type def sdtype = [arch_defvw{}/8]i8 # shuf data type
def get_shufs{step, wv, len} = { def get_shufs{step, wv, len} = {
def i = iota{len*step} def i = iota{len*step}
split{step, (i - i%wv)/wv} split{step, (i - i%wv)/wv}
@ -240,7 +240,7 @@ fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = { def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = {
assert{kv > 0} assert{kv > 0}
def V = [arch_defvw/width{T}]T def V = [arch_defvw{}/width{T}]T
@for (x over n) { @for (x over n) {
v := V**x v := V**x
@loop (j to kv) store{*V~~r, j, v} @loop (j to kv) store{*V~~r, j, v}
@ -264,7 +264,7 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
} }
} }
def wT = width{T} def wT = width{T}
def vn = arch_defvw/wT def vn = arch_defvw{}/wT
def V = [vn]T def V = [vn]T
def max_shuffle = 2*vn def max_shuffle = 2*vn
if (wv <= max_shuffle) { if (wv <= max_shuffle) {

View File

@ -17,7 +17,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = {
} }
def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = { def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = {
def step = arch_defvw/width{T} def step = arch_defvw{}/width{T}
def V = [step]T def V = [step]T
p:= V**init p:= V**init
xv:= *V~~x xv:= *V~~x
@ -42,7 +42,7 @@ def scan_idem = scan_scal
fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = { fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = {
def {scan, last} = get_scan_last{op, make_scan_idem{T, op}} def {scan, last} = get_scan_last{op, make_scan_idem{T, op}}
def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) } def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) }
def step = arch_defvw/width{T} def step = arch_defvw{}/width{T}
def V = [step]T def V = [step]T
p:= V**init p:= V**init
xv:= *V ~~ x xv:= *V ~~ x
@ -183,7 +183,7 @@ fn scan_stride_assoc{op, T, Ret, check_over}(xv:*void, rv:*void, ia:usz, l:usz)
} }
} }
} else if (same{op, +} and T<=i32 and has_simd and (has_shuf or l>=vl)) { } else if (same{op, +} and T<=i32 and has_simd and (has_shuf or l>=vl)) {
def vl = arch_defvw/wT; def V = [vl]T def vl = arch_defvw{}/wT; def V = [vl]T
@for_masked{vl} (x in tup{V, x+l}, r in tup{V, r+l}, p in tup{V, r}, M in 'm' over ia-l) { @for_masked{vl} (x in tup{V, x+l}, r in tup{V, r+l}, p in tup{V, r}, M in 'm' over ia-l) {
r = op_chk{M, p, x} r = op_chk{M, p, x}
} }
@ -225,7 +225,7 @@ fn scan_neq{}(c:u64, x:*u64, r:*u64, nw:usz) : void = {
} }
} }
fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:usz) : void = { fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:usz) : void = {
def vl = arch_defvw / 64 def vl = arch_defvw{} / 64
def V = [vl]u64 def V = [vl]u64
c := V**c0 c := V**c0
@for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) { @for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) {
@ -362,9 +362,9 @@ def addChk{a:T, b:T==f64} = tup{0, a+b}
def widenFull{E, xs} = { def widenFull{E, xs} = {
merge{...each{{x:X=[k]T} => { merge{...each{{x:X=[k]T} => {
def tb = width{E} * k def tb = width{E} * k
if (tb<=arch_defvw) tup{widen{[k]E, x}} if (tb<=arch_defvw{}) tup{widen{[k]E, x}}
else if (1) { else if (1) {
assert{tb == 2*arch_defvw} assert{tb == 2*arch_defvw{}}
tup{ tup{
widen{[k/2]E, half{x,0}}, widen{[k/2]E, half{x,0}},
widen{[k/2]E, half{x,1}} widen{[k/2]E, half{x,1}}
@ -395,7 +395,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
# Sum as many vector registers as possible; modifies c and i # Sum as many vector registers as possible; modifies c and i
def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = { def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
def b = __max{width{R}/2, width{X}} def b = __max{width{R}/2, width{X}}
def bulk = arch_defvw/b def bulk = arch_defvw{}/b
def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size
def WE = tern{wd, w_d{X}, X} # working copy element type def WE = tern{wd, w_d{X}, X} # working copy element type
@ -407,7 +407,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} } if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} }
cv:= [arch_defvw/width{R}]R ** c cv:= [arch_defvw{}/width{R}]R ** c
if (R==f64 and c != __floor{c}) goto{'end'} if (R==f64 and c != __floor{c}) goto{'end'}
while (1) { while (1) {
@ -494,7 +494,7 @@ def loop_with_unaligned_mask{x, r, nw, l, step} = {
} }
} }
def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
def vl = arch_defvw / 64 def vl = arch_defvw{} / 64
def V = [vl]u64 def V = [vl]u64
{ms, d} := unaligned_spaced_mask_mod{l} {ms, d} := unaligned_spaced_mask_mod{l}
d4:usz = width{V} % l d4:usz = width{V} % l
@ -513,7 +513,7 @@ def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
} }
def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry, apply_carry} = { def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry, apply_carry} = {
assert{l >= 64} assert{l >= 64}
def vl = arch_defvw / 64 def vl = arch_defvw{} / 64
def V = [vl]u64 def V = [vl]u64
def get_m = loose_mask_gen{V, l} def get_m = loose_mask_gen{V, l}
c := V**id # carry, 0 or 1 c := V**id # carry, 0 or 1

View File

@ -20,13 +20,13 @@ def find_first{C, M, F, ...v1} = {
} }
def search{E, x, n:(u64), OP} = { def search{E, x, n:(u64), OP} = {
def bulk = arch_defvw/width{E} def bulk = arch_defvw{}/width{E}
def VT = [bulk]E def VT = [bulk]E
def end = make_branch{ def end = make_branch{
tup{u64, ty_u{VT}}, tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + promote{u64, first_hom{c}}} {i,c} => return{i*bulk + promote{u64, first_hom{c}}}
} }
@for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { @for_mu{bulk, tern{arch_defvw{}>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
eq:= each{OP, x} eq:= each{OP, x}
if (any_hom{M, ...eq}) { if (any_hom{M, ...eq}) {
find_first{ find_first{
@ -52,7 +52,7 @@ fn searchNormalizable{}(x:*f64, len:u64) : u64 = {
fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
def E = f64 def E = f64
def bulk = arch_defvw/width{E} def bulk = arch_defvw{}/width{E}
def VT = [bulk]E def VT = [bulk]E
@for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { @for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
if (any_hom{M, x!=x}) return{1} if (any_hom{M, x!=x}) return{1}
@ -72,7 +72,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
# In-register bit table # In-register bit table
def arch_vec{T} = [arch_defvw/width{T}]T def arch_vec{T} = [arch_defvw{}/width{T}]T
def TI = i8 # Table values def TI = i8 # Table values
def VI = arch_vec{TI} def VI = arch_vec{TI}
def simd_bittab = hasarch{'SSSE3'} or hasarch{'AARCH64'} def simd_bittab = hasarch{'SSSE3'} or hasarch{'AARCH64'}
@ -312,7 +312,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
min1:= undefined{E} min1:= undefined{E}
max1:= undefined{E} max1:= undefined{E}
if (has_simd) { if (has_simd) {
def bulk = arch_defvw/width{E} def bulk = arch_defvw{}/width{E}
def VT = [bulk]E def VT = [bulk]E
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2} def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
def min_a = assoc_accumulator{__min, unr, VT} def min_a = assoc_accumulator{__min, unr, VT}

View File

@ -55,7 +55,7 @@ def vptr{VT=[_]E, ptr} = tup{VT, *E~~ptr}
fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = { fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = {
def cyc = cast_i{TI,cyc0} def cyc = cast_i{TI,cyc0}
if (has_simd) { if (has_simd) {
def bulk = arch_defvw / width{TI} def bulk = arch_defvw{} / width{TI}
def VT = [bulk]TI def VT = [bulk]TI
@for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) { @for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
dst = blend_hom{src, src + VT**cyc, src < VT**0} dst = blend_hom{src, src + VT**cyc, src < VT**0}
@ -161,7 +161,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
fn select_rows_widen{sh}(src:*void, dst:*void, max:ux, n:ux) : ux = { fn select_rows_widen{sh}(src:*void, dst:*void, max:ux, n:ux) : ux = {
if (sh != 0) { if (sh != 0) {
def bulk = (arch_defvw/8) >> sh def bulk = (arch_defvw{}/8) >> sh
def WV = [bulk<<sh]u8 def WV = [bulk<<sh]u8
def V = [bulk]u8 def V = [bulk]u8
if ((n<<sh) > inds_buf_max) return{0} if ((n<<sh) > inds_buf_max) return{0}
@ -257,7 +257,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
} }
1 1
} }
def select_fn{TI, TD} = select_fn{arch_defvw, TI, TD} def select_fn{TI, TD} = select_fn{arch_defvw{}, TI, TD}
export_tab{'si_select_tab', join{table{select_fn, export_tab{'si_select_tab', join{table{select_fn,
tup{i8, i16, i32}, # indices tup{i8, i16, i32}, # indices
@ -269,7 +269,7 @@ export_tab{'si_select_tab', join{table{select_fn,
(if(has_sel) { (if(has_sel) {
fn simd_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { fn simd_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def TI = i8 def TI = i8
def VI = [arch_defvw/8]TI def VI = [arch_defvw{}/8]TI
def VU = ty_u{VI} def VU = ty_u{VI}
w:= *VI ~~ w0 w:= *VI ~~ w0
xlf:= VI**cast_i{TI, xl} xlf:= VI**cast_i{TI, xl}
@ -305,7 +305,7 @@ export_tab{'si_select_tab', join{table{select_fn,
export{'simd_select_bool128', simd_select_bool128} export{'simd_select_bool128', simd_select_bool128}
fn si_select_rows_8bit(inds:*u8, indn:ux, src:*void, dst:*void, rows:ux) : void = { # leaves zeroes in result cells above indn fn si_select_rows_8bit(inds:*u8, indn:ux, src:*void, dst:*void, rows:ux) : void = { # leaves zeroes in result cells above indn
def bulk = arch_defvw / 8 def bulk = arch_defvw{} / 8
def V = [bulk]u8 def V = [bulk]u8
def lut = shuf_u8bits{inds, indn} def lut = shuf_u8bits{inds, indn}
@for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) { @for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {

View File

@ -59,7 +59,7 @@ def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = {
} else { } else {
if (has_simd) { if (has_simd) {
def bufw = bufn * tw def bufw = bufn * tw
def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw; def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw{}} / tw;
def R = [vc]T def R = [vc]T
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc } @unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc }
assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack

View File

@ -147,12 +147,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
} else assert{0} } else assert{0}
} }
export{'si_squeeze_i8', squeeze{arch_defvw, i8, 0, 0}} export{'si_squeeze_i8', squeeze{arch_defvw{}, i8, 0, 0}}
export{'si_squeeze_i16', squeeze{arch_defvw, i16, 0, 0}} export{'si_squeeze_i16', squeeze{arch_defvw{}, i16, 0, 0}}
export{'si_squeeze_i32', squeeze{arch_defvw, i32, 0, 0}} export{'si_squeeze_i32', squeeze{arch_defvw{}, i32, 0, 0}}
export{'si_squeeze_f64', squeeze{arch_defvw, f64, 0, 0}} export{'si_squeeze_f64', squeeze{arch_defvw{}, f64, 0, 0}}
export{'si_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}} export{'si_squeeze_numB', squeeze{arch_defvw{}, f64, 0, 1}}
export{'si_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}} export{'si_squeeze_c16', squeeze{arch_defvw{}, u16, 1, 0}}
export{'si_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}} export{'si_squeeze_c32', squeeze{arch_defvw{}, u32, 1, 0}}
export{'si_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} export{'si_squeeze_chrB', squeeze{arch_defvw{}, u64, 1, 1}}

View File

@ -53,12 +53,12 @@ def store1of2{a:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) {
} }
def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
def store_k{VT, dst, x, l, h} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} def store_k{VT, dst, x, l, h} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
def load_k {VT, src, l, w if width{VT} < arch_defvw} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} def load_k {VT, src, l, w if width{VT} < arch_defvw{}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
def store_k{VT, dst, x, l, h if width{VT} < arch_defvw} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} def store_k{VT, dst, x, l, h if width{VT} < arch_defvw{}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
# Transpose kernel of size kw,kh in size w,h array # Transpose kernel of size kw,kh in size w,h array
def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = { def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = {
def n = (kw*kh*width{T}) / arch_defvw # Number of vectors def n = (kw*kh*width{T}) / arch_defvw{} # Number of vectors
def xvs = load_k{[kw]T, src, n, w} def xvs = load_k{[kw]T, src, n, w}
def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n
def rvs = if (n==kw) xt else halved_pass{n,xt} # To kh by kh for packed square def rvs = if (n==kw) xt else halved_pass{n,xt} # To kh by kh for packed square
@ -89,7 +89,7 @@ def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = {
def kernel = kernel_part{0} def kernel = kernel_part{0}
def kernel_part_h{part_h}{src:*T==i8, dst:*T, kw==16, kh==16, w, h} = { def kernel_part_h{part_h}{src:*T==i8, dst:*T, kw==16, kh==16, w, h} = {
def n = (kw*kh*width{T}) / arch_defvw def n = (kw*kh*width{T}) / arch_defvw{}
def VT = [kw]T def VT = [kw]T
off := part_h - kh/2 off := part_h - kh/2
def xvs = @unroll (i to n) { s := src + i*w; load2{*VT~~s, *VT~~(s+off*w)} } def xvs = @unroll (i to n) { s := src + i*w; load2{*VT~~s, *VT~~(s+off*w)} }
@ -221,13 +221,13 @@ fn interleave{T}(r0:*void, x0:*void, x1:*void, n:u64) : void = {
} }
# SIMD implementations # SIMD implementations
def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} or width{T}>=32 or hasarch{'SSSE3'})} = { def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} or width{T}>=32 or hasarch{'SSSE3'})} = {
def l = arch_defvw / width{T} def l = arch_defvw{} / width{T}
def V = [l]T def V = [l]T
rv0 := *V~~r0; rv1 := *V~~r1; xv := *V~~xp rv0 := *V~~r0; rv1 := *V~~r1; xv := *V~~xp
nv := n / l nv := n / l
def uz = if (not hasarch{'X86_64'}) unzip else ({...xs} => { def uz = if (not hasarch{'X86_64'}) unzip else ({...xs} => {
def reinterpret{V, xs if ktup{xs}} = each{~~{V,.}, xs} def reinterpret{V, xs if ktup{xs}} = each{~~{V,.}, xs}
def q = tr_quads{arch_defvw/128} def q = tr_quads{arch_defvw{}/128}
def k = flat_table{+, iota{2}, 2 * iota{64 / width{T}}} def k = flat_table{+, iota{2}, 2 * iota{64 / width{T}}}
def px = each{shuf{., k}, xs} def px = each{shuf{., k}, xs}
V~~each{q, zip128{...re_el{u64,V}~~px}} V~~each{q, zip128{...re_el{u64,V}~~px}}
@ -244,11 +244,11 @@ def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} o
} }
} }
fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = { fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = {
def l = arch_defvw / width{T} def l = arch_defvw{} / width{T}
def V = [l]T def V = [l]T
xv0 := *V~~x0; xv1 := *V~~x1; rv := *V~~r xv0 := *V~~x0; xv1 := *V~~x1; rv := *V~~r
nv := n / l nv := n / l
def q = tr_quads{arch_defvw/128} def q = tr_quads{arch_defvw{}/128}
@for (x0 in xv0, x1 in xv1 over i to nv) { @for (x0 in xv0, x1 in xv1 over i to nv) {
each{store{rv+2*i, ., .}, iota{2}, zip128{q{x0},q{x1}}} each{store{rv+2*i, ., .}, iota{2}, zip128{q{x0},q{x1}}}
} }
@ -294,7 +294,7 @@ def loop_fixed_width{xp, rp, h, k, st, kern} = {
# Transpose a contiguous kernel of width w*p from x to r with stride rst # Transpose a contiguous kernel of width w*p from x to r with stride rst
def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = { def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = {
def h = arch_defvw / 8 def h = arch_defvw{} / 8
def ih = iota{h}; def iw = iota{w} def ih = iota{h}; def iw = iota{w}
def I = [h]u8; def V = I def I = [h]u8; def V = I
def e = width{T} / 8 def e = width{T} / 8
@ -324,7 +324,7 @@ def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = {
each{perm_store, xs} each{perm_store, xs}
} }
def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = { def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = {
def h = arch_defvw / 8 def h = arch_defvw{} / 8
def V = [h]u8; def U = n_h{V} def V = [h]u8; def U = n_h{V}
def ih = iota{h}%16; def e = width{T} / 8 def ih = iota{h}%16; def e = width{T} / 8
# Permutation to unzip by 4 within each lane # Permutation to unzip by 4 within each lane
@ -343,7 +343,7 @@ def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = {
} }
def transpose_fixed_width{rp:*T, xp:*T, wk, h, hs} = { def transpose_fixed_width{rp:*T, xp:*T, wk, h, hs} = {
def p = if (wk%2) 1 else 2; def w = wk/p def p = if (wk%2) 1 else 2; def w = wk/p
def vl = arch_defvw / (p*width{T}) def vl = arch_defvw{} / (p*width{T})
loop_fixed_width{xp, rp, h, vl, wk, modular_kernel{w,p}{., ., hs}} loop_fixed_width{xp, rp, h, vl, wk, modular_kernel{w,p}{., ., hs}}
} }
def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = { def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = {
@ -353,7 +353,7 @@ def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = {
# Transpose a kernel of height w*p from x with stride xst to contiguous r # Transpose a kernel of height w*p from x with stride xst to contiguous r
# w and h are named for the result, not argument, to match modular_kernel # w and h are named for the result, not argument, to match modular_kernel
def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = { def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = {
def h = arch_defvw / 8 def h = arch_defvw{} / 8
def ih = iota{h}; def iw = iota{w} def ih = iota{h}; def iw = iota{w}
def I = [h]u8; def V = I def I = [h]u8; def V = I
def e = width{T} / 8 def e = width{T} / 8
@ -387,18 +387,18 @@ def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = {
each{store{*V~~rp, ., .}, iw, select{xs, h/e*iw % w}} each{store{*V~~rp, ., .}, iw, select{xs, h/e*iw % w}}
} }
def modular_kernel_rev{2,2}{xp0:*T, rp:*T, xst:(u64)} = { def modular_kernel_rev{2,2}{xp0:*T, rp:*T, xst:(u64)} = {
def V = [arch_defvw / width{T}]T; def U = n_h{V} def V = [arch_defvw{} / width{T}]T; def U = n_h{V}
xl := @unroll (i to 2) { xl := @unroll (i to 2) {
xp := xp0 + i*xst xp := xp0 + i*xst
x := load2{*U~~xp, *U~~(xp + 2*xst)} x := load2{*U~~xp, *U~~(xp + 2*xst)}
if (arch_defvw==128) x else shuf{[8]u32, x, tr_iota{1,2,0}} if (arch_defvw{}==128) x else shuf{[8]u32, x, tr_iota{1,2,0}}
} }
xs := unpack_typed{...unpack_typed{...xl}} xs := unpack_typed{...unpack_typed{...xl}}
each{store{*V~~rp, ., .}, iota{2}, each{~~{V,.},xs}} each{store{*V~~rp, ., .}, iota{2}, each{~~{V,.},xs}}
} }
def transpose_fixed_height{rp:*T, xp:*T, w, ws, hk} = { def transpose_fixed_height{rp:*T, xp:*T, w, ws, hk} = {
def p = if (hk%2) 1 else 2; def h = hk/p def p = if (hk%2) 1 else 2; def h = hk/p
def vl = arch_defvw / (p*width{T}) def vl = arch_defvw{} / (p*width{T})
loop_fixed_height{xp, rp, w, vl, hk, modular_kernel_rev{h,p}{., ., ws}} loop_fixed_height{xp, rp, w, vl, hk, modular_kernel_rev{h,p}{., ., ws}}
} }
def transpose_fixed_height{rp:*T, xp:*T, w, ws, 2} = { def transpose_fixed_height{rp:*T, xp:*T, w, ws, 2} = {
@ -409,7 +409,7 @@ fn transpose{T, {k, kh}}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : voi
rp:*T = *T~~r0 rp:*T = *T~~r0
xp:*T = *T~~x0 xp:*T = *T~~x0
def wT = width{T} def wT = width{T}
def vl = arch_defvw / wT def vl = arch_defvw{} / wT
# Transposes with code dedicated to a particular width or height # Transposes with code dedicated to a particular width or height
def try_fixed_dim{tr, l, lst, nl, l_max} = { def try_fixed_dim{tr, l, lst, nl, l_max} = {
def incl{l} = if (k>4) 1 else l!=4 def incl{l} = if (k>4) 1 else l!=4