diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 8504314f..a47b14de 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -161,7 +161,7 @@ def el_s{V=[_]T} = re_el{w_h{T}, V} # vector definitions -def arch_defvw = if (hasarch{'AVX2'}) 256 else 128 +def arch_defvw{} = if (hasarch{'AVX2'}) 256 else 128 def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'} def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'} def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 754492f3..4df7778d 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -22,7 +22,7 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}} -export_tab{'simd_bitsel', table{arch_defvw}} +export_tab{'simd_bitsel', table{arch_defvw{}}} fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : void = { if (same{E,'!'}) { @@ -30,7 +30,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo } else if (E==u1) { emit{void, 'blendArrScalarBits', rp, zero, cast_i{E, one0}, mask, len} } else { - def bulk = arch_defvw / width{E} + def bulk = arch_defvw{} / width{E} def VT = [bulk]E def one = VT**cast_i{E, one0} @for_mu{bulk,mask_unr{bulk}}(r in tup{'g',*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r{eachx{blend_hom, zero, one, mask}} @@ -43,7 +43,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u fn bitwiden_n_8(src:*void, dst:*void, csz:ux, cam:ux) : void = { assert{cam>0} assert{(csz>1) & (csz<8)} - def bulk = arch_defvw / 8 + def bulk = arch_defvw{} / 8 def V = [bulk]u8 bitalign{tup{2,8,csz}, 8, {s, align} => { @for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) { @@ -59,7 +59,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u fn bitnarrow_8_n(src:*void, dst:*void, csz:ux, cam:ux) : void = { assert{cam>0} assert{(csz>1) & (csz<8)} - def bulk = arch_defvw / 8 + def bulk = arch_defvw{} / 8 def V = [bulk]u8 dstC:= *u8~~dst dstE:= *u8~~dst + cdiv{csz*cam, 8} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index ff191cd9..1c284a14 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -130,9 +130,9 @@ def tableAA{w, F, F_bit} = table{1, F, w, {} => bitAA{F_bit}, {V, u} => aa2bit{V def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}} -export_tab{'simd_eqAS', tableAS{arch_defvw, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}} -export_tab{'simd_neAS', tableAS{arch_defvw, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw, __ne, {a,b}=>a ^ b}} -export_tab{'simd_gtAS', tableAS{arch_defvw, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw, __gt, {a,b}=>a & ~b}} -export_tab{'simd_geAS', tableAS{arch_defvw, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw, __ge, {a,b}=>a | ~b}} -export_tab{'simd_ltAS', tableAS{arch_defvw, __lt}} -export_tab{'simd_leAS', tableAS{arch_defvw, __le}} +export_tab{'simd_eqAS', tableAS{arch_defvw{}, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw{}, __eq, {a,b}=>a ^ ~b}} +export_tab{'simd_neAS', tableAS{arch_defvw{}, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw{}, __ne, {a,b}=>a ^ b}} +export_tab{'simd_gtAS', tableAS{arch_defvw{}, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw{}, __gt, {a,b}=>a & ~b}} +export_tab{'simd_geAS', tableAS{arch_defvw{}, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw{}, __ge, {a,b}=>a | ~b}} +export_tab{'simd_ltAS', tableAS{arch_defvw{}, __lt}} +export_tab{'simd_leAS', tableAS{arch_defvw{}, __le}} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 6ff3ca2e..b8e0e871 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -14,7 +14,7 @@ def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = { fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { - def vw = arch_defvw + def vw = arch_defvw{} assert{l!=0} def bulk = vw/__max{width{X}, width{R}} @@ -68,7 +68,7 @@ fn copy_ubit{R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { # if ((xs&7)==0) { # copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw) # } else { - def vw = arch_defvw + def vw = arch_defvw{} def bulk = vw/width{R} def RV = [bulk]R rp:= *R~~r diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 128cdae1..ef43508f 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -6,7 +6,7 @@ def inc{ptr, ind} = inc{ptr, ind, 1} # Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { - def vbits = arch_defvw + def vbits = arch_defvw{} def vec = vbits/width{T} def TU = ty_u{T} def V = [vec]T @@ -77,7 +77,7 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { } fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = { - def vl = arch_defvw/16 + def vl = arch_defvw{}/16 def V = [vl]u16 def bot = tail{15} on:usz = 0 @@ -147,7 +147,7 @@ def count_adapt_runs{x0, tab, n} = { } } def mark_run_ends{x:*T, m:(ux)} = { - def vec = arch_defvw/width{T} + def vec = arch_defvw{}/width{T} def V = [vec]T @unroll (j to width{ux} / vec) { def jv = j*vec @@ -175,7 +175,7 @@ fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = count_adapt_runs{x, tab, n} # ov is sorted but may not be unique, and oc contains multiples of 128 # Return the shared length of ov and oc fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = { - def V = [arch_defvw/width{T}]T + def V = [arch_defvw{}/width{T}]T def block = 128 i:usz = 0 on:usz = 0 diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index a53632f0..959dfcb5 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -164,7 +164,7 @@ fn arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = { } def arithAA{mode, F, W, X, R} = { - def vw = arch_defvw + def vw = arch_defvw{} if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R} else arithAAu{vw, mode, F, W, X, R} } @@ -192,7 +192,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 else len } -def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R} +def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw{}, mode, F, swap, W, X, R} fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { assert{vw>=64} @@ -203,6 +203,6 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { @for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull} } -export{'simd_andBytes', andBytes{arch_defvw}} +export{'simd_andBytes', andBytes{arch_defvw{}}} export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}} include 'gen/arDefs' diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 9c9e4e85..7694cd9a 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -12,7 +12,7 @@ def swap{w,x} = { # width{W} ≤ width{X} fn equal{W, X}(w:*void, x:*void, l:ux, d:u64) : u1 = { - def vw = arch_defvw + def vw = arch_defvw{} def bulk = vw / width{X} if (W!=X) if (d!=0) swap{w,x} assert{l>0} @@ -83,7 +83,7 @@ def any_ne_reflexive_qnan{M, a:V=[k](f64), b:V} = { # (a==b) | (isQNaN{a} & isQN fn equal_reflexive{}(w:*void, x:*void, l:ux, d:u64) : u1 = { def w = *f64~~w def x = *f64~~x - def bulk = arch_defvw / width{f64} + def bulk = arch_defvw{} / width{f64} def unr = if (hasarch{'AARCH64'}) 2 else 1 def V = [bulk]f64 @for_mu{bulk, unr}(w in tup{V,w}, x in tup{V,x}, M in 'm' over i to l) { diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 270ee768..63bd9941 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -29,7 +29,7 @@ fn fold_idem{T, op}(x:*T, len:u64) : T = { a } fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = { - def bulk = arch_defvw/width{T} + def bulk = arch_defvw{}/width{T} def V = [bulk]T xv:= *V ~~ x r:V = [bulk]f64**0 @@ -62,7 +62,7 @@ fn fold_assoc_0{T, op}(x:*T, len:u64) : T = { a } fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { - def bulk = arch_defvw/width{T} + def bulk = arch_defvw{}/width{T} def V = [bulk]T xv:= *V ~~ x e:= len / bulk @@ -75,7 +75,7 @@ export{'si_sum_f64', fold_assoc_0{f64,+}} def extract_column_pow2{T, x0, r0, nv, k} = { - def V = [arch_defvw / width{T}]T + def V = [arch_defvw{} / width{T}]T xv := *V~~x0 @for (r in *V~~r0 over i to nv) { xs := each{load{xv, .}, iota{k}} @@ -234,7 +234,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { # Maximum of n result values, return actual number written fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = { n <<= el - def vl = arch_defvw / 8 + def vl = arch_defvw{} / 8 def thr = __min{vl+2, 20} if ((not has_simd) or n < vl or l > usz~~thr>>el or l<= thr) return{0} nv := n / vl @@ -529,7 +529,7 @@ fn sum_small{T if has_simd}(xv:*void, ia:usz) : i64 = { def A = if (width{T}<=16) i32 else i64 xv:= *T~~xv def unr = 4 - def bulk = arch_defvw / width{T} + def bulk = arch_defvw{} / width{T} def V = [bulk]T def acc = sum_accumulator{A, unr, V} diff --git a/src/singeli/src/group.singeli b/src/singeli/src/group.singeli index ce7575f1..43fce528 100644 --- a/src/singeli/src/group.singeli +++ b/src/singeli/src/group.singeli @@ -21,7 +21,7 @@ def group_statistics{T} = { fn group_statistics{T}(w:*void, xn:usz, outs:each{__pnt,types}) : void = { def w = *T~~w def accs = if (has_simd) { - def bulk = arch_defvw/width{T} + def bulk = arch_defvw{}/width{T} def V = [bulk]T def VU = ty_u{V} diff --git a/src/singeli/src/lut.singeli b/src/singeli/src/lut.singeli index 3e0d4d0f..34baa5cc 100644 --- a/src/singeli/src/lut.singeli +++ b/src/singeli/src/lut.singeli @@ -36,7 +36,7 @@ local def loader{G} = { } def unzip_load{E, n, TG} = each{merge, unzip_load{E, n/2, TG}, unzip_load{E, n/2, TG{'offset', E, n}}} -def unzip_load{E, n, TG if width{E}*n <= arch_defvw} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}} +def unzip_load{E, n, TG if width{E}*n <= arch_defvw{}} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}} def widen_tup{(u32), is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is widen{[8]u32, is}, diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 9fe168e1..becdbcdf 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -6,7 +6,7 @@ include './mask' fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { - def bulk = arch_defvw/width{T} + def bulk = arch_defvw{}/width{T} def VT = [bulk]T @for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk} diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index d2a7bdd9..37b45956 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -82,7 +82,7 @@ def fact_inds = slice{iota{fact_size+1},8} def fact_tab = makefact{basic_rep, fact_inds} factors:*u8 = fact_tab -def sdtype = [arch_defvw/8]i8 # shuf data type +def sdtype = [arch_defvw{}/8]i8 # shuf data type def get_shufs{step, wv, len} = { def i = iota{len*step} split{step, (i - i%wv)/wv} @@ -240,7 +240,7 @@ fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = { assert{kv > 0} - def V = [arch_defvw/width{T}]T + def V = [arch_defvw{}/width{T}]T @for (x over n) { v := V**x @loop (j to kv) store{*V~~r, j, v} @@ -264,7 +264,7 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { } } def wT = width{T} - def vn = arch_defvw/wT + def vn = arch_defvw{}/wT def V = [vn]T def max_shuffle = 2*vn if (wv <= max_shuffle) { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 4209c299..dbd66b09 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -17,7 +17,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = { } def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = { - def step = arch_defvw/width{T} + def step = arch_defvw{}/width{T} def V = [step]T p:= V**init xv:= *V~~x @@ -42,7 +42,7 @@ def scan_idem = scan_scal fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = { def {scan, last} = get_scan_last{op, make_scan_idem{T, op}} def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) } - def step = arch_defvw/width{T} + def step = arch_defvw{}/width{T} def V = [step]T p:= V**init xv:= *V ~~ x @@ -183,7 +183,7 @@ fn scan_stride_assoc{op, T, Ret, check_over}(xv:*void, rv:*void, ia:usz, l:usz) } } } else if (same{op, +} and T<=i32 and has_simd and (has_shuf or l>=vl)) { - def vl = arch_defvw/wT; def V = [vl]T + def vl = arch_defvw{}/wT; def V = [vl]T @for_masked{vl} (x in tup{V, x+l}, r in tup{V, r+l}, p in tup{V, r}, M in 'm' over ia-l) { r = op_chk{M, p, x} } @@ -225,7 +225,7 @@ fn scan_neq{}(c:u64, x:*u64, r:*u64, nw:usz) : void = { } } fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:usz) : void = { - def vl = arch_defvw / 64 + def vl = arch_defvw{} / 64 def V = [vl]u64 c := V**c0 @for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) { @@ -362,9 +362,9 @@ def addChk{a:T, b:T==f64} = tup{0, a+b} def widenFull{E, xs} = { merge{...each{{x:X=[k]T} => { def tb = width{E} * k - if (tb<=arch_defvw) tup{widen{[k]E, x}} + if (tb<=arch_defvw{}) tup{widen{[k]E, x}} else if (1) { - assert{tb == 2*arch_defvw} + assert{tb == 2*arch_defvw{}} tup{ widen{[k/2]E, half{x,0}}, widen{[k/2]E, half{x,1}} @@ -395,7 +395,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { # Sum as many vector registers as possible; modifies c and i def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = { def b = __max{width{R}/2, width{X}} - def bulk = arch_defvw/b + def bulk = arch_defvw{}/b def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size def WE = tern{wd, w_d{X}, X} # working copy element type @@ -407,7 +407,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = { if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} } - cv:= [arch_defvw/width{R}]R ** c + cv:= [arch_defvw{}/width{R}]R ** c if (R==f64 and c != __floor{c}) goto{'end'} while (1) { @@ -494,7 +494,7 @@ def loop_with_unaligned_mask{x, r, nw, l, step} = { } } def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { - def vl = arch_defvw / 64 + def vl = arch_defvw{} / 64 def V = [vl]u64 {ms, d} := unaligned_spaced_mask_mod{l} d4:usz = width{V} % l @@ -513,7 +513,7 @@ def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { } def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry, apply_carry} = { assert{l >= 64} - def vl = arch_defvw / 64 + def vl = arch_defvw{} / 64 def V = [vl]u64 def get_m = loose_mask_gen{V, l} c := V**id # carry, 0 or 1 diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index d6a0d7c2..08b25ead 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -20,13 +20,13 @@ def find_first{C, M, F, ...v1} = { } def search{E, x, n:(u64), OP} = { - def bulk = arch_defvw/width{E} + def bulk = arch_defvw{}/width{E} def VT = [bulk]E def end = make_branch{ tup{u64, ty_u{VT}}, {i,c} => return{i*bulk + promote{u64, first_hom{c}}} } - @for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { + @for_mu{bulk, tern{arch_defvw{}>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { eq:= each{OP, x} if (any_hom{M, ...eq}) { find_first{ @@ -52,7 +52,7 @@ fn searchNormalizable{}(x:*f64, len:u64) : u64 = { fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { def E = f64 - def bulk = arch_defvw/width{E} + def bulk = arch_defvw{}/width{E} def VT = [bulk]E @for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { if (any_hom{M, x!=x}) return{1} @@ -72,7 +72,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { # In-register bit table -def arch_vec{T} = [arch_defvw/width{T}]T +def arch_vec{T} = [arch_defvw{}/width{T}]T def TI = i8 # Table values def VI = arch_vec{TI} def simd_bittab = hasarch{'SSSE3'} or hasarch{'AARCH64'} @@ -312,7 +312,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { min1:= undefined{E} max1:= undefined{E} if (has_simd) { - def bulk = arch_defvw/width{E} + def bulk = arch_defvw{}/width{E} def VT = [bulk]E def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2} def min_a = assoc_accumulator{__min, unr, VT} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index d9ba939e..dca531e6 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -55,7 +55,7 @@ def vptr{VT=[_]E, ptr} = tup{VT, *E~~ptr} fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = { def cyc = cast_i{TI,cyc0} if (has_simd) { - def bulk = arch_defvw / width{TI} + def bulk = arch_defvw{} / width{TI} def VT = [bulk]TI @for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) { dst = blend_hom{src, src + VT**cyc, src < VT**0} @@ -161,7 +161,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max} fn select_rows_widen{sh}(src:*void, dst:*void, max:ux, n:ux) : ux = { if (sh != 0) { - def bulk = (arch_defvw/8) >> sh + def bulk = (arch_defvw{}/8) >> sh def WV = [bulk< inds_buf_max) return{0} @@ -257,7 +257,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { } 1 } -def select_fn{TI, TD} = select_fn{arch_defvw, TI, TD} +def select_fn{TI, TD} = select_fn{arch_defvw{}, TI, TD} export_tab{'si_select_tab', join{table{select_fn, tup{i8, i16, i32}, # indices @@ -269,7 +269,7 @@ export_tab{'si_select_tab', join{table{select_fn, (if(has_sel) { fn simd_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { def TI = i8 - def VI = [arch_defvw/8]TI + def VI = [arch_defvw{}/8]TI def VU = ty_u{VI} w:= *VI ~~ w0 xlf:= VI**cast_i{TI, xl} @@ -305,7 +305,7 @@ export_tab{'si_select_tab', join{table{select_fn, export{'simd_select_bool128', simd_select_bool128} fn si_select_rows_8bit(inds:*u8, indn:ux, src:*void, dst:*void, rows:ux) : void = { # leaves zeroes in result cells above indn - def bulk = arch_defvw / 8 + def bulk = arch_defvw{} / 8 def V = [bulk]u8 def lut = shuf_u8bits{inds, indn} @for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) { diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 24df73b0..3b0ac17e 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -59,7 +59,7 @@ def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = { } else { if (has_simd) { def bufw = bufn * tw - def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw; + def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw{}} / tw; def R = [vc]T @unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc } assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 027a0e18..5cef8c57 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -147,12 +147,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { } else assert{0} } -export{'si_squeeze_i8', squeeze{arch_defvw, i8, 0, 0}} -export{'si_squeeze_i16', squeeze{arch_defvw, i16, 0, 0}} -export{'si_squeeze_i32', squeeze{arch_defvw, i32, 0, 0}} -export{'si_squeeze_f64', squeeze{arch_defvw, f64, 0, 0}} -export{'si_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}} +export{'si_squeeze_i8', squeeze{arch_defvw{}, i8, 0, 0}} +export{'si_squeeze_i16', squeeze{arch_defvw{}, i16, 0, 0}} +export{'si_squeeze_i32', squeeze{arch_defvw{}, i32, 0, 0}} +export{'si_squeeze_f64', squeeze{arch_defvw{}, f64, 0, 0}} +export{'si_squeeze_numB', squeeze{arch_defvw{}, f64, 0, 1}} -export{'si_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}} -export{'si_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}} -export{'si_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} +export{'si_squeeze_c16', squeeze{arch_defvw{}, u16, 1, 0}} +export{'si_squeeze_c32', squeeze{arch_defvw{}, u32, 1, 0}} +export{'si_squeeze_chrB', squeeze{arch_defvw{}, u64, 1, 1}} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 5e180bae..da38f62b 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -53,12 +53,12 @@ def store1of2{a:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) { } def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} def store_k{VT, dst, x, l, h} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} -def load_k {VT, src, l, w if width{VT} < arch_defvw} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} -def store_k{VT, dst, x, l, h if width{VT} < arch_defvw} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} +def load_k {VT, src, l, w if width{VT} < arch_defvw{}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} +def store_k{VT, dst, x, l, h if width{VT} < arch_defvw{}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} # Transpose kernel of size kw,kh in size w,h array def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = { - def n = (kw*kh*width{T}) / arch_defvw # Number of vectors + def n = (kw*kh*width{T}) / arch_defvw{} # Number of vectors def xvs = load_k{[kw]T, src, n, w} def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n def rvs = if (n==kw) xt else halved_pass{n,xt} # To kh by kh for packed square @@ -89,7 +89,7 @@ def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = { def kernel = kernel_part{0} def kernel_part_h{part_h}{src:*T==i8, dst:*T, kw==16, kh==16, w, h} = { - def n = (kw*kh*width{T}) / arch_defvw + def n = (kw*kh*width{T}) / arch_defvw{} def VT = [kw]T off := part_h - kh/2 def xvs = @unroll (i to n) { s := src + i*w; load2{*VT~~s, *VT~~(s+off*w)} } @@ -221,13 +221,13 @@ fn interleave{T}(r0:*void, x0:*void, x1:*void, n:u64) : void = { } # SIMD implementations def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} or width{T}>=32 or hasarch{'SSSE3'})} = { - def l = arch_defvw / width{T} + def l = arch_defvw{} / width{T} def V = [l]T rv0 := *V~~r0; rv1 := *V~~r1; xv := *V~~xp nv := n / l def uz = if (not hasarch{'X86_64'}) unzip else ({...xs} => { def reinterpret{V, xs if ktup{xs}} = each{~~{V,.}, xs} - def q = tr_quads{arch_defvw/128} + def q = tr_quads{arch_defvw{}/128} def k = flat_table{+, iota{2}, 2 * iota{64 / width{T}}} def px = each{shuf{., k}, xs} V~~each{q, zip128{...re_el{u64,V}~~px}} @@ -244,11 +244,11 @@ def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} o } } fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = { - def l = arch_defvw / width{T} + def l = arch_defvw{} / width{T} def V = [l]T xv0 := *V~~x0; xv1 := *V~~x1; rv := *V~~r nv := n / l - def q = tr_quads{arch_defvw/128} + def q = tr_quads{arch_defvw{}/128} @for (x0 in xv0, x1 in xv1 over i to nv) { each{store{rv+2*i, ., .}, iota{2}, zip128{q{x0},q{x1}}} } @@ -294,7 +294,7 @@ def loop_fixed_width{xp, rp, h, k, st, kern} = { # Transpose a contiguous kernel of width w*p from x to r with stride rst def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = { - def h = arch_defvw / 8 + def h = arch_defvw{} / 8 def ih = iota{h}; def iw = iota{w} def I = [h]u8; def V = I def e = width{T} / 8 @@ -324,7 +324,7 @@ def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = { each{perm_store, xs} } def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = { - def h = arch_defvw / 8 + def h = arch_defvw{} / 8 def V = [h]u8; def U = n_h{V} def ih = iota{h}%16; def e = width{T} / 8 # Permutation to unzip by 4 within each lane @@ -343,7 +343,7 @@ def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = { } def transpose_fixed_width{rp:*T, xp:*T, wk, h, hs} = { def p = if (wk%2) 1 else 2; def w = wk/p - def vl = arch_defvw / (p*width{T}) + def vl = arch_defvw{} / (p*width{T}) loop_fixed_width{xp, rp, h, vl, wk, modular_kernel{w,p}{., ., hs}} } def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = { @@ -353,7 +353,7 @@ def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = { # Transpose a kernel of height w*p from x with stride xst to contiguous r # w and h are named for the result, not argument, to match modular_kernel def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = { - def h = arch_defvw / 8 + def h = arch_defvw{} / 8 def ih = iota{h}; def iw = iota{w} def I = [h]u8; def V = I def e = width{T} / 8 @@ -387,18 +387,18 @@ def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = { each{store{*V~~rp, ., .}, iw, select{xs, h/e*iw % w}} } def modular_kernel_rev{2,2}{xp0:*T, rp:*T, xst:(u64)} = { - def V = [arch_defvw / width{T}]T; def U = n_h{V} + def V = [arch_defvw{} / width{T}]T; def U = n_h{V} xl := @unroll (i to 2) { xp := xp0 + i*xst x := load2{*U~~xp, *U~~(xp + 2*xst)} - if (arch_defvw==128) x else shuf{[8]u32, x, tr_iota{1,2,0}} + if (arch_defvw{}==128) x else shuf{[8]u32, x, tr_iota{1,2,0}} } xs := unpack_typed{...unpack_typed{...xl}} each{store{*V~~rp, ., .}, iota{2}, each{~~{V,.},xs}} } def transpose_fixed_height{rp:*T, xp:*T, w, ws, hk} = { def p = if (hk%2) 1 else 2; def h = hk/p - def vl = arch_defvw / (p*width{T}) + def vl = arch_defvw{} / (p*width{T}) loop_fixed_height{xp, rp, w, vl, hk, modular_kernel_rev{h,p}{., ., ws}} } def transpose_fixed_height{rp:*T, xp:*T, w, ws, 2} = { @@ -409,7 +409,7 @@ fn transpose{T, {k, kh}}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : voi rp:*T = *T~~r0 xp:*T = *T~~x0 def wT = width{T} - def vl = arch_defvw / wT + def vl = arch_defvw{} / wT # Transposes with code dedicated to a particular width or height def try_fixed_dim{tr, l, lst, nl, l_max} = { def incl{l} = if (k>4) 1 else l!=4