diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 3b4aa0f7..b14c29b3 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -360,6 +360,26 @@ def tr_iota{...bs} = { } def tr_iota{{...bs}} = tr_iota{...bs} +def get_boolvec_writer{V, r, rlen} = { + def vwords = width{V}/64 + nw := cdiv{rlen, 64} + rv := *V~~r + re := rv + nw / vwords + last_res:V = V**0 + def end = makelabel{} + def output{v:(V)} = { + last_res = v + if (rv==re) goto{end} + store{rv, 0, v}; ++rv + } + def flush{} = { + setlabel{end} + q := nw & (vwords-1) + if (q != 0) homMaskStoreF{rv, V~~maskOf{re_el{u64,V}, q}, last_res} + } + tup{output, flush} +} + def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 oper // ({a,b}=>floor{a/b}) infix left 40 def avx2 = hasarch{'AVX2'} @@ -369,20 +389,17 @@ def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 def mkV = make{V, .} def selH = sel{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} - nv := cdiv{rlen, width{V}} def id{xv} = xv + def {output, flush} = get_boolvec_writer{V, r, rlen} def run24{x, proc_xv, exh} = { - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { + i:usz = 0; while (1) { xv := proc_xv{load{*V~~(x+i)}}; ++i # Store 1 or 2 result vectors def getr = zip128{exh{xv}, exh{V~~(re_el{u16,V}~~xv>>4)}, .} - store{rv, j, V~~getr{0}}; ++j; if (j==nv) goto{end} - store{rv, j, V~~getr{1}}; ++j + output{V~~getr{0}} + output{V~~getr{1}} } - setlabel{end} } if (wv == 2) { def init = if (avx2) shuf{[4]u64, ., 4b3120} else id @@ -400,13 +417,14 @@ def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 def exh{x} = re_el{u16, V}~~tabr{x & m2} run24{*(if (avx2) [2]u64 else u64)~~x, init, exh} } else { # wv == 8 - @for (r in *V~~r over i to nv) { - xh := load{*[16]u8~~(*ty_u{vl}~~x + i)} + i:usz = 0; while (1) { + xh := load{*[16]u8~~(*ty_u{vl}~~x + i)}; ++i xv := if (avx2) pair{xh, xh} else xh xe := selH{xv, mkV{iV // 8}} - r = (xe & mkV{1 << (iV % 8)}) > V**0 + output{(xe & mkV{1 << (iV % 8)}) > V**0} } } + flush{} } # For odd numbers: @@ -420,8 +438,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 def iV = iota{vl} def mkV = make{V, .}; def selV = sel{V, ., .} def W = [2]u64 - - nv := cdiv{rlen, width{V}} + def {output, flush} = get_boolvec_writer{V, r, rlen} # Within-byte transformation def get_ttab{k} = each{{is} => mkV{tr_iota{is}}, split{4, k*iota{8} % 8}} @@ -436,9 +453,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 # Cases are 3; 5 7; 9 11 13 15 if (wv < 4) { # 3: dedicated loop - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { + i:usz = 0; while (1) { # 01234567 to 05316427 on each byte xv := get_perm_x{i}; ++i # Overhang from previous 64-bit elements @@ -453,7 +468,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 b := W~~(selV{xv, ind} & mask) r := V~~((b<<3) - b) o := selV{xo, mkV{flat_table{max, oi, 255*(0> mask_sh)) } b := W~~(selV{xv, ind} & mask) - r = V~~((b<>(64-wv) ro:= [4]u32~~vshl{po, o, 1} - r |= V~~(ro + (ro > [4]u32**0)) + output{rv | V~~(ro + (ro > [4]u32**0))} } } + flush{} } export{'si_constrep_bool', rep_const_bool{}}