diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli
index a14aa44c..5a29cf96 100644
--- a/src/singeli/src/replicate.singeli
+++ b/src/singeli/src/replicate.singeli
@@ -293,9 +293,22 @@ exportT{'si_constrep', each{rep_const, dat_types}}
fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = {
def has_pdep = hasarch{'BMI2'}
if (wv > 32) return{0}
- if (hasarch{'SSSE3'} and wv<=8 and wv!=6) {
- rep_const_bool_ssse3{wv, x, r, rlen}
- return{1}
+ if (hasarch{'SSSE3'}) {
+ if (wv&1 == 0) {
+ p := ctz{wv | 8} # Power of two for second replicate
+ if (wv>>p == 1) {
+ rep_const_bool_ssse3_div8{wv, x, r, rlen}
+ } else {
+ tlen := rlen>>p
+ t := r + cdiv{rlen, 64} - cdiv{tlen, 64}
+ rep_const_bool{}(wv >>p, x, t, tlen)
+ rep_const_bool{}(usz~~1<
floor{a/b}) infix left 40
def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1<xv, exh}
- }
- def run{4} = {
- dup := mkV{iV//2} # Double each byte
+ } else if (wv == 4) {
+ # Double each byte
+ dup := mkV{iV//2}
# Expander for two bits in either bottom or next-to-bottom position
tabr := mkV{bv{tup{0,4,0,4}} * 2b1111}
m2 := mkV{2b11 << (2*(iV%2))}
def exh{x} = re_el{u16, V}~~selV{tabr, x & m2}
run24{*u64~~x, selV{., dup}, exh}
- }
-
- def run{8} = {
+ } else { # wv == 8
@for (r in *V~~r over i to nv) {
xv := load{*V~~(*u16~~x + i)}
xe := selV{xv, mkV{iV//8}}
r = (xe & mkV{1 << (iV % 8)}) > V**0
}
}
+}
+# For odd numbers:
+# - permute each byte sending bit i to position k*i % 8
+# - replicate each byte by k, making position k*i contain bit i
+# - mask out those bits and spread over [ k*i, k*(i+1) )
+# - ...except where it crosses words; handle this overhang separately
+def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15
+ oper // ({a,b}=>floor{a/b}) infix left 40
+ def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1< mkV{bv{is}}, split{4, k*iota{8} % 8}}
+ ttab:*V = join{each{get_ttab, 2*iota{4} + 1}}
+ {t0, t4} := each{load{ttab + (wv & 6), .}, iota{2}}
+ m4 := V**0xf
+ def get_perm_x{i} = {
+ xv := load{*V~~x, i}
+ selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4}
+ }
+
+ # Cases are 3; 5 7; 9 11 13 15
+ if (wv < 4) {
+ # 3: dedicated loop
i:usz = 0; j:usz = 0
rv := *V~~r
def end = makelabel{}; while (j < nv) {
- xv := load{*V~~x, i}; ++i
# 01234567 to 05316427 on each byte
- {t0, t4} := each{{is} => mkV{bv{is}}, split{4, 3*iota{8} % 8}}
- m4 := V**0xf
- xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4}
+ xv := get_perm_x{i}; ++i
# Overhang from previous 64-bit elements
def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word
def ib = ix // 8 # byte index
@@ -420,13 +449,11 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6
}
}
setlabel{end}
- }
- def run{{5,7}} = {
- {t0, t4, xom, xse, ind0, mask0, ind_up, ind_inc, mask_sh} := undef{tup{
- V, V, W, V, V, V, V, V, usz }}
+ } else if (wv < 8) {
+ # 5, 7: precompute constants, then shared loop
+ {xom, xse, ind0, mask0, ind_up, ind_inc, mask_sh} := undef{tup{
+ W, V, V, V, V, V, usz }}
def set_consts{k} = {
- # Within-byte transformation
- tup{t0, t4} = each{{is} => mkV{bv{is}}, split{4, k*iota{8} % 8}}
# Overhang from previous 64-bit elements
def ix = 64*slice{iota{k},1} // k # bits that overhang within a word
def ib = ix // 8 # byte index
@@ -448,9 +475,7 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6
@for (r in *V~~r over j to nv) {
--q; if (q == 0) { q = wv
# Load and permute bytes
- xv = load{*V~~x, i}; ++i
- m4 := V**0xf
- xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4}
+ xv = get_perm_x{i}; ++i
# Bytes for overhang
xo = V~~((W~~xv & xom) >> (8 - wv))
xo += xo > V**0
@@ -468,12 +493,36 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6
r = V~~((b<= k}
+ {m, d} := unaligned_spaced_mask_mod{wv}
+ mask0:= V~~make{W, m, m>>d|m<<(wv-d)}
+ iu:= iota{V} + V**cast_i{u8,vl-wv}; ia:= iu>=V**vl
+ ind_up := iu - (wV&ia)
+ ind_inc:= V**(wv <= vl) - ia
+ mask_sh:= d+d; if (mask_sh >= wv) mask_sh-= wv
+ # State
+ xv:V = V**0; o:=W**0; ind:=xv; mask:=xv
+ i:usz = 0; q:usz = 1
+ @for (r in *V~~r over j to nv) {
+ --q; if (q == 0) { q = wv
+ xv = get_perm_x{i}; ++i
+ ind = ind0
+ mask = mask0
+ } else {
+ ind = selV{ind, ind_up} + ind_inc
+ mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh))
+ }
+ # Handle overhang here; won't fit in a single vector
+ b := W~~(selV{xv, ind} & mask)
+ r = V~~((b<>(64-wv)
+ ro:= [4]u32~~vshl{po, o, 1}
+ r |= V~~(ro + (ro > [4]u32**0))
+ }
}
-
- if (wv==2) run{2}
- else if (wv==3) run{3}
- else if (wv==4) run{4}
- else if (wv==8) run{8}
- else run{tup{5,7}}
}
export{'si_constrep_bool', rep_const_bool{}}