From a439a0a430e7eddcf1b413db1aecc3793146c84b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 6 Aug 2024 10:02:27 -0400 Subject: [PATCH] =?UTF-8?q?SSSE3=20k/bool=20for=202=E2=89=A4k=E2=89=A48,?= =?UTF-8?q?=20k=E2=89=A06?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/replicate.singeli | 180 +++++++++++++++++++++++------- 1 file changed, 142 insertions(+), 38 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 0bf0712a..a14aa44c 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -291,46 +291,12 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - if (hasarch{'SSSE3'} and wv==3) { - def vl = 16; def V = [vl]u8 - nv := cdiv{rlen, width{V}} - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { - xv := load{*V~~x, i}; ++i - # 01234567 to 05316427 on each byte - def bv{bs} = fold{flat_table{+,...}, reverse{each{tup{0,.}, 1< make{V, bv{is}}, split{4, 3*iota{8} % 8}} - m4 := V**0xf - xv = sel{V, t0, xv & m4} | sel{V, t4, V~~([8]u16~~xv>>4) & m4} - # Overhang from previous 64-bit elements - def os = 8-3 # right shift - def ix = __floor{64*slice{iota{3},1} / 3} # bits that overhang within a word - def ib = __floor{ix / 8} # byte index - def io = 8*ib + 3*ix%8 # where they are in xv - def wi = split{2, tup{255, ...ib, 255, ...8+ib}} - def W = [2]u64 - xo := V~~((W~~xv & W**fold{|, 1<> os) - xo += xo > V**0 - # Permute and mask bytes - def step{jj, oi, ind, mask} = { - b := W~~(sel{V, xv, ind} & mask) - r := V~~((b<<3) - b) - o := sel{V, xo, make{V, flat_table{max, oi, 255*(0 32) return{0} + if (hasarch{'SSSE3'} and wv<=8 and wv!=6) { + rep_const_bool_ssse3{wv, x, r, rlen} + return{1} + } if (not has_pdep and wv <= 8) return{0} m:u64 = spaced_mask_of{wv} xw:u64 = 0 @@ -372,4 +338,142 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } 1 } +def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6 + oper // ({a,b}=>floor{a/b}) infix left 40 + def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1<>4)}, .} + store{rv, j, V~~getr{0}}; ++j; if (j==nv) goto{end} + store{rv, j, V~~getr{1}}; ++j + } + setlabel{end} + } + def run{2} = { + # Expander for half byte + tabr := mkV{bv{2*iota{4}} * 2b11} + m4 := V**0xf + def exh{x} = selV{tabr, x & m4} + run24{*V~~x, {xv}=>xv, exh} + } + def run{4} = { + dup := mkV{iV//2} # Double each byte + # Expander for two bits in either bottom or next-to-bottom position + tabr := mkV{bv{tup{0,4,0,4}} * 2b1111} + m2 := mkV{2b11 << (2*(iV%2))} + def exh{x} = re_el{u16, V}~~selV{tabr, x & m2} + run24{*u64~~x, selV{., dup}, exh} + } + + def run{8} = { + @for (r in *V~~r over i to nv) { + xv := load{*V~~(*u16~~x + i)} + xe := selV{xv, mkV{iV//8}} + r = (xe & mkV{1 << (iV % 8)}) > V**0 + } + } + + # For odd numbers: + # - permute each byte sending bit i to position k*i % 8 + # - replicate each byte by k, making position k*i contain bit i + # - mask out those bits and spread over [ k*i, k*(i+1) ) + # - ...except where it crosses words; handle this overhang separately + # 3 is handled separately and unrolled; 5 and 7 share a loop + def run{3} = { + i:usz = 0; j:usz = 0 + rv := *V~~r + def end = makelabel{}; while (j < nv) { + xv := load{*V~~x, i}; ++i + # 01234567 to 05316427 on each byte + {t0, t4} := each{{is} => mkV{bv{is}}, split{4, 3*iota{8} % 8}} + m4 := V**0xf + xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + # Overhang from previous 64-bit elements + def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word + def ib = ix // 8 # byte index + def io = 8*ib + 3*ix%8 # where they are in xv + def wi = split{2, tup{255, ...ib, 255, ...8+ib}} + xo := V~~((W~~xv & W**fold{|, 1<> (8-3)) + xo += xo > V**0 + # Permute and mask bytes + def step{jj, oi, ind, mask} = { + b := W~~(selV{xv, ind} & mask) + r := V~~((b<<3) - b) + o := selV{xo, mkV{flat_table{max, oi, 255*(0 mkV{bv{is}}, split{4, k*iota{8} % 8}} + # Overhang from previous 64-bit elements + def ix = 64*slice{iota{k},1} // k # bits that overhang within a word + def ib = ix // 8 # byte index + def io = 8*ib + k*ix%8 # where they are in xv + def wi = tup{255, ...ib, 255, ...8+ib} + xom = W**fold{|, 1<=vl + ind_up = mkV{iu - k*ia} + ind_inc = mkV{vl//k + ia} + mask_sh = width{V} % k + } + if (wv == 5) set_consts{5} else set_consts{7} + xv:V = V**0; xo:=xv; ind:=xv; mask:=xv # state + i:usz = 0; q:usz = 1 + @for (r in *V~~r over j to nv) { + --q; if (q == 0) { q = wv + # Load and permute bytes + xv = load{*V~~x, i}; ++i + m4 := V**0xf + xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + # Bytes for overhang + xo = V~~((W~~xv & xom) >> (8 - wv)) + xo += xo > V**0 + xo = selV{xo, xse} + # Initialize state vectors + ind = ind0 + mask = mask0 + } else { + # Update state vectors + xo = shr{V, xo, 1} + ind = selV{ind, ind_up} + ind_inc + mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + } + b := W~~(selV{xv, ind} & mask) + r = V~~((b<