From a4b6d8d827f1c2a6b3411c296009b1f0796c52e3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 5 Aug 2024 15:27:36 -0400 Subject: [PATCH] SSSE3 3/bool, faster than BMI2 --- src/singeli/src/replicate.singeli | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index fc596203..0bf0712a 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -291,6 +291,44 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { + if (hasarch{'SSSE3'} and wv==3) { + def vl = 16; def V = [vl]u8 + nv := cdiv{rlen, width{V}} + i:usz = 0; j:usz = 0 + rv := *V~~r + def end = makelabel{}; while (j < nv) { + xv := load{*V~~x, i}; ++i + # 01234567 to 05316427 on each byte + def bv{bs} = fold{flat_table{+,...}, reverse{each{tup{0,.}, 1< make{V, bv{is}}, split{4, 3*iota{8} % 8}} + m4 := V**0xf + xv = sel{V, t0, xv & m4} | sel{V, t4, V~~([8]u16~~xv>>4) & m4} + # Overhang from previous 64-bit elements + def os = 8-3 # right shift + def ix = __floor{64*slice{iota{3},1} / 3} # bits that overhang within a word + def ib = __floor{ix / 8} # byte index + def io = 8*ib + 3*ix%8 # where they are in xv + def wi = split{2, tup{255, ...ib, 255, ...8+ib}} + def W = [2]u64 + xo := V~~((W~~xv & W**fold{|, 1<> os) + xo += xo > V**0 + # Permute and mask bytes + def step{jj, oi, ind, mask} = { + b := W~~(sel{V, xv, ind} & mask) + r := V~~((b<<3) - b) + o := sel{V, xo, make{V, flat_table{max, oi, 255*(0 32) return{0} if (not has_pdep and wv <= 8) return{0}