From 302c2f926f89f5972eb3aad4ed28fa1aa1fc5106 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Mar 2025 15:26:50 -0500 Subject: [PATCH 1/3] AVX2 k/bool for odd 64= 2}; assert{wv <= 64} + assert{wv >= 2} nw := cdiv{rlen, 64} if (wv&1 == 0) { p := ctz{wv | 8} # Power of two for second replicate @@ -555,14 +555,15 @@ def proc_mod_dat{swap_data:W} = { else tup{8, get_byteperm{}} } # Fill in higher steps - def get_mod_permuter{} = { + def get_mod_permuter{width} = { def get_swap{l} = { def mask = extract_modperm_mask{swap_data, swap_lane, l} modperm_shuf_step{., l, mask} } - def swap_x = on_len_range{get_swap, partwidth, ww} + def swap_x = on_len_range{get_swap, partwidth, width} {x} => partperm{swap_x{x}} } + def get_mod_permuter{} = get_mod_permuter{ww} tup{partperm, get_mod_permuter} } @@ -577,15 +578,18 @@ def rep_const_bool_odd{k, x, r, nw} = { # Modular permutation: small-k cases may use a limited permutation # on bytes or 32-bit ints; general case uses the whole thing swtab:*W = each{modperm_dat{W, .}, 1+2*iota{32}} - swap_data := load{swtab, k>>1} + swap_data := load{swtab, (k%64)>>1} def {partperm, get_full_permute} = proc_mod_dat{swap_data} def sp_max = if (any_sel) 8 else 4 if (k < sp_max) { rep_const_bool_small_odd{W, sp_max, k, getter{partperm}, output} - } else { + } else if (not avx2 or k < 64) { def get_swap_x = getter{get_full_permute{}} rep_const_bool_odd_mask4{W, k, get_swap_x, output, cdiv{nw, vcount{W}}} + } else { + def get_swap_x = getter{get_full_permute{64}} + rep_const_bool_odd_loose_mask{W, k, get_swap_x, output} } flush{} } @@ -599,6 +603,7 @@ def rep_const_bool_odd_mask4{ } = { def ifvec{g} = match (M) { {[_](u64)} => g; {_} => ({v}=>v) } def scal = ifvec{{v} => M**v} + assert{k < 64} # Fundamental operation: shifts act as order-k cyclic group on masks def advance{m, sh} = advance_spaced_mask{k, m, sh} @@ -729,4 +734,55 @@ def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = { } } +# Odd factors larger than 64 +# AVX2-only because scalar should be about as good otherwise +def rep_const_bool_odd_loose_mask{V=[vl==4](u64), k, get_modperm_x, output if hasarch{'AVX2'}} = { + assert{k > 64} + # Distance from end to previous row boundary (-k <= q < 0) + q := -make{V, 64*(1+iota{vl})} + def q_mod{} = { d:=q+V**k; q = blend_top{q,d, d} } + o:u64 = width{V}; while (o>k) { o-=k; q_mod{} } + + km:= k%64 + i:usz = 0; iv:usz = 0 # Words and vectors completed + def step{perm, diff} = { + # Mod-64 mask with 1 bit per word + m:= V**1 << (q & V**63) + # Indicator of which bits are actual boundaries + def S = ty_s{V}; a:= S**(-65) < S~~q + q-= V**o; q_mod{} + # Set to bit from perm, but below a&m xor with diff to get previous + base:= (m & perm) == m + md:= (a & m) & diff + output{base ^ (md + (md==m))} + } + {xp, xd, perm, diff}:= 4**(V**0) + while (1) { + # get_modperm_x permutes each 64-bit word + # Each iteration of this loop handles one permuted word + def first = shuf{., 4**0} + if (i%4 == 0) { + xp = get_modperm_x{} + # Shift by 1, or k%64 in mod-space + # Then the low bit of each word has to be moved to the next + # As before, first bit is wrong but unused + xl:= xp>>(64-km) + xo:= (xp< Date: Fri, 7 Mar 2025 18:17:17 -0500 Subject: [PATCH 2/3] =?UTF-8?q?Use=20a=20possibly-overlapping=20word=20and?= =?UTF-8?q?=20constant-length=20loop=20in=20k/bool,=20like=20=E2=8A=A3`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 24 +++++++++++++++++++----- src/singeli/src/replicate.singeli | 17 +++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 94378e12..8827153a 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -782,13 +782,27 @@ B slash_c2(B t, B w, B x) { u64* rp; r = m_bitarrv(&rp, s); #if SINGELI_AVX2 if (wv <= 256) si_constrep_bool(wv, xp, rp, s); - else #elif SINGELI - if (wv <= 64) si_constrep_bool(wv, xp, rp, s); - else + if (wv <= 128) si_constrep_bool(wv, xp, rp, s); + #else + if (wv <= 64) { BOOL_REP_XOR_SCAN(wv) } #endif - if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) } - else { BOOL_REP_OVER(wv, xlen) } + else { + // Like BOOL_REP_OVER but predictable + u64 ri=0, c=0; usz j=0; + usz n=wv/64-1; + for (usz i = 0; i < xlen; i++) { + u64 v = -(1 & xp[i/64]>>(i%64)); + u64 r0 = c ^ ((v^c) << (ri%64)); + c = v; + ri += wv; usz e = ri/64; + rp[e-1] = v; // This allows the loop to be constant-length + rp[j] = r0; + for (usz k=0; k= 64) { + # Identical to the non-Singeli case for large wv + # but we assume n==0 and drop the loop + assert{wv < 128} + i:usz = 0 + ri:usz = 0; j:usz = 0 # Bit index in r; ri/64 + c:u64 = 0 + while (j < nw-1) { # e<=j+2 below, so e-1>(i%64)); ++i + r0:= c ^ ((v^c) << (ri%64)) + c = v + ri+= wv; e:= ri/64 + store{r, e-1, c} + store{r, j, r0} + j = e + } + if (ri%64 != 0) store{r, j, c} } else { rep_const_bool_odd{wv, x, r, nw} } From b69a2c2469e9e90d166f6f88ea0fe12413bf42f5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Mar 2025 21:25:00 -0500 Subject: [PATCH 3/3] Move swtab outside of its function; it was being compiled as a dynamic array --- src/singeli/src/replicate.singeli | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index b2e6ee41..721e8190 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -489,10 +489,8 @@ def modperm_dat{T, k} = { def w = width{T} def i = iota{lb{w}} def bits = ~(1 & (k*iota{w} >> merge{0, replicate{1< make{T, each{base{2,.}, split{width{E}, bits}}} - {_} => T~~base{2, bits} - } + def E = match (T) { {[_]E} => E; {_} => T } + each{{x} => E~~base{2,x}, split{width{E}, bits}} } # Permutation step evaluators # shift takes a top-half mask; others take both-halves @@ -584,24 +582,24 @@ def proc_mod_dat{swap_data:W} = { tup{partperm, get_mod_permuter} } -def rep_const_bool_odd{k, x, r, nw} = { - def avx2 = hasarch{'AVX2'} - def W = if (has_simd) [if (avx2) 4 else 2]u64 else u64 +def MP = if (has_simd) [if (hasarch{'AVX2'}) 4 else 2]u64 else u64 +swtab:*u64 = join{each{modperm_dat{MP, .}, 1+2*iota{32}}} +def rep_const_bool_odd{k, x, r, nw} = { + def W = MP def {output, check_done, flush} = get_boolvec_writer{W, r, nw} xp := *W~~x def getter{perm}{} = { check_done{}; xv := load{xp}; ++xp; perm{xv} } # Modular permutation: small-k cases may use a limited permutation # on bytes or 32-bit ints; general case uses the whole thing - swtab:*W = each{modperm_dat{W, .}, 1+2*iota{32}} - swap_data := load{swtab, (k%64)>>1} + swap_data := load{*W~~swtab, (k%64)>>1} def {partperm, get_full_permute} = proc_mod_dat{swap_data} def sp_max = if (any_sel) 8 else 4 if (k < sp_max) { rep_const_bool_small_odd{W, sp_max, k, getter{partperm}, output} - } else if (not avx2 or k < 64) { + } else if (not hasarch{'AVX2'} or k < 64) { def get_swap_x = getter{get_full_permute{}} rep_const_bool_odd_mask4{W, k, get_swap_x, output, cdiv{nw, vcount{W}}} } else {