From 0602927b17d6c1ea4709898151505f2a3b7d6794 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 5 Aug 2024 09:57:05 -0400 Subject: [PATCH 01/20] Move pdep-based k/bool code to Singeli --- src/builtins/slash.c | 37 ++++------------------------ src/singeli/src/replicate.singeli | 40 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index bad784ba..98810542 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -82,6 +82,8 @@ extern void (*const si_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2); #define SINGELI_FILE slash #include "../utils/includeSingeli.h" + extern uint64_t* const si_spaced_masks; + #define get_spaced_mask(i) si_spaced_masks[i-1] #define SINGELI_FILE replicate #include "../utils/includeSingeli.h" #endif @@ -765,38 +767,9 @@ B slash_c2(B t, B w, B x) { if (xl == 0) { u64* xp = bitarr_ptr(x); u64* rp; r = m_bitarrv(&rp, s); - #if FAST_PDEP - if (wv <= 52) { - #if SINGELI - u64 m = si_spaced_masks[wv-1]; - #else - u64 m = (u64)-1 / (((u64)1<>= d; - if ((j&(wv-1))==0) xw = xp[++i]; - u64 rw = _pdep_u64(xw, m); - rp[j] = (rw<> (xi%8); - u64 ex = (xw&mt)<>o|(xw&1)); - o += q; - bool oo = o>=wv; xi+=d+oo; o-=wv&-oo; - } - } - goto atmW_maybesh; - } + #if SINGELI + if (si_constrep_bool(wv, xp, rp, s)) ; // Handles small wv + else #endif if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) } else { BOOL_REP_OVER(wv, xlen) } diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index d4fe0aba..c789c571 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -1,5 +1,7 @@ include './base' +if_inline (hasarch{'BMI2'}) include './bmi2' include './mask' +include './spaced' def ind_types = tup{i8, i16, i32} def dat_types = tup{...ind_types, u64} @@ -284,3 +286,41 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { } exportT{'si_constrep', each{rep_const, dat_types}} + + + +# Constant replicate on boolean +fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = 0 +fn rep_const_bool{if hasarch{'BMI2'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { + if (wv > 52) return{0} + m:u64 = spaced_mask_of{wv} + xw:u64 = 0 + d := cast_i{usz, popc{m}} # == 64/wv + nw := cdiv{rlen, 64} + if (m&1 != 0) { # Power of two + i := -usz~~1 + @for (r over j to nw) { + xw >>= d + if ((j&(wv-1))==0) { ++i; xw = load{x, i} } + rw := pdep{xw, m} + r = (rw<> (xi%8) + ex := (xw & mt) << tsh + rw := pdep{xw, m} + r = ((rw-ex)<<(wv-o)) - (rw>>o|(xw&1)) + o += q + oo := o>=wv; xi+=d+promote{usz,oo}; o-=wv&-oo + } + } + 1 +} +export{'si_constrep_bool', rep_const_bool{}} From 7f6c401eb3f3dc4a6549435dba9a071bd22701d5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 5 Aug 2024 11:06:14 -0400 Subject: [PATCH 02/20] Generic-architecture k/bool for small 8 52) return{0} +fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { + def has_pdep = hasarch{'BMI2'} + if (wv > 32) return{0} + if (not has_pdep and wv <= 8) return{0} m:u64 = spaced_mask_of{wv} xw:u64 = 0 d := cast_i{usz, popc{m}} # == 64/wv nw := cdiv{rlen, 64} if (m&1 != 0) { # Power of two i := -usz~~1 + def expand = if (has_pdep) pdep{., m} else { + mult:u64 = spaced_mask_of{wv-1} >> d + xm := (u64~~1 << d) - 1 + {xw} => ((xw&xm)*mult) & m + } @for (r over j to nw) { xw >>= d if ((j&(wv-1))==0) { ++i; xw = load{x, i} } - rw := pdep{xw, m} + rw := expand{xw} r = (rw< ((xw&xm)*mult) & m + } @for (r over j to nw) { xw = loadu{*u64~~(xb + xi/8)} >> (xi%8) ex := (xw & mt) << tsh - rw := pdep{xw, m} + rw := expand{xw} r = ((rw-ex)<<(wv-o)) - (rw>>o|(xw&1)) o += q oo := o>=wv; xi+=d+promote{usz,oo}; o-=wv&-oo From a4b6d8d827f1c2a6b3411c296009b1f0796c52e3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 5 Aug 2024 15:27:36 -0400 Subject: [PATCH 03/20] SSSE3 3/bool, faster than BMI2 --- src/singeli/src/replicate.singeli | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index fc596203..0bf0712a 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -291,6 +291,44 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { + if (hasarch{'SSSE3'} and wv==3) { + def vl = 16; def V = [vl]u8 + nv := cdiv{rlen, width{V}} + i:usz = 0; j:usz = 0 + rv := *V~~r + def end = makelabel{}; while (j < nv) { + xv := load{*V~~x, i}; ++i + # 01234567 to 05316427 on each byte + def bv{bs} = fold{flat_table{+,...}, reverse{each{tup{0,.}, 1< make{V, bv{is}}, split{4, 3*iota{8} % 8}} + m4 := V**0xf + xv = sel{V, t0, xv & m4} | sel{V, t4, V~~([8]u16~~xv>>4) & m4} + # Overhang from previous 64-bit elements + def os = 8-3 # right shift + def ix = __floor{64*slice{iota{3},1} / 3} # bits that overhang within a word + def ib = __floor{ix / 8} # byte index + def io = 8*ib + 3*ix%8 # where they are in xv + def wi = split{2, tup{255, ...ib, 255, ...8+ib}} + def W = [2]u64 + xo := V~~((W~~xv & W**fold{|, 1<> os) + xo += xo > V**0 + # Permute and mask bytes + def step{jj, oi, ind, mask} = { + b := W~~(sel{V, xv, ind} & mask) + r := V~~((b<<3) - b) + o := sel{V, xo, make{V, flat_table{max, oi, 255*(0 32) return{0} if (not has_pdep and wv <= 8) return{0} From a439a0a430e7eddcf1b413db1aecc3793146c84b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 6 Aug 2024 10:02:27 -0400 Subject: [PATCH 04/20] =?UTF-8?q?SSSE3=20k/bool=20for=202=E2=89=A4k?= =?UTF-8?q?=E2=89=A48,=20k=E2=89=A06?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/replicate.singeli | 180 +++++++++++++++++++++++------- 1 file changed, 142 insertions(+), 38 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 0bf0712a..a14aa44c 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -291,46 +291,12 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - if (hasarch{'SSSE3'} and wv==3) { - def vl = 16; def V = [vl]u8 - nv := cdiv{rlen, width{V}} - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { - xv := load{*V~~x, i}; ++i - # 01234567 to 05316427 on each byte - def bv{bs} = fold{flat_table{+,...}, reverse{each{tup{0,.}, 1< make{V, bv{is}}, split{4, 3*iota{8} % 8}} - m4 := V**0xf - xv = sel{V, t0, xv & m4} | sel{V, t4, V~~([8]u16~~xv>>4) & m4} - # Overhang from previous 64-bit elements - def os = 8-3 # right shift - def ix = __floor{64*slice{iota{3},1} / 3} # bits that overhang within a word - def ib = __floor{ix / 8} # byte index - def io = 8*ib + 3*ix%8 # where they are in xv - def wi = split{2, tup{255, ...ib, 255, ...8+ib}} - def W = [2]u64 - xo := V~~((W~~xv & W**fold{|, 1<> os) - xo += xo > V**0 - # Permute and mask bytes - def step{jj, oi, ind, mask} = { - b := W~~(sel{V, xv, ind} & mask) - r := V~~((b<<3) - b) - o := sel{V, xo, make{V, flat_table{max, oi, 255*(0 32) return{0} + if (hasarch{'SSSE3'} and wv<=8 and wv!=6) { + rep_const_bool_ssse3{wv, x, r, rlen} + return{1} + } if (not has_pdep and wv <= 8) return{0} m:u64 = spaced_mask_of{wv} xw:u64 = 0 @@ -372,4 +338,142 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } 1 } +def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6 + oper // ({a,b}=>floor{a/b}) infix left 40 + def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1<>4)}, .} + store{rv, j, V~~getr{0}}; ++j; if (j==nv) goto{end} + store{rv, j, V~~getr{1}}; ++j + } + setlabel{end} + } + def run{2} = { + # Expander for half byte + tabr := mkV{bv{2*iota{4}} * 2b11} + m4 := V**0xf + def exh{x} = selV{tabr, x & m4} + run24{*V~~x, {xv}=>xv, exh} + } + def run{4} = { + dup := mkV{iV//2} # Double each byte + # Expander for two bits in either bottom or next-to-bottom position + tabr := mkV{bv{tup{0,4,0,4}} * 2b1111} + m2 := mkV{2b11 << (2*(iV%2))} + def exh{x} = re_el{u16, V}~~selV{tabr, x & m2} + run24{*u64~~x, selV{., dup}, exh} + } + + def run{8} = { + @for (r in *V~~r over i to nv) { + xv := load{*V~~(*u16~~x + i)} + xe := selV{xv, mkV{iV//8}} + r = (xe & mkV{1 << (iV % 8)}) > V**0 + } + } + + # For odd numbers: + # - permute each byte sending bit i to position k*i % 8 + # - replicate each byte by k, making position k*i contain bit i + # - mask out those bits and spread over [ k*i, k*(i+1) ) + # - ...except where it crosses words; handle this overhang separately + # 3 is handled separately and unrolled; 5 and 7 share a loop + def run{3} = { + i:usz = 0; j:usz = 0 + rv := *V~~r + def end = makelabel{}; while (j < nv) { + xv := load{*V~~x, i}; ++i + # 01234567 to 05316427 on each byte + {t0, t4} := each{{is} => mkV{bv{is}}, split{4, 3*iota{8} % 8}} + m4 := V**0xf + xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + # Overhang from previous 64-bit elements + def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word + def ib = ix // 8 # byte index + def io = 8*ib + 3*ix%8 # where they are in xv + def wi = split{2, tup{255, ...ib, 255, ...8+ib}} + xo := V~~((W~~xv & W**fold{|, 1<> (8-3)) + xo += xo > V**0 + # Permute and mask bytes + def step{jj, oi, ind, mask} = { + b := W~~(selV{xv, ind} & mask) + r := V~~((b<<3) - b) + o := selV{xo, mkV{flat_table{max, oi, 255*(0 mkV{bv{is}}, split{4, k*iota{8} % 8}} + # Overhang from previous 64-bit elements + def ix = 64*slice{iota{k},1} // k # bits that overhang within a word + def ib = ix // 8 # byte index + def io = 8*ib + k*ix%8 # where they are in xv + def wi = tup{255, ...ib, 255, ...8+ib} + xom = W**fold{|, 1<=vl + ind_up = mkV{iu - k*ia} + ind_inc = mkV{vl//k + ia} + mask_sh = width{V} % k + } + if (wv == 5) set_consts{5} else set_consts{7} + xv:V = V**0; xo:=xv; ind:=xv; mask:=xv # state + i:usz = 0; q:usz = 1 + @for (r in *V~~r over j to nv) { + --q; if (q == 0) { q = wv + # Load and permute bytes + xv = load{*V~~x, i}; ++i + m4 := V**0xf + xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + # Bytes for overhang + xo = V~~((W~~xv & xom) >> (8 - wv)) + xo += xo > V**0 + xo = selV{xo, xse} + # Initialize state vectors + ind = ind0 + mask = mask0 + } else { + # Update state vectors + xo = shr{V, xo, 1} + ind = selV{ind, ind_up} + ind_inc + mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + } + b := W~~(selV{xv, ind} & mask) + r = V~~((b< Date: Tue, 6 Aug 2024 22:16:20 -0400 Subject: [PATCH 05/20] =?UTF-8?q?Extend=20odd=20k/bool=20SSSE3=20algorithm?= =?UTF-8?q?=20to=20k<16,=20factor=20even=20k=E2=89=A432?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/replicate.singeli | 127 +++++++++++++++++++++--------- 1 file changed, 88 insertions(+), 39 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index a14aa44c..5a29cf96 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -293,9 +293,22 @@ exportT{'si_constrep', each{rep_const, dat_types}} fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { def has_pdep = hasarch{'BMI2'} if (wv > 32) return{0} - if (hasarch{'SSSE3'} and wv<=8 and wv!=6) { - rep_const_bool_ssse3{wv, x, r, rlen} - return{1} + if (hasarch{'SSSE3'}) { + if (wv&1 == 0) { + p := ctz{wv | 8} # Power of two for second replicate + if (wv>>p == 1) { + rep_const_bool_ssse3_div8{wv, x, r, rlen} + } else { + tlen := rlen>>p + t := r + cdiv{rlen, 64} - cdiv{tlen, 64} + rep_const_bool{}(wv >>p, x, t, tlen) + rep_const_bool{}(usz~~1<floor{a/b}) infix left 40 def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1<xv, exh} - } - def run{4} = { - dup := mkV{iV//2} # Double each byte + } else if (wv == 4) { + # Double each byte + dup := mkV{iV//2} # Expander for two bits in either bottom or next-to-bottom position tabr := mkV{bv{tup{0,4,0,4}} * 2b1111} m2 := mkV{2b11 << (2*(iV%2))} def exh{x} = re_el{u16, V}~~selV{tabr, x & m2} run24{*u64~~x, selV{., dup}, exh} - } - - def run{8} = { + } else { # wv == 8 @for (r in *V~~r over i to nv) { xv := load{*V~~(*u16~~x + i)} xe := selV{xv, mkV{iV//8}} r = (xe & mkV{1 << (iV % 8)}) > V**0 } } +} +# For odd numbers: +# - permute each byte sending bit i to position k*i % 8 +# - replicate each byte by k, making position k*i contain bit i +# - mask out those bits and spread over [ k*i, k*(i+1) ) +# - ...except where it crosses words; handle this overhang separately +def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 + oper // ({a,b}=>floor{a/b}) infix left 40 + def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1< mkV{bv{is}}, split{4, k*iota{8} % 8}} + ttab:*V = join{each{get_ttab, 2*iota{4} + 1}} + {t0, t4} := each{load{ttab + (wv & 6), .}, iota{2}} + m4 := V**0xf + def get_perm_x{i} = { + xv := load{*V~~x, i} + selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + } + + # Cases are 3; 5 7; 9 11 13 15 + if (wv < 4) { + # 3: dedicated loop i:usz = 0; j:usz = 0 rv := *V~~r def end = makelabel{}; while (j < nv) { - xv := load{*V~~x, i}; ++i # 01234567 to 05316427 on each byte - {t0, t4} := each{{is} => mkV{bv{is}}, split{4, 3*iota{8} % 8}} - m4 := V**0xf - xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + xv := get_perm_x{i}; ++i # Overhang from previous 64-bit elements def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word def ib = ix // 8 # byte index @@ -420,13 +449,11 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6 } } setlabel{end} - } - def run{{5,7}} = { - {t0, t4, xom, xse, ind0, mask0, ind_up, ind_inc, mask_sh} := undef{tup{ - V, V, W, V, V, V, V, V, usz }} + } else if (wv < 8) { + # 5, 7: precompute constants, then shared loop + {xom, xse, ind0, mask0, ind_up, ind_inc, mask_sh} := undef{tup{ + W, V, V, V, V, V, usz }} def set_consts{k} = { - # Within-byte transformation - tup{t0, t4} = each{{is} => mkV{bv{is}}, split{4, k*iota{8} % 8}} # Overhang from previous 64-bit elements def ix = 64*slice{iota{k},1} // k # bits that overhang within a word def ib = ix // 8 # byte index @@ -448,9 +475,7 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6 @for (r in *V~~r over j to nv) { --q; if (q == 0) { q = wv # Load and permute bytes - xv = load{*V~~x, i}; ++i - m4 := V**0xf - xv = selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} + xv = get_perm_x{i}; ++i # Bytes for overhang xo = V~~((W~~xv & xom) >> (8 - wv)) xo += xo > V**0 @@ -468,12 +493,36 @@ def rep_const_bool_ssse3{wv, x, r, rlen} = { # 2<=wv<=8, wv!=6 r = V~~((b<= k} + {m, d} := unaligned_spaced_mask_mod{wv} + mask0:= V~~make{W, m, m>>d|m<<(wv-d)} + iu:= iota{V} + V**cast_i{u8,vl-wv}; ia:= iu>=V**vl + ind_up := iu - (wV&ia) + ind_inc:= V**(wv <= vl) - ia + mask_sh:= d+d; if (mask_sh >= wv) mask_sh-= wv + # State + xv:V = V**0; o:=W**0; ind:=xv; mask:=xv + i:usz = 0; q:usz = 1 + @for (r in *V~~r over j to nv) { + --q; if (q == 0) { q = wv + xv = get_perm_x{i}; ++i + ind = ind0 + mask = mask0 + } else { + ind = selV{ind, ind_up} + ind_inc + mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + } + # Handle overhang here; won't fit in a single vector + b := W~~(selV{xv, ind} & mask) + r = V~~((b<>(64-wv) + ro:= [4]u32~~vshl{po, o, 1} + r |= V~~(ro + (ro > [4]u32**0)) + } } - - if (wv==2) run{2} - else if (wv==3) run{3} - else if (wv==4) run{4} - else if (wv==8) run{8} - else run{tup{5,7}} } export{'si_constrep_bool', rep_const_bool{}} From 1a4cada0cbeeeee15021ab27ad2b5c4e053e1ccf Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 7 Aug 2024 10:35:21 -0400 Subject: [PATCH 06/20] AVX2 support in rep_const_bool_ssse3_div8 --- src/singeli/src/replicate.singeli | 46 ++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 5a29cf96..3b4aa0f7 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -352,13 +352,25 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { 1 } +# Generalized flat transpose of iota{1<floor{a/b}) infix left 40 - def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1<>4)}, .} + def getr = zip128{exh{xv}, exh{V~~(re_el{u16,V}~~xv>>4)}, .} store{rv, j, V~~getr{0}}; ++j; if (j==nv) goto{end} store{rv, j, V~~getr{1}}; ++j } setlabel{end} } if (wv == 2) { + def init = if (avx2) shuf{[4]u64, ., 4b3120} else id # Expander for half byte - tabr := mkV{bv{2*iota{4}} * 2b11} + def tabr = makeTab{tr_iota{2*iota{4}} * 2b11} m4 := V**0xf - def exh{x} = selV{tabr, x & m4} - run24{*V~~x, {xv}=>xv, exh} + run24{*V~~x, init, {x} => tabr{x & m4}} } else if (wv == 4) { - # Double each byte - dup := mkV{iV//2} + # Unzip 32-bit elements (result lanes) across AVX2 lanes + def pre = if (avx2) sel{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id + def init{xv} = { u:=pre{xv}; zip128{u,u,0} } # Expander for two bits in either bottom or next-to-bottom position - tabr := mkV{bv{tup{0,4,0,4}} * 2b1111} + def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111} m2 := mkV{2b11 << (2*(iV%2))} - def exh{x} = re_el{u16, V}~~selV{tabr, x & m2} - run24{*u64~~x, selV{., dup}, exh} + def exh{x} = re_el{u16, V}~~tabr{x & m2} + run24{*(if (avx2) [2]u64 else u64)~~x, init, exh} } else { # wv == 8 @for (r in *V~~r over i to nv) { - xv := load{*V~~(*u16~~x + i)} - xe := selV{xv, mkV{iV//8}} + xh := load{*[16]u8~~(*ty_u{vl}~~x + i)} + xv := if (avx2) pair{xh, xh} else xh + xe := selH{xv, mkV{iV // 8}} r = (xe & mkV{1 << (iV % 8)}) > V**0 } } } + # For odd numbers: # - permute each byte sending bit i to position k*i % 8 # - replicate each byte by k, making position k*i contain bit i @@ -401,7 +416,6 @@ def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 # - ...except where it crosses words; handle this overhang separately def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 oper // ({a,b}=>floor{a/b}) infix left 40 - def bv{bs} = fold{flat_table{|,...}, reverse{each{tup{0,.}, 1< mkV{bv{is}}, split{4, k*iota{8} % 8}} + def get_ttab{k} = each{{is} => mkV{tr_iota{is}}, split{4, k*iota{8} % 8}} ttab:*V = join{each{get_ttab, 2*iota{4} + 1}} {t0, t4} := each{load{ttab + (wv & 6), .}, iota{2}} m4 := V**0xf From 696a23af9e4ce8e333aa08f2d7da52782b366368 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 7 Aug 2024 17:38:57 -0400 Subject: [PATCH 07/20] Mask off last k/bool vector properly --- src/singeli/src/replicate.singeli | 62 +++++++++++++++++++------------ 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 3b4aa0f7..b14c29b3 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -360,6 +360,26 @@ def tr_iota{...bs} = { } def tr_iota{{...bs}} = tr_iota{...bs} +def get_boolvec_writer{V, r, rlen} = { + def vwords = width{V}/64 + nw := cdiv{rlen, 64} + rv := *V~~r + re := rv + nw / vwords + last_res:V = V**0 + def end = makelabel{} + def output{v:(V)} = { + last_res = v + if (rv==re) goto{end} + store{rv, 0, v}; ++rv + } + def flush{} = { + setlabel{end} + q := nw & (vwords-1) + if (q != 0) homMaskStoreF{rv, V~~maskOf{re_el{u64,V}, q}, last_res} + } + tup{output, flush} +} + def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 oper // ({a,b}=>floor{a/b}) infix left 40 def avx2 = hasarch{'AVX2'} @@ -369,20 +389,17 @@ def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 def mkV = make{V, .} def selH = sel{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} - nv := cdiv{rlen, width{V}} def id{xv} = xv + def {output, flush} = get_boolvec_writer{V, r, rlen} def run24{x, proc_xv, exh} = { - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { + i:usz = 0; while (1) { xv := proc_xv{load{*V~~(x+i)}}; ++i # Store 1 or 2 result vectors def getr = zip128{exh{xv}, exh{V~~(re_el{u16,V}~~xv>>4)}, .} - store{rv, j, V~~getr{0}}; ++j; if (j==nv) goto{end} - store{rv, j, V~~getr{1}}; ++j + output{V~~getr{0}} + output{V~~getr{1}} } - setlabel{end} } if (wv == 2) { def init = if (avx2) shuf{[4]u64, ., 4b3120} else id @@ -400,13 +417,14 @@ def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 def exh{x} = re_el{u16, V}~~tabr{x & m2} run24{*(if (avx2) [2]u64 else u64)~~x, init, exh} } else { # wv == 8 - @for (r in *V~~r over i to nv) { - xh := load{*[16]u8~~(*ty_u{vl}~~x + i)} + i:usz = 0; while (1) { + xh := load{*[16]u8~~(*ty_u{vl}~~x + i)}; ++i xv := if (avx2) pair{xh, xh} else xh xe := selH{xv, mkV{iV // 8}} - r = (xe & mkV{1 << (iV % 8)}) > V**0 + output{(xe & mkV{1 << (iV % 8)}) > V**0} } } + flush{} } # For odd numbers: @@ -420,8 +438,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 def iV = iota{vl} def mkV = make{V, .}; def selV = sel{V, ., .} def W = [2]u64 - - nv := cdiv{rlen, width{V}} + def {output, flush} = get_boolvec_writer{V, r, rlen} # Within-byte transformation def get_ttab{k} = each{{is} => mkV{tr_iota{is}}, split{4, k*iota{8} % 8}} @@ -436,9 +453,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 # Cases are 3; 5 7; 9 11 13 15 if (wv < 4) { # 3: dedicated loop - i:usz = 0; j:usz = 0 - rv := *V~~r - def end = makelabel{}; while (j < nv) { + i:usz = 0; while (1) { # 01234567 to 05316427 on each byte xv := get_perm_x{i}; ++i # Overhang from previous 64-bit elements @@ -453,7 +468,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 b := W~~(selV{xv, ind} & mask) r := V~~((b<<3) - b) o := selV{xo, mkV{flat_table{max, oi, 255*(0> mask_sh)) } b := W~~(selV{xv, ind} & mask) - r = V~~((b<>(64-wv) ro:= [4]u32~~vshl{po, o, 1} - r |= V~~(ro + (ro > [4]u32**0)) + output{rv | V~~(ro + (ro > [4]u32**0))} } } + flush{} } export{'si_constrep_bool', rep_const_bool{}} From 7d7d36b35482fb56d685f2a8b6a0d049645ef473 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 7 Aug 2024 22:03:55 -0400 Subject: [PATCH 08/20] Extend SSSE3 k/bool to k<32 using pairwise swaps --- src/singeli/src/replicate.singeli | 99 +++++++++++++++++-------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index b14c29b3..2d584525 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -291,25 +291,8 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - def has_pdep = hasarch{'BMI2'} + def has_pdep = 0 # Obselete, kept here for descriptiveness if (wv > 32) return{0} - if (hasarch{'SSSE3'}) { - if (wv&1 == 0) { - p := ctz{wv | 8} # Power of two for second replicate - if (wv>>p == 1) { - rep_const_bool_ssse3_div8{wv, x, r, rlen} - } else { - tlen := rlen>>p - t := r + cdiv{rlen, 64} - cdiv{tlen, 64} - rep_const_bool{}(wv >>p, x, t, tlen) - rep_const_bool{}(usz~~1< 32) return{0} + if (wv&1 == 0) { + p := ctz{wv | 8} # Power of two for second replicate + if (wv>>p == 1) { + rep_const_bool_ssse3_div8{wv, x, r, rlen} + } else { + tlen := rlen>>p + t := r + cdiv{rlen, 64} - cdiv{tlen, 64} + rep_const_bool{}(wv >>p, x, t, tlen) + rep_const_bool{}(usz~~1<>4) & m4} } @@ -455,7 +455,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 # 3: dedicated loop i:usz = 0; while (1) { # 01234567 to 05316427 on each byte - xv := get_perm_x{i}; ++i + xv := perm_x{load{*V~~x, i}}; ++i # Overhang from previous 64-bit elements def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word def ib = ix // 8 # byte index @@ -503,7 +503,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 while (1) { --q; if (q == 0) { q = wv # Load and permute bytes - xv = get_perm_x{i}; ++i + xv = perm_x{load{*V~~x, i}}; ++i # Bytes for overhang xo = V~~((W~~xv & xom) >> (8 - wv)) xo += xo > V**0 @@ -522,35 +522,42 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 o := xo & mkV{255 * (iV%8 == 0)} # overhang output{rv | o} } - } else { # wv < 16 - # 9, 11, 13, 15: shared constant computation and loop - # Initializers, relying on vl%wv == vl-wv in various ways - wV:= V**cast_i{u8,wv} - ind0 := (iota{V} < wV) + V**1 # mkV{iV >= k} + } else { # wv < 32 + # 9 to 31: extend k*i % 8 transform to k*i % 128 by pairwise swaps + # Swap data goes in a pre-computed table + def swdat{k} = { + def bits = (k*iota{128} >> merge{0, replicate{1<>1)-4} + swap_masks := each{{l} => selV{swap_data, mkV{l+iV%l}}, swap_lens} + # Every-k-bits mask, same as before {m, d} := unaligned_spaced_mask_mod{wv} - mask0:= V~~make{W, m, m>>d|m<<(wv-d)} - iu:= iota{V} + V**cast_i{u8,vl-wv}; ia:= iu>=V**vl - ind_up := iu - (wV&ia) - ind_inc:= V**(wv <= vl) - ia - mask_sh:= d+d; if (mask_sh >= wv) mask_sh-= wv + mask := V~~make{W, m, m>>d|m<<(wv-d)} + mask_sh := d+d; if (mask_sh >= wv) mask_sh-= wv # State - xv:V = V**0; o:=W**0; ind:=xv; mask:=xv + xv:V = V**0; o:=W**0 i:usz = 0; q:usz = 1 while (1) { - --q; if (q == 0) { q = wv - xv = get_perm_x{i}; ++i - ind = ind0 - mask = mask0 - } else { - ind = selV{ind, ind_up} + ind_inc - mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + # Load xv, send bit i to position wv*i % 128 + xv = load{*V~~x, i}; ++i + def swap_step{l, m} = { + xv = (xv & m) | (selV{xv, mkV{iV + (l-2*(iV&l))}} &~ m) + } + each{swap_step, swap_lens, swap_masks} + xv = perm_x{xv} + # Write wv vectors based on that + @for (wv) { + b := W~~(xv & mask) + mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + rv:= V~~((b<>(64-wv) + ro:= [4]u32~~vshl{po, o, 1} + output{rv | V~~(ro + (ro > [4]u32**0))} } - # Handle overhang here; won't fit in a single vector - b := W~~(selV{xv, ind} & mask) - rv:= V~~((b<>(64-wv) - ro:= [4]u32~~vshl{po, o, 1} - output{rv | V~~(ro + (ro > [4]u32**0))} } } flush{} From 1621c7d07cb67f114b43575e8d511118dabb6eaa Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 8 Aug 2024 18:08:49 -0400 Subject: [PATCH 09/20] Generic-arch k/bool algorithm for odd k with mask unpacking --- src/singeli/src/replicate.singeli | 108 +++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 2d584525..51b6ae65 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -293,12 +293,15 @@ exportT{'si_constrep', each{rep_const, dat_types}} fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { def has_pdep = 0 # Obselete, kept here for descriptiveness if (wv > 32) return{0} - if (not has_pdep and wv <= 8) return{0} m:u64 = spaced_mask_of{wv} xw:u64 = 0 d := cast_i{usz, popc{m}} # == 64/wv nw := cdiv{rlen, 64} - if (m&1 != 0) { # Power of two + if (wv&1 != 0) { + rep_const_bool_generic_odd{wv, x, r, nw, m, d} + } else if (not has_pdep and wv <= 8) { + return{0} + } else if (m&1 != 0) { # Power of two i := -usz~~1 def expand = if (has_pdep) pdep{., m} else { mult:u64 = spaced_mask_of{wv-1} >> d @@ -335,6 +338,105 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { 1 } +def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = { + # Every-k-bits mask + mask_sh := cast_i{usz, ctz{m}} # == 64%k + mask := m<<(k-mask_sh) | 1 + + # Transform sending bit i to k*i % 64 by pairwise swaps + # Swap data goes in a pre-computed table + def swdat{lw, k} = { + def i = iota{lw} + def bits = (k*iota{1<> merge{0, replicate{1<>1} + swsel:u64 = ~u64~~0 + def gsw{l} = { + swsel ^= swsel << l # Low l bits out of every 2*l + sm := swap_data &~ swsel + swap_data &= swsel; swap_data |= swap_data<>l) + } + def swap_step{l==32, m} = { # Use rotate + mm := m | m>>l + x = (x &~ mm) | ((x<>l) & mm) + } + each{swap_step, swap_lens, swap_masks} + x + } + + # Output + j:usz = 0 + def output{rw} = { + store{rp, j, rw} + ++j; if (j==nw) return{1} + } + o:u64 = 0 # carry + # Dedicated loop for 3, shared for other factors + if (k == 3) { + while (1) { + x := get_swap_x{} + @unroll (jj to 3) { + b := x & mask + mask = (mask << (3 - 64%3)) | (mask >> (64%3)) + def os = (64-3) + 1 + iota{select{tup{0,2,1}, jj}} + output{fold{|, (b<<3) - b, each{>>{o,.}, os}}} + o = b + } + } + } else { + # Fundamental operation: shifts act as order-k cyclic group on masks + def advance{m, sh} = m<<(k-sh) | m>>sh + sm0 := mask # starting mask + s1 := mask_sh # single iteration shift + # Get cumulative mask for 4 iterations, and shift to advance 4 + def double{{mc, s}} = { + def ss = s+s + tup{mc|advance{mc,s}, ss - (k &- (ss>k))} + } + {mc4, s4} := double{double{tup{sm0, s1}}} + # Submasks pick one mask out of a combination of 4 + def or_adv{m, s} = { m |= advance{m,s} } + @for (min{k/4 - 1, nw/4}) or_adv{sm0,s4} + submasks := scan{advance, tup{sm0, ...3**s1}} + mask_tail := advance{sm0, s4} &~ sm0 + while (1) { + x := get_swap_x{} + mask = mc4 + # Write result word given starting bits + def step{b} = { + r := (b< 0})} + o = b>>(64-k) + } + # Fast unrolled iterations + @for (k/4) { + xm := x & mask + each{{mm} => step{xm & mm}, submasks} + mask = advance{mask, s4} + } + # Single-step for tail + mask = mask_tail + @for (k%4) { + step{x & mask} + mask = advance{mask, s1} + } + } + } +} + fn rep_const_bool{if hasarch{'SSSE3'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { if (wv > 32) return{0} if (wv&1 == 0) { @@ -539,7 +641,7 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 mask_sh := d+d; if (mask_sh >= wv) mask_sh-= wv # State xv:V = V**0; o:=W**0 - i:usz = 0; q:usz = 1 + i:usz = 0 while (1) { # Load xv, send bit i to position wv*i % 128 xv = load{*V~~x, i}; ++i From 13ec029d9f6f72f4ae8216e396b502d76c4b52f0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 9 Aug 2024 08:29:43 -0400 Subject: [PATCH 10/20] Get odd k/bool carry by shifting in the unified register, not after masking --- src/singeli/src/replicate.singeli | 33 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 51b6ae65..f40475b6 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -412,25 +412,24 @@ def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = { @for (min{k/4 - 1, nw/4}) or_adv{sm0,s4} submasks := scan{advance, tup{sm0, ...3**s1}} mask_tail := advance{sm0, s4} &~ sm0 + # Mask out carry bit + mr := u64~~1<>(64-k); o=xo&1; xo=(xo&~1)|os mask = mc4 # Write result word given starting bits - def step{b} = { - r := (b< 0})} - o = b>>(64-k) - } + def step{b, c} = output{c - b - promote{u64, c&mr != 0}} + def step{b, c, m} = step{b&m, c&m} # Fast unrolled iterations @for (k/4) { - xm := x & mask - each{{mm} => step{xm & mm}, submasks} + each{step{x & mask, xo & mask, .}, submasks} mask = advance{mask, s4} } # Single-step for tail mask = mask_tail @for (k%4) { - step{x & mask} + step{x, xo, mask} mask = advance{mask, s1} } } @@ -637,8 +636,10 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 swap_masks := each{{l} => selV{swap_data, mkV{l+iV%l}}, swap_lens} # Every-k-bits mask, same as before {m, d} := unaligned_spaced_mask_mod{wv} - mask := V~~make{W, m, m>>d|m<<(wv-d)} + mask := make{W, m, m>>d|m<<(wv-d)} mask_sh := d+d; if (mask_sh >= wv) mask_sh-= wv + # Mask out carry bit + mr := [4]u32~~W**(u64~~1<>(64-wv)}; o=xo&w1; xo=(xo&~w1)|os # Write wv vectors based on that @for (wv) { - b := W~~(xv & mask) - mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) - rv:= V~~((b<>(64-wv) - ro:= [4]u32~~vshl{po, o, 1} - output{rv | V~~(ro + (ro > [4]u32**0))} + c := xo & mask; cu := [4]u32~~c + output{V~~(W~~(cu + (mr&cu > [4]u32**0)) - b)} + mask = (mask << (wv - mask_sh)) | (mask >> mask_sh) } } } From 3e5dbdbf8dd33880b2041239f1bb9c439f6969cd Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 9 Aug 2024 14:44:33 -0400 Subject: [PATCH 11/20] Dedicated generic k/bool for divisors of 8; factor; remove pdep emulation --- src/singeli/src/replicate.singeli | 94 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index f40475b6..6eec5b65 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -1,5 +1,4 @@ include './base' -if_inline (hasarch{'BMI2'}) include './bmi2' include './mask' include './spaced' @@ -291,53 +290,60 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - def has_pdep = 0 # Obselete, kept here for descriptiveness - if (wv > 32) return{0} - m:u64 = spaced_mask_of{wv} - xw:u64 = 0 - d := cast_i{usz, popc{m}} # == 64/wv + if (wv > 64) return{0} nw := cdiv{rlen, 64} - if (wv&1 != 0) { - rep_const_bool_generic_odd{wv, x, r, nw, m, d} - } else if (not has_pdep and wv <= 8) { - return{0} - } else if (m&1 != 0) { # Power of two - i := -usz~~1 - def expand = if (has_pdep) pdep{., m} else { - mult:u64 = spaced_mask_of{wv-1} >> d - xm := (u64~~1 << d) - 1 - {xw} => ((xw&xm)*mult) & m - } - @for (r over j to nw) { - xw >>= d - if ((j&(wv-1))==0) { ++i; xw = load{x, i} } - rw := expand{xw} - r = (rw<>p + if (wf == 1) { + rep_const_bool_div8{wv, x, r, nw} + } else { + tlen := rlen>>p + wq := usz~~1<

=52)) { # Expanding odd second is faster + tlen = rlen / wf + t:=wf; wf=wq; wq=t + } + t := r + cdiv{rlen, 64} - cdiv{tlen, 64} + rep_const_bool{}(wf, x, t, tlen) + rep_const_bool{}(wq, t, r, rlen) } } else { - q := cast_i{usz, ctz{m}} # == 64%wv - m = m<<(wv-q) | 1 - mt := u64~~1 << (d+1) # Bit d+1 may be needed, isn't pdep-ed - tsh := d*wv-(d+1) - xb := *u8~~x - xi:usz=0; o:usz=0 - def expand = if (has_pdep) pdep{., m} else { - {mult, _} := unaligned_spaced_mask_mod{wv-1} - xm := mt - 1 - {xw} => ((xw&xm)*mult) & m - } - @for (r over j to nw) { - xw = loadu{*u64~~(xb + xi/8)} >> (xi%8) - ex := (xw & mt) << tsh - rw := expand{xw} - r = ((rw-ex)<<(wv-o)) - (rw>>o|(xw&1)) - o += q - oo := o>=wv; xi+=d+promote{usz,oo}; o-=wv&-oo - } + m:u64 = spaced_mask_of{wv} + d := cast_i{usz, popc{m}} # == 64/wv + rep_const_bool_generic_odd{wv, x, r, nw, m, d} } 1 } +def rep_const_bool_div8{wv, x, r, nw} = { + def run{k} = { + # 2 -> 64w0x33, 12 -> 64w0x000f, etc. + def getm{sh} = base{2, iota{64}&sh == 0} + def osh{v, s} = v | v< fold{ + {v, sh} => osh{v, sh} & getm{sh}, + ., 1 << reverse{iota{5}} + } + {4} => fold{ + {v, sh} => osh{osh{v, sh}, 2*sh} & getm{sh}, + ., tup{12, 3} + } + {8} => { + def mult = base{1<<7, 8**1} + {x} => (x | ((x&~1) * mult)) & 64w0x01 + } + } + @for (xt in *ty_u{64/k}~~x, r over nw) { + def v = expand{promote{u64, xt}} + r = v<> merge{0, replicate{1<>1} swsel:u64 = ~u64~~0 @@ -441,7 +447,7 @@ fn rep_const_bool{if hasarch{'SSSE3'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = if (wv&1 == 0) { p := ctz{wv | 8} # Power of two for second replicate if (wv>>p == 1) { - rep_const_bool_ssse3_div8{wv, x, r, rlen} + rep_const_bool_div8{wv, x, r, rlen} } else { tlen := rlen>>p t := r + cdiv{rlen, 64} - cdiv{tlen, 64} @@ -482,7 +488,7 @@ def get_boolvec_writer{V, r, rlen} = { tup{output, flush} } -def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8 +def rep_const_bool_div8{wv, x, r, rlen if hasarch{'SSSE3'}} = { # wv in 2,4,8 oper // ({a,b}=>floor{a/b}) infix left 40 def avx2 = hasarch{'AVX2'} def vl = if (avx2) 32 else 16 From 2be022921e858bd70a3ec9b8fb6bbdc15f3e3e25 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 10 Aug 2024 14:20:22 -0400 Subject: [PATCH 12/20] Combine generic and SSSE3 general-case odd/bool, so SSSE3 uses mask unpacking --- src/singeli/src/replicate.singeli | 367 +++++++++++++++--------------- 1 file changed, 185 insertions(+), 182 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 6eec5b65..351a3644 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -300,7 +300,8 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } else { tlen := rlen>>p wq := usz~~1<

=52)) { # Expanding odd second is faster + if ((not hasarch{'SSSE3'} and (p == 1 or (p == 2 and wv>=52))) or (not hasarch{'AVX2'} and p == 1 and wv>=24)) { + # Expanding odd second is faster tlen = rlen / wf t:=wf; wf=wq; wq=t } @@ -309,14 +310,12 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { rep_const_bool{}(wq, t, r, rlen) } } else { - m:u64 = spaced_mask_of{wv} - d := cast_i{usz, popc{m}} # == 64/wv - rep_const_bool_generic_odd{wv, x, r, nw, m, d} + rep_const_bool_odd{wv, x, r, nw} } 1 } -def rep_const_bool_div8{wv, x, r, nw} = { +def rep_const_bool_div8{wv, x, r, nw} = { # wv in 2,4,8 def run{k} = { # 2 -> 64w0x33, 12 -> 64w0x000f, etc. def getm{sh} = base{2, iota{64}&sh == 0} @@ -344,133 +343,8 @@ def rep_const_bool_div8{wv, x, r, nw} = { cases{2} } -def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = { - # Every-k-bits mask - mask_sh := cast_i{usz, ctz{m}} # == 64%k - mask := m<<(k-mask_sh) | 1 - - # Transform sending bit i to k*i % 64 by pairwise swaps - # Swap data goes in a pre-computed table - def swdat{lw, k} = { - def i = iota{lw} - def bits = (k*iota{1<> merge{0, replicate{1<>1} - swsel:u64 = ~u64~~0 - def gsw{l} = { - swsel ^= swsel << l # Low l bits out of every 2*l - sm := swap_data &~ swsel - swap_data &= swsel; swap_data |= swap_data<>l) - } - def swap_step{l==32, m} = { # Use rotate - mm := m | m>>l - x = (x &~ mm) | ((x<>l) & mm) - } - each{swap_step, swap_lens, swap_masks} - x - } - - # Output - j:usz = 0 - def output{rw} = { - store{rp, j, rw} - ++j; if (j==nw) return{1} - } - o:u64 = 0 # carry - # Dedicated loop for 3, shared for other factors - if (k == 3) { - while (1) { - x := get_swap_x{} - @unroll (jj to 3) { - b := x & mask - mask = (mask << (3 - 64%3)) | (mask >> (64%3)) - def os = (64-3) + 1 + iota{select{tup{0,2,1}, jj}} - output{fold{|, (b<<3) - b, each{>>{o,.}, os}}} - o = b - } - } - } else { - # Fundamental operation: shifts act as order-k cyclic group on masks - def advance{m, sh} = m<<(k-sh) | m>>sh - sm0 := mask # starting mask - s1 := mask_sh # single iteration shift - # Get cumulative mask for 4 iterations, and shift to advance 4 - def double{{mc, s}} = { - def ss = s+s - tup{mc|advance{mc,s}, ss - (k &- (ss>k))} - } - {mc4, s4} := double{double{tup{sm0, s1}}} - # Submasks pick one mask out of a combination of 4 - def or_adv{m, s} = { m |= advance{m,s} } - @for (min{k/4 - 1, nw/4}) or_adv{sm0,s4} - submasks := scan{advance, tup{sm0, ...3**s1}} - mask_tail := advance{sm0, s4} &~ sm0 - # Mask out carry bit - mr := u64~~1<>(64-k); o=xo&1; xo=(xo&~1)|os - mask = mc4 - # Write result word given starting bits - def step{b, c} = output{c - b - promote{u64, c&mr != 0}} - def step{b, c, m} = step{b&m, c&m} - # Fast unrolled iterations - @for (k/4) { - each{step{x & mask, xo & mask, .}, submasks} - mask = advance{mask, s4} - } - # Single-step for tail - mask = mask_tail - @for (k%4) { - step{x, xo, mask} - mask = advance{mask, s1} - } - } - } -} - -fn rep_const_bool{if hasarch{'SSSE3'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - if (wv > 32) return{0} - if (wv&1 == 0) { - p := ctz{wv | 8} # Power of two for second replicate - if (wv>>p == 1) { - rep_const_bool_div8{wv, x, r, rlen} - } else { - tlen := rlen>>p - t := r + cdiv{rlen, 64} - cdiv{tlen, 64} - rep_const_bool{}(wv >>p, x, t, tlen) - rep_const_bool{}(usz~~1<floor{a/b}) infix left 40 def avx2 = hasarch{'AVX2'} def vl = if (avx2) 32 else 16 @@ -498,7 +372,7 @@ def rep_const_bool_div8{wv, x, r, rlen if hasarch{'SSSE3'}} = { # wv in 2,4,8 def selH = sel{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} def id{xv} = xv - def {output, flush} = get_boolvec_writer{V, r, rlen} + def {output, flush} = get_boolvec_writer{V, r, nw} def run24{x, proc_xv, exh} = { i:usz = 0; while (1) { @@ -535,29 +409,189 @@ def rep_const_bool_div8{wv, x, r, rlen if hasarch{'SSSE3'}} = { # wv in 2,4,8 flush{} } +# Data for the permutation that sends bit i to k*i % width{T} +def modperm_dat{T, k} = { + def w = width{T} + def lw= lb{w} + def i = iota{lw} + def bits = ~(1 & (k*iota{w} >> merge{0, replicate{1< make{T, each{base{2,.}, split{width{E}, bits}}} + {_} => T~~base{2, bits} + } +} +def modperm_step{x, l, m} = { + def d = (x ^ x<>l) +} +def modperm_step{x:T, l=(width{T}/2), m} = { + def mm = m | m>>l + (x &~ mm) | ((x<>l) & mm) # rotate +} +def modperm_step{x:T=[_](u8), l, m} = { + def W = re_el{u64, T} + T~~modperm_step{W~~x, l, W~~m} +} +def swap_elts{x:V, el_bytes if w128{V}} = { + def n = 16; def I = [n]u8 + def swi{len, l} = { def i = iota{len}; i + (l - 2*(i&l)) } + if (el_bytes >= 4) shuf{[4]u32, x, base{4, swi{4, el_bytes/4}}} + else sel{I, x, make{I, swi{16, el_bytes}}} +} +def modperm_step{x, l, m:V=[_]T if l%8==0} = { + (x & m) | (swap_elts{x, l/8} &~ m) +} +def modperm_get_byteperm{sw_bytes:V=([16]u8) if hasarch{'SSSE3'}} = { + def shW{op, v, s} = V~~op{re_el{u64,V}~~v, s} + m4 := V**0xf + t0 := fold{{v,a}=>modperm_step{v,...a}, iota{V}, tup{ + tup{4, sw_bytes &~ m4}, + tup{2, ({v} => v|shW{<<, v, 4}){sw_bytes&(V**0xc)}} + }} + t4 := shW{<<, t0&m4, 4} | shW{>>, t0&~m4, 4} + {xv} => sel{V, t0, xv & m4} | sel{V, t4, shW{>>, xv, 4} & m4} +} +def fold_multi{f, init, ...ls} = fold{{v,a}=>f{v,...a}, init, flip{ls}} + +def advance_spaced_mask{k, m, sh} = m<<(k-sh) | m>>sh + +# General-case loop for odd replication factors +def rep_const_bool_odd_mask4{ + M, # read/write type + k, # replication factor + get_modperm_x, # permuted input + output, n, # output, number of writes + mask:(u64), # starting mask + mask_sh # single iteration shift +} = { + def ifvec{g} = match (M) { {[_](u64)} => g; {_} => ({v}=>v) } + def scal = ifvec{{v} => M**v} + + # Fundamental operation: shifts act as order-k cyclic group on masks + def advance{m, sh} = advance_spaced_mask{k, m, sh} + # Double a cumulative mask, shift combination + # If s advances l iterations, mc combines iterations iota{l} + def double_gen{comb}{{mc, s}} = { + def mn = comb{mc, advance{mc,s}} + def ss = s+s + tup{mn, ss - (k &- (ss>k))} + } + # Mask and shift for one iteration + {sm0, s1} := ifvec{double_gen{make{M,...}}}{tup{mask, mask_sh}} + # Combined mask for 4 iterations, and shift to advance 4 + def double = double_gen{|} + {mc4, s4} := double{double{tup{sm0, s1}}} + # Submasks pick one mask out of a combination of 4 + def or_adv{m, s} = { m |= advance{m,s} } + @for (min{k/4 - 1, n/4}) or_adv{sm0,s4} + submasks := scan{advance, tup{sm0, ...3**s1}} + mask_tail := advance{sm0, s4} &~ sm0 + + # Carry: shifting and word-crossing is done on the initial permuted x + o:M = scal{0} # Carry for x + mr := scal{u64~~1< { + ca := if (hasarch{'SSE4.2'}) { def S = [l]i64; S~~c > S**0 } + else { def S = [2*l]i32; cm := S~~c; cm != shuf{S, cm, 4b2301} } + a + M~~ca + } + {_} => a - promote{u64, c != 0} + } + + while (1) { + x:M = get_modperm_x{} + def vrot1 = ifvec{{x} => vshl{x, x, vcount{type{x}}-1}} + k1:M = scal{1} + os:=o; xo:=x<>(64-k)}; o=xo&k1; xo=(xo&~k1)|os + # Write result word given starting bits + def step{b, c} = output{sub_carry{c - b, c & mr}} + def step{b, c, m} = step{b&m, c&m} + # Fast unrolled iterations + mask := mc4 + @for (k/4) { + each{step{x & mask, xo & mask, .}, submasks} + mask = advance{mask, s4} + } + # Single-step for tail + mask = mask_tail + @for (k%4) { + step{x, xo, mask} + mask = advance{mask, s1} + } + } +} + +def rep_const_bool_odd{k, xp, rp, nw} = { + # Every-k-bits mask + m:u64 = spaced_mask_of{k} + d := cast_i{usz, popc{m}} # == 64/k + mask_sh := cast_i{usz, ctz{m}} # == 64%k + mask := m<<(k-mask_sh) | 1 + + # Transform sending bit i to k*i % 64 by pairwise swaps + # Swap data goes in a pre-computed table + swtab:*u64 = each{modperm_dat{u64, .}, 1+2*iota{32}} + def swap_lens = reverse{2 << iota{5}} + swap_data := load{swtab, k>>1} + swsel:u64 = ~u64~~0 + def gsw{l} = { + swsel ^= swsel << l # Low l bits out of every 2*l + sm := swap_data &~ swsel + swap_data &= swsel; swap_data |= swap_data<f{x}, n**init} + def masks = reps{advance_spaced_mask{k, ., 64%k}, k, mask} + def step{x}{m, n_over} = { + b := x & m + def os = (64-k) + 1 + iota{n_over} + output{fold{|, (b<>{o,.}, os}}} + o = b + } + while (1) each{step{get_swap_x{}}, masks, (-iota{k}*64)%k} + } + unrolled_iter{3} + } else { + rep_const_bool_odd_mask4{u64, k, get_swap_x, output, nw, mask, mask_sh} + } +} + # For odd numbers: # - permute each byte sending bit i to position k*i % 8 # - replicate each byte by k, making position k*i contain bit i # - mask out those bits and spread over [ k*i, k*(i+1) ) # - ...except where it crosses words; handle this overhang separately -def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 +def rep_const_bool_odd{wv, x, r, nw if hasarch{'SSSE3'}} = { # wv odd, wv<=15 oper // ({a,b}=>floor{a/b}) infix left 40 def vl = 16; def V = [vl]u8 def iV = iota{vl} def mkV = make{V, .}; def selV = sel{V, ., .} def W = [2]u64 - def {output, flush} = get_boolvec_writer{V, r, rlen} + def {output, flush} = get_boolvec_writer{V, r, nw} + # Swap data goes in a pre-computed table + swtab:*V = each{modperm_dat{V, .}, 1+2*iota{32}} + swap_data := load{swtab, wv>>1} # Within-byte transformation - def get_ttab{k} = each{{is} => mkV{tr_iota{is}}, split{4, k*iota{8} % 8}} - ttab:*V = join{each{get_ttab, 2*iota{4} + 1}} - {t0, t4} := each{load{ttab + (wv & 6), .}, iota{2}} - m4 := V**0xf - def perm_x{xv} = { - selV{t0, xv & m4} | selV{t4, V~~([8]u16~~xv>>4) & m4} - } + def perm_x = modperm_get_byteperm{selV{swap_data, V**0}} - # Cases are 3; 5 7; 9 11 13 15 if (wv < 4) { # 3: dedicated loop i:usz = 0; while (1) { @@ -629,47 +663,16 @@ def rep_const_bool_ssse3_odd{wv, x, r, rlen} = { # wv odd, wv<=15 o := xo & mkV{255 * (iV%8 == 0)} # overhang output{rv | o} } - } else { # wv < 32 - # 9 to 31: extend k*i % 8 transform to k*i % 128 by pairwise swaps - # Swap data goes in a pre-computed table - def swdat{k} = { - def bits = (k*iota{128} >> merge{0, replicate{1<>1)-4} - swap_masks := each{{l} => selV{swap_data, mkV{l+iV%l}}, swap_lens} - # Every-k-bits mask, same as before - {m, d} := unaligned_spaced_mask_mod{wv} - mask := make{W, m, m>>d|m<<(wv-d)} - mask_sh := d+d; if (mask_sh >= wv) mask_sh-= wv - # Mask out carry bit - mr := [4]u32~~W**(u64~~1< selV{~swap_data, mkV{l+iV%l}}, swap_lens} + def swap_x = fold_multi{modperm_step, ., 8*swap_lens, swap_masks} i:usz = 0 - while (1) { - # Load xv, send bit i to position wv*i % 128 - xv = load{*V~~x, i}; ++i - def swap_step{l, m} = { - xv = (xv & m) | (selV{xv, mkV{iV + (l-2*(iV&l))}} &~ m) - } - each{swap_step, swap_lens, swap_masks} - xv = perm_x{xv} - xw := W~~xv - def vrot1{x} = vshl{x, x, vcount{type{x}}-1} - w1 := W**1 - os:=o; xo:=xw<>(64-wv)}; o=xo&w1; xo=(xo&~w1)|os - # Write wv vectors based on that - @for (wv) { - b := xw & mask - # Handle overhang here; won't fit in a single vector - c := xo & mask; cu := [4]u32~~c - output{V~~(W~~(cu + (mr&cu > [4]u32**0)) - b)} - mask = (mask << (wv - mask_sh)) | (mask >> mask_sh) - } - } + def get_swap_x{} = { xv := perm_x{swap_x{load{*V~~x, i}}}; ++i; xv } + # Every-k-bits mask + {m, d} := unaligned_spaced_mask_mod{wv} + rep_const_bool_odd_mask4{W, wv, {}=>W~~get_swap_x{}, {v}=>output{V~~v}, cdiv{nw, vcount{W}}, m, d} } flush{} } From 36e9ca58144c35e8b472f01cf643ac13b4204a66 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 11 Aug 2024 17:24:19 -0400 Subject: [PATCH 13/20] Extend odd/bool SSSE3 code to support AVX2 --- src/singeli/src/replicate.singeli | 127 ++++++++++++++++++------------ 1 file changed, 76 insertions(+), 51 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 351a3644..2cd0cc9e 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -300,7 +300,7 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } else { tlen := rlen>>p wq := usz~~1<

=52))) or (not hasarch{'AVX2'} and p == 1 and wv>=24)) { + if ((not hasarch{'SSSE3'} and (p == 1 or (p == 2 and wv>=52))) or (p == 1 and wv>=24)) { # Expanding odd second is faster tlen = rlen / wf t:=wf; wf=wq; wq=t @@ -412,8 +412,7 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSSE3'}} = { # Data for the permutation that sends bit i to k*i % width{T} def modperm_dat{T, k} = { def w = width{T} - def lw= lb{w} - def i = iota{lw} + def i = iota{lb{w}} def bits = ~(1 & (k*iota{w} >> merge{0, replicate{1< make{T, each{base{2,.}, split{width{E}, bits}}} @@ -432,24 +431,29 @@ def modperm_step{x:T=[_](u8), l, m} = { def W = re_el{u64, T} T~~modperm_step{W~~x, l, W~~m} } -def swap_elts{x:V, el_bytes if w128{V}} = { - def n = 16; def I = [n]u8 +def swap_elts{x:V=[vl](u8), el_bytes} = { # Reverse each pair of elements def swi{len, l} = { def i = iota{len}; i + (l - 2*(i&l)) } - if (el_bytes >= 4) shuf{[4]u32, x, base{4, swi{4, el_bytes/4}}} - else sel{I, x, make{I, swi{16, el_bytes}}} + if (el_bytes >= 4) { + def wd = max{4, el_bytes/2} + shuf{[4]ty_u{wd*8}, x, base{4, swi{4, el_bytes/wd}}} + } else { + def i = swi{16, el_bytes} + sel{[16]u8, x, make{V, cycle{vl, i}}} + } } def modperm_step{x, l, m:V=[_]T if l%8==0} = { (x & m) | (swap_elts{x, l/8} &~ m) } -def modperm_get_byteperm{sw_bytes:V=([16]u8) if hasarch{'SSSE3'}} = { +def modperm_get_byteperm{sw_bytes:V=[_](u8) if hasarch{'SSSE3'}} = { def shW{op, v, s} = V~~op{re_el{u64,V}~~v, s} m4 := V**0xf - t0 := fold{{v,a}=>modperm_step{v,...a}, iota{V}, tup{ + t0 := fold{{v,a}=>modperm_step{v,...a}, make{V,iota{vcount{V}}%16}, tup{ tup{4, sw_bytes &~ m4}, tup{2, ({v} => v|shW{<<, v, 4}){sw_bytes&(V**0xc)}} }} t4 := shW{<<, t0&m4, 4} | shW{>>, t0&~m4, 4} - {xv} => sel{V, t0, xv & m4} | sel{V, t4, shW{>>, xv, 4} & m4} + def selI = sel{[16]u8, ., .} + {xv} => selI{t0, xv & m4} | selI{t4, shW{>>, xv, 4} & m4} } def fold_multi{f, init, ...ls} = fold{{v,a}=>f{v,...a}, init, flip{ls}} @@ -477,7 +481,12 @@ def rep_const_bool_odd_mask4{ tup{mn, ss - (k &- (ss>k))} } # Mask and shift for one iteration - {sm0, s1} := ifvec{double_gen{make{M,...}}}{tup{mask, mask_sh}} + def fillmask{T} = match (T) { + {(u64)} => tup{mask, mask_sh} + {[2]E} => double_gen{make{T,...}}{fillmask{E}} + {[4]E} => double_gen{pair}{fillmask{[2]E}} + } + {sm0, s1} := fillmask{M} # Combined mask for 4 iterations, and shift to advance 4 def double = double_gen{|} {mc4, s4} := double{double{tup{sm0, s1}}} @@ -488,7 +497,8 @@ def rep_const_bool_odd_mask4{ mask_tail := advance{sm0, s4} &~ sm0 # Carry: shifting and word-crossing is done on the initial permuted x - o:M = scal{0} # Carry for x + # No need to carry across input words since they align with output words + # First bit of each word in xo below is wrong, but it doesn't matter! mr := scal{u64~~1< { @@ -501,9 +511,9 @@ def rep_const_bool_odd_mask4{ while (1) { x:M = get_modperm_x{} - def vrot1 = ifvec{{x} => vshl{x, x, vcount{type{x}}-1}} - k1:M = scal{1} - os:=o; xo:=x<>(64-k)}; o=xo&k1; xo=(xo&~k1)|os + def vrot1 = ifvec{{x} => if (w128{M}) vshl{x, x, vcount{type{x}}-1} + else shuf{M, x, 4b2103}} + xo := x<>(64-k)} # Write result word given starting bits def step{b, c} = output{sub_carry{c - b, c & mr}} def step{b, c, m} = step{b&m, c&m} @@ -524,10 +534,7 @@ def rep_const_bool_odd_mask4{ def rep_const_bool_odd{k, xp, rp, nw} = { # Every-k-bits mask - m:u64 = spaced_mask_of{k} - d := cast_i{usz, popc{m}} # == 64/k - mask_sh := cast_i{usz, ctz{m}} # == 64%k - mask := m<<(k-mask_sh) | 1 + {mask, mask_sh} := unaligned_spaced_mask_mod{k} # Transform sending bit i to k*i % 64 by pairwise swaps # Swap data goes in a pre-computed table @@ -553,19 +560,18 @@ def rep_const_bool_odd{k, xp, rp, nw} = { store{rp, j, rw} ++j; if (j==nw) return{1} } - o:u64 = 0 # carry # Dedicated loop for 3, shared for other factors - if (k == 3) { + if (k == 3) while (1) { def unrolled_iter{k} = { def reps{f, n, init} = scan{{x,_}=>f{x}, n**init} def masks = reps{advance_spaced_mask{k, ., 64%k}, k, mask} - def step{x}{m, n_over} = { + def step{x}{o, m, n_over} = { b := x & m def os = (64-k) + 1 + iota{n_over} output{fold{|, (b<>{o,.}, os}}} - o = b + b } - while (1) each{step{get_swap_x{}}, masks, (-iota{k}*64)%k} + fold_multi{step{get_swap_x{}}, 0, masks, (-iota{k}*64)%k} } unrolled_iter{3} } else { @@ -578,21 +584,45 @@ def rep_const_bool_odd{k, xp, rp, nw} = { # - replicate each byte by k, making position k*i contain bit i # - mask out those bits and spread over [ k*i, k*(i+1) ) # - ...except where it crosses words; handle this overhang separately -def rep_const_bool_odd{wv, x, r, nw if hasarch{'SSSE3'}} = { # wv odd, wv<=15 - oper // ({a,b}=>floor{a/b}) infix left 40 - def vl = 16; def V = [vl]u8 +def rep_const_bool_odd{k, x, r, nw if hasarch{'SSSE3'}} = { + def avx2 = hasarch{'AVX2'} + def vl = if (avx2) 32 else 16; def V = [vl]u8 def iV = iota{vl} - def mkV = make{V, .}; def selV = sel{V, ., .} - def W = [2]u64 - def {output, flush} = get_boolvec_writer{V, r, nw} + def mkV = make{V, .}; def selV = sel{[16]u8, ., .} + def W = re_el{u64, V} + def {output, flush} = get_boolvec_writer{W, r, nw} # Swap data goes in a pre-computed table swtab:*V = each{modperm_dat{V, .}, 1+2*iota{32}} - swap_data := load{swtab, wv>>1} + swap_data := load{swtab, k>>1} + swap_lane := if (avx2) shuf{[4]u64, swap_data, 4b1010} else swap_data # Within-byte transformation - def perm_x = modperm_get_byteperm{selV{swap_data, V**0}} + def perm_x = modperm_get_byteperm{selV{swap_lane, V**0}} + def sp_max = if (avx2) 4 else 8 + if (k < sp_max) { + rep_const_bool_odd_special{V, sp_max, k, x, perm_x, {v}=>output{W~~v}} + } else { + {m, d} := unaligned_spaced_mask_mod{k} + # General case + def get_mask{l} = selV{~swap_lane, mkV{l+iV%l}} + def get_mask{16} = shuf{[4]u64, ~swap_data, 4b3232} + def swap_lens = reverse{1 << iota{4 + avx2}} + swap_masks := each{get_mask, swap_lens} + def swap_x = fold_multi{modperm_step, ., 8*swap_lens, swap_masks} + i:usz = 0 + def get_swap_x{} = { xv := W~~perm_x{swap_x{load{*V~~x, i}}}; ++i; xv } + rep_const_bool_odd_mask4{W, k, get_swap_x, output, cdiv{nw, vcount{W}}, m, d} + } + flush{} +} - if (wv < 4) { +def rep_const_bool_odd_special{V=[vl](u8), max_wv, wv, x, perm_x, output} = { + oper // ({a,b}=>floor{a/b}) infix left 40 + def iV = iota{vl} + def mkV = make{V, .}; def selV = sel{[16]u8, ., .} + def W = re_el{u64, V} + if (max_wv <= 4 or wv < 4) { + def ll = 16; def dup{v} = if (vl>ll) merge{v,v} else v # 3: dedicated loop i:usz = 0; while (1) { # 01234567 to 05316427 on each byte @@ -601,24 +631,29 @@ def rep_const_bool_odd{wv, x, r, nw if hasarch{'SSSE3'}} = { # wv odd, wv<=15 def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word def ib = ix // 8 # byte index def io = 8*ib + 3*ix%8 # where they are in xv - def wi = split{2, tup{255, ...ib, 255, ...8+ib}} + def wi = split{vl/8, dup{tup{255, ...ib, 255, ...8+ib}}} xo := V~~((W~~xv & W**fold{|, 1<> (8-3)) xo += xo > V**0 # Permute and mask bytes def step{jj, oi, ind, mask} = { - b := W~~(selV{xv, ind} & mask) + def getv = if (vl==ll or jj==1) ({x}=>x) + else shuf{[4]u64, ., 4b1010 + 4b2222*(jj>1)} + def selx{x, i} = sel{[ll]u8, getv{x}, i} + b := W~~(selx{xv, ind} & mask) r := V~~((b<<3) - b) - o := selV{xo, mkV{flat_table{max, oi, 255*(0 selV{~swap_data, mkV{l+iV%l}}, swap_lens} - def swap_x = fold_multi{modperm_step, ., 8*swap_lens, swap_masks} - i:usz = 0 - def get_swap_x{} = { xv := perm_x{swap_x{load{*V~~x, i}}}; ++i; xv } - # Every-k-bits mask - {m, d} := unaligned_spaced_mask_mod{wv} - rep_const_bool_odd_mask4{W, wv, {}=>W~~get_swap_x{}, {v}=>output{V~~v}, cdiv{nw, vcount{W}}, m, d} } - flush{} } + export{'si_constrep_bool', rep_const_bool{}} From ee27717c970af8dee37f8c2d6f1c0d284dacd6c8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 13 Aug 2024 11:42:12 -0400 Subject: [PATCH 14/20] SSE2 odd/bool implementation using [4]u32 shuffles --- src/singeli/src/replicate.singeli | 330 +++++++++++++++++------------- 1 file changed, 183 insertions(+), 147 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 2cd0cc9e..be48bfb6 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -347,23 +347,38 @@ def get_boolvec_writer{V, r, nw} = { def vwords = width{V}/64 rv := *V~~r re := rv + nw / vwords + rs := rv + cdiv{nw, vwords} + # Avoid reading/processing an extra word past the end of input + def done = makelabel{} + def check_done{} = if (rv == rs) goto{done} + # If the last result is partial, jump to flush to write it last_res:V = V**0 - def end = makelabel{} + def l_flush = makelabel{} def output{v:(V)} = { last_res = v - if (rv==re) goto{end} + if (rv == re) goto{l_flush} store{rv, 0, v}; ++rv } def flush{} = { - setlabel{end} + setlabel{l_flush} q := nw & (vwords-1) if (q != 0) homMaskStoreF{rv, V~~maskOf{re_el{u64,V}, q}, last_res} + setlabel{done} } - tup{output, flush} + tup{output, check_done, flush} +} +def get_boolvec_writer{T=(u64), r:*T, nw} = { + def done = makelabel{} + j:usz = 0 + def output{rw} = { + store{r, j, rw} + ++j; if (j==nw) goto{done} + } + def flush{} = setlabel{done} + tup{output, {}=>{}, flush} } def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSSE3'}} = { - oper // ({a,b}=>floor{a/b}) infix left 40 def avx2 = hasarch{'AVX2'} def vl = if (avx2) 32 else 16 def V = [vl]u8 @@ -372,7 +387,7 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSSE3'}} = { def selH = sel{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} def id{xv} = xv - def {output, flush} = get_boolvec_writer{V, r, nw} + def {output, done, flush} = get_boolvec_writer{V, r, nw} def run24{x, proc_xv, exh} = { i:usz = 0; while (1) { @@ -409,6 +424,19 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSSE3'}} = { flush{} } +def sel_imm{V=[4]T, x, {...inds} if hasarch{'SSE2'} and (T==u32 or T==u64)} = { + assert{length{inds} == 4} + shuf{V, x, base{4, inds}} +} +def sel_imm{V=([16]u8), x:X, {...inds}} = { + def I = re_el{u8,X}; def [n]_ = I + def l = length{inds} + assert{l==16 or l==n} + sel{V, x, make{I, cycle{n, inds}}} +} + +def advance_spaced_mask{k, m, sh} = m<<(k-sh) | m>>sh + # Data for the permutation that sends bit i to k*i % width{T} def modperm_dat{T, k} = { def w = width{T} @@ -419,45 +447,119 @@ def modperm_dat{T, k} = { {_} => T~~base{2, bits} } } -def modperm_step{x, l, m} = { +# Permutation step evaluators +# shift takes a top-half mask; others take both-halves +def modperm_shift_step{x, l, m} = { def d = (x ^ x<>l) } -def modperm_step{x:T, l=(width{T}/2), m} = { - def mm = m | m>>l - (x &~ mm) | ((x<>l) & mm) # rotate +def modperm_rot_step{x:T, l=(width{T}/2), m} = { + (x &~ m) | ((x<>l) & m) } -def modperm_step{x:T=[_](u8), l, m} = { - def W = re_el{u64, T} - T~~modperm_step{W~~x, l, W~~m} +def modperm_shuf_step{x:V=[_]T, l, m if l%8==0} = { + (x &~ m) | (swap_elts{x, l/8} & m) } -def swap_elts{x:V=[vl](u8), el_bytes} = { # Reverse each pair of elements - def swi{len, l} = { def i = iota{len}; i + (l - 2*(i&l)) } +# Reverse each pair of elements +def swap_elts{x:V=[_]_, el_bytes} = { + def selx{n, wd} = { + def l = el_bytes/wd + def i = iota{n} + def rev = i + (l - 2*(i&l)) + sel_imm{[n]ty_u{wd*8}, x, rev} + } if (el_bytes >= 4) { - def wd = max{4, el_bytes/2} - shuf{[4]ty_u{wd*8}, x, base{4, swi{4, el_bytes/wd}}} + selx{4, max{4, el_bytes/2}} + } else if (hasarch{'SSSE3'}) { + selx{16, 1} } else { - def i = swi{16, el_bytes} - sel{[16]u8, x, make{V, cycle{vl, i}}} + def sh = el_bytes*8 + xe := re_el{ty_u{2*sh}, V} ~~ x + V~~(xe<>sh) } } -def modperm_step{x, l, m:V=[_]T if l%8==0} = { - (x & m) | (swap_elts{x, l/8} &~ m) +# Extract swap functions from modperm_dat +def extract_modperm_mask{full:W, lane, l} = { + def f = l == 128 # Use full width + def w = if (l<32) 8 else if (f) 64 else 32 # Shuffle element width + def n = (if (f) 256 else 128) / w # and number of elements + def o = l / w # Extract [o,2*o) + def i = o + iota{n}%o + sel_imm{[n]ty_u{w}, if (f) full else lane, i} } -def modperm_get_byteperm{sw_bytes:V=[_](u8) if hasarch{'SSSE3'}} = { - def shW{op, v, s} = V~~op{re_el{u64,V}~~v, s} - m4 := V**0xf - t0 := fold{{v,a}=>modperm_step{v,...a}, make{V,iota{vcount{V}}%16}, tup{ - tup{4, sw_bytes &~ m4}, - tup{2, ({v} => v|shW{<<, v, 4}){sw_bytes&(V**0xc)}} - }} - t4 := shW{<<, t0&m4, 4} | shW{>>, t0&~m4, 4} - def selI = sel{[16]u8, ., .} - {xv} => selI{t0, xv & m4} | selI{t4, shW{>>, xv, 4} & m4} +def proc_mod_dat{swap_data:W} = { + def ww = width{W}; def avx2 = ww==256 + def on_len_range{get_swap, lo, hi} = { + def lens = reverse{1 << slice{iota{lb{hi}}, lb{lo}}} + fold{{x,s}=>s{x}, ., each{get_swap, lens}} + } + swap_lane := if (avx2) shuf{W, swap_data, 4b1010} else swap_data + # Transform width-w units with shifts only + def get_shiftperm{data, w} = { + dat := data + bot:W = W**base{2, cycle{64, replicate{w, tup{1,0}}}} + def gsw{l} = { + bot ^= bot << l # Low l bits out of every 2*l + sm := dat &~ bot + dat &= bot; dat |= dat<>l} + } + on_len_range{gsw, 2, w} + } + # Within-byte transformation + def get_byteperm{} = { + def V = re_el{u8, W} + def sw_bytes = sel{[16]u8, swap_lane, V**0} + m4 := W~~V**0xf + t0 := fold{{v,a}=>modperm_shift_step{v,...a}, W~~make{V,iota{vcount{V}}%16}, tup{ + tup{4, sw_bytes &~ m4}, + tup{2, ({v} => v | v<<4){sw_bytes & W~~V**0xc}} + }} + t4 := (t0&m4)<<4 | (t0&~m4)>>4 + def selI{v,i} = sel{[16]u8, v, V~~i} + {xv} => selI{t0, xv & m4} | selI{t4, (xv>>4) & m4} + } + def {partwidth, partperm} = { + def sh{w, d} = tup{w, get_shiftperm{d, w}} + if (ww==64) sh{64, swap_data} + else if (not hasarch{'SSSE3'}) sh{32, shuf{[4]u32, swap_data, 4b0000}} + else tup{8, get_byteperm{}} + } + # Fill in higher steps + def get_mod_permuter{} = { + def get_swap{l} = { + def mask = extract_modperm_mask{swap_data, swap_lane, l} + modperm_shuf_step{., l, mask} + } + def swap_x = on_len_range{get_swap, partwidth, ww} + {x} => partperm{swap_x{x}} + } + tup{partperm, get_mod_permuter} } -def fold_multi{f, init, ...ls} = fold{{v,a}=>f{v,...a}, init, flip{ls}} -def advance_spaced_mask{k, m, sh} = m<<(k-sh) | m>>sh +def rep_const_bool_odd{k, x, r, nw} = { + def avx2 = hasarch{'AVX2'} + def W = if (hasarch{'SSE2'}) [if (avx2) 4 else 2]u64 else u64 + + def {output, check_done, flush} = get_boolvec_writer{W, r, nw} + xp := *W~~x + def getter{perm}{} = { check_done{}; xv := load{xp}; ++xp; perm{xv} } + + # Modular permutation: small-k cases may use a limited permutation + # on bytes or 32-bit ints; general case uses the whole thing + swtab:*W = each{modperm_dat{W, .}, 1+2*iota{32}} + swap_data := load{swtab, k>>1} + def {partperm, get_full_permute} = proc_mod_dat{swap_data} + + def sp_max = if (hasarch{'SSSE3'} and not avx2) 8 else 4 + if (k < sp_max) { + rep_const_bool_small_odd{W, sp_max, k, getter{partperm}, output} + } else { + def get_swap_x = getter{get_full_permute{}} + rep_const_bool_odd_mask4{W, k, get_swap_x, output, cdiv{nw, vcount{W}}} + } + flush{} +} # General-case loop for odd replication factors def rep_const_bool_odd_mask4{ @@ -465,8 +567,6 @@ def rep_const_bool_odd_mask4{ k, # replication factor get_modperm_x, # permuted input output, n, # output, number of writes - mask:(u64), # starting mask - mask_sh # single iteration shift } = { def ifvec{g} = match (M) { {[_](u64)} => g; {_} => ({v}=>v) } def scal = ifvec{{v} => M**v} @@ -480,6 +580,8 @@ def rep_const_bool_odd_mask4{ def ss = s+s tup{mn, ss - (k &- (ss>k))} } + # Starting word mask and single-word shift + {mask, mask_sh} := unaligned_spaced_mask_mod{k} # Mask and shift for one iteration def fillmask{T} = match (T) { {(u64)} => tup{mask, mask_sh} @@ -511,7 +613,7 @@ def rep_const_bool_odd_mask4{ while (1) { x:M = get_modperm_x{} - def vrot1 = ifvec{{x} => if (w128{M}) vshl{x, x, vcount{type{x}}-1} + def vrot1 = ifvec{{x} => if (w128{M}) shuf{[4]u32, x, 4b1032} else shuf{M, x, 4b2103}} xo := x<>(64-k)} # Write result word given starting bits @@ -532,131 +634,66 @@ def rep_const_bool_odd_mask4{ } } -def rep_const_bool_odd{k, xp, rp, nw} = { - # Every-k-bits mask - {mask, mask_sh} := unaligned_spaced_mask_mod{k} - - # Transform sending bit i to k*i % 64 by pairwise swaps - # Swap data goes in a pre-computed table - swtab:*u64 = each{modperm_dat{u64, .}, 1+2*iota{32}} - def swap_lens = reverse{2 << iota{5}} - swap_data := load{swtab, k>>1} - swsel:u64 = ~u64~~0 - def gsw{l} = { - swsel ^= swsel << l # Low l bits out of every 2*l - sm := swap_data &~ swsel - swap_data &= swsel; swap_data |= swap_data<f{x}, n**init} - def masks = reps{advance_spaced_mask{k, ., 64%k}, k, mask} - def step{x}{o, m, n_over} = { - b := x & m - def os = (64-k) + 1 + iota{n_over} - output{fold{|, (b<>{o,.}, os}}} - b - } - fold_multi{step{get_swap_x{}}, 0, masks, (-iota{k}*64)%k} - } - unrolled_iter{3} - } else { - rep_const_bool_odd_mask4{u64, k, get_swap_x, output, nw, mask, mask_sh} +def rep_const_bool_small_odd{(u64), 4, wv, get_swap_x, output} = { + def k = 3 + def step{x}{o, n_over} = { + b := x & base{2, cycle{64, n_over == iota{k}}} + def os = (64-k) + 1 + iota{n_over} + output{fold{|, (b<>{o,.}, os}}} + b } + while (1) fold{step{get_swap_x{}}, 0, (-iota{k}*64)%k} } -# For odd numbers: +# For small odd numbers: # - permute each byte sending bit i to position k*i % 8 # - replicate each byte by k, making position k*i contain bit i # - mask out those bits and spread over [ k*i, k*(i+1) ) # - ...except where it crosses words; handle this overhang separately -def rep_const_bool_odd{k, x, r, nw if hasarch{'SSSE3'}} = { - def avx2 = hasarch{'AVX2'} - def vl = if (avx2) 32 else 16; def V = [vl]u8 - def iV = iota{vl} - def mkV = make{V, .}; def selV = sel{[16]u8, ., .} - def W = re_el{u64, V} - def {output, flush} = get_boolvec_writer{W, r, nw} - - # Swap data goes in a pre-computed table - swtab:*V = each{modperm_dat{V, .}, 1+2*iota{32}} - swap_data := load{swtab, k>>1} - swap_lane := if (avx2) shuf{[4]u64, swap_data, 4b1010} else swap_data - # Within-byte transformation - def perm_x = modperm_get_byteperm{selV{swap_lane, V**0}} - def sp_max = if (avx2) 4 else 8 - if (k < sp_max) { - rep_const_bool_odd_special{V, sp_max, k, x, perm_x, {v}=>output{W~~v}} - } else { - {m, d} := unaligned_spaced_mask_mod{k} - # General case - def get_mask{l} = selV{~swap_lane, mkV{l+iV%l}} - def get_mask{16} = shuf{[4]u64, ~swap_data, 4b3232} - def swap_lens = reverse{1 << iota{4 + avx2}} - swap_masks := each{get_mask, swap_lens} - def swap_x = fold_multi{modperm_step, ., 8*swap_lens, swap_masks} - i:usz = 0 - def get_swap_x{} = { xv := W~~perm_x{swap_x{load{*V~~x, i}}}; ++i; xv } - rep_const_bool_odd_mask4{W, k, get_swap_x, output, cdiv{nw, vcount{W}}, m, d} - } - flush{} -} - -def rep_const_bool_odd_special{V=[vl](u8), max_wv, wv, x, perm_x, output} = { - oper // ({a,b}=>floor{a/b}) infix left 40 - def iV = iota{vl} - def mkV = make{V, .}; def selV = sel{[16]u8, ., .} - def W = re_el{u64, V} +# If 1-byte shuffle isn't available, use 4-byte units instead +def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = { + def ov_bytes{o} = { def V = re_el{u8, W}; v := V~~o; v += v > V**0; W~~v } if (max_wv <= 4 or wv < 4) { - def ll = 16; def dup{v} = if (vl>ll) merge{v,v} else v + def ww = width{W} + def ew = if (hasarch{'SSSE3'}) 8 else 32 # width of shuffle-able elements + def ne = ww/ew; def se = 64/ew + def lanes = ww > 128 + def dup{v} = if (lanes) merge{v,v} else v # 3: dedicated loop - i:usz = 0; while (1) { + while (1) { # 01234567 to 05316427 on each byte - xv := perm_x{load{*V~~x, i}}; ++i + xv := get_perm_x{} # Overhang from previous 64-bit elements def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word - def ib = ix // 8 # byte index - def io = 8*ib + 3*ix%8 # where they are in xv - def wi = split{vl/8, dup{tup{255, ...ib, 255, ...8+ib}}} - xo := V~~((W~~xv & W**fold{|, 1<> (8-3)) - xo += xo > V**0 + def ib = ix // ew # byte index + def io = ew*ib + 3*ix%ew # where they are in xv + def wi = split{wl, dup{tup{255, ...ib, 255, ...se+ib}}} + xo := ov_bytes{(xv & W**fold{|, 1<> (ew-3)} # Permute and mask bytes def step{jj, oi, ind, mask} = { - def getv = if (vl==ll or jj==1) ({x}=>x) + def getv = if (not lanes or jj==1) ({x}=>x) else shuf{[4]u64, ., 4b1010 + 4b2222*(jj>1)} - def selx{x, i} = sel{[ll]u8, getv{x}, i} - b := W~~(selx{xv, ind} & mask) - r := V~~((b<<3) - b) - o := selx{xo, mkV{flat_table{max, oi, 255*(0=vl ind_up = mkV{iu - k*ia} ind_inc = mkV{vl//k + ia} mask_sh = width{V} % k } if (wv == 5) set_consts{5} else set_consts{7} - xv:V = V**0; xo:=xv; ind:=xv; mask:=xv # state - i:usz = 0; q:usz = 1 + xv:=W**0; xo:=xv; mask:=xv; ind:=V**0 # state + q:usz = 1 while (1) { --q; if (q == 0) { q = wv # Load and permute bytes - xv = perm_x{load{*V~~x, i}}; ++i + xv = get_perm_x{} # Bytes for overhang - xo = V~~((W~~xv & xom) >> (8 - wv)) - xo += xo > V**0 + xo = ov_bytes{(xv & xom) >> (8 - wv)} xo = selV{xo, xse} # Initialize state vectors ind = ind0 @@ -691,11 +727,11 @@ def rep_const_bool_odd_special{V=[vl](u8), max_wv, wv, x, perm_x, output} = { # Update state vectors xo = shr{V, xo, 1} ind = selV{ind, ind_up} + ind_inc - mask = V~~((W~~mask << (wv - mask_sh)) | (W~~mask >> mask_sh)) + mask = advance_spaced_mask{wv, mask, mask_sh} } - b := W~~(selV{xv, ind} & mask) - rv:= V~~((b< Date: Tue, 13 Aug 2024 15:59:48 -0400 Subject: [PATCH 15/20] SSE2 k/bool where k divides 8 implementations --- src/singeli/src/replicate.singeli | 86 +++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index be48bfb6..0071bdfd 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -300,7 +300,7 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } else { tlen := rlen>>p wq := usz~~1<

=52))) or (p == 1 and wv>=24)) { + if (p == 1 and (not hasarch{'SSSE3'} or wv>=24)) { # Expanding odd second is faster tlen = rlen / wf t:=wf; wf=wq; wq=t @@ -378,49 +378,81 @@ def get_boolvec_writer{T=(u64), r:*T, nw} = { tup{output, {}=>{}, flush} } -def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSSE3'}} = { +def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSE2'}} = { def avx2 = hasarch{'AVX2'} def vl = if (avx2) 32 else 16 def V = [vl]u8 def iV = iota{vl} def mkV = make{V, .} - def selH = sel{[16]u8, ., .} - def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} - def id{xv} = xv - def {output, done, flush} = get_boolvec_writer{V, r, nw} + # Not the same as a 1-byte shift but extra bits are always masked away + def H = re_el{u16, V} + def __shl{x:(V), a} = V~~(H~~x << a) + def __shr{x:(V), a} = V~~(H~~x >> a) - def run24{x, proc_xv, exh} = { + def {output, done, flush} = get_boolvec_writer{V, r, nw} + def run24{x, get_halves} = { i:usz = 0; while (1) { - xv := proc_xv{load{*V~~(x+i)}}; ++i - # Store 1 or 2 result vectors - def getr = zip128{exh{xv}, exh{V~~(re_el{u16,V}~~xv>>4)}, .} + xv := load{*V~~(x+i)}; ++i + def getr = zip128{...get_halves{xv}, .} output{V~~getr{0}} output{V~~getr{1}} } } - if (wv == 2) { - def init = if (avx2) shuf{[4]u64, ., 4b3120} else id - # Expander for half byte - def tabr = makeTab{tr_iota{2*iota{4}} * 2b11} - m4 := V**0xf - run24{*V~~x, init, {x} => tabr{x & m4}} - } else if (wv == 4) { - # Unzip 32-bit elements (result lanes) across AVX2 lanes - def pre = if (avx2) sel{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id - def init{xv} = { u:=pre{xv}; zip128{u,u,0} } - # Expander for two bits in either bottom or next-to-bottom position - def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111} - m2 := mkV{2b11 << (2*(iV%2))} - def exh{x} = re_el{u16, V}~~tabr{x & m2} - run24{*(if (avx2) [2]u64 else u64)~~x, init, exh} - } else { # wv == 8 + def run24{x, pre, exh, sh} = run24{x, + {xv} => { p := pre{xv}; tup{exh{p}, exh{p>>sh}} } + } + def run8{rep_bytes} = { i:usz = 0; while (1) { xh := load{*[16]u8~~(*ty_u{vl}~~x + i)}; ++i xv := if (avx2) pair{xh, xh} else xh - xe := selH{xv, mkV{iV // 8}} + xe := rep_bytes{xv} output{(xe & mkV{1 << (iV % 8)}) > V**0} } } + if (not hasarch{'SSSE3'}) { + if (wv == 2) { + run24{*V~~x, {xv} => { + def swap{x, l, m} = { def d = (x ^ x>>l) & V**m; x ^ (d|d<>1 + tup{o0, o1} + }} + } else if (wv == 4) { + def pre{xv} = { + a := xv ^ ((xv & V**0x55) << 1) + e := zip128{a, a>>4, 0} + } + def out{h} = (-(h & V**1)) ^ (-(h<<3 & V**0x10)) + run24{*u64~~x, pre, out, 2} + } else { # wv == 8 + def z{x} = zip{x,x,0} + run8{{x} => z{z{z{x}}}} + } + } else { # hasarch{'SSSE3'} + def selH = sel{[16]u8, ., .} + def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} + def id{xv} = xv + if (wv == 2) { + def init = if (avx2) shuf{[4]u64, ., 4b3120} else id + # Expander for half byte + def tabr = makeTab{tr_iota{2*iota{4}} * 2b11} + m4 := V**0xf + run24{*V~~x, init, {x} => tabr{x & m4}, 4} + } else if (wv == 4) { + # Unzip 32-bit elements (result lanes) across AVX2 lanes + def pre = if (avx2) sel{[8]u32, ., make{[8]u32,tr_iota{1,2,0}}} else id + def init{xv} = { u:=pre{xv}; zip128{u,u,0} } + # Expander for two bits in either bottom or next-to-bottom position + def tabr = makeTab{tr_iota{0,4,0,4} * 2b1111} + m2 := mkV{2b11 << (2*(iV%2))} + def exh{x} = re_el{u16, V}~~tabr{x & m2} + run24{*(if (avx2) [2]u64 else u64)~~x, init, exh, 4} + } else { # wv == 8 + run8{selH{., mkV{iV // 8}}} + } + } flush{} } From fc1265b3f2001750ed8dfa542d282bbbc5b4e667 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 13 Aug 2024 20:12:19 -0400 Subject: [PATCH 16/20] Enable (non-boolean) constrep on SSSE3 --- src/singeli/src/replicate.singeli | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 0071bdfd..1a311fa3 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -59,7 +59,7 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}} # Constant replicate -if_inline (not (hasarch{'AVX2'} or hasarch{'AARCH64'})) { +if_inline (not (hasarch{'SSSE3'} or hasarch{'AARCH64'})) { fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}} @@ -67,6 +67,8 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { } else { +def has_bytesel_128 = not hasarch{'AVX2'} + def incl{a,b} = slice{iota{b+1},a} # 1+˝∨`⌾⌽0=div|⌜range @@ -80,8 +82,6 @@ def fact_inds = slice{iota{fact_size},8} def fact_tab = makefact{basic_rep, fact_inds} factors:*u8 = fact_tab - - def sdtype = [arch_defvw/8]i8 # shuf data type def get_shufs{step, wv, len} = { def i = iota{len*step} @@ -91,7 +91,7 @@ def get_shuf_data{wv, len} = get_shufs{vcount{sdtype}, wv, len} # [len] byte-sel def get_shuf_data{wv} = get_shuf_data{wv, wv} # all shuffle vectors for 𝕨≤7 -def special_2 = ~hasarch{'AARCH64'} # handle 2 specially on x86-64 +def special_2 = not has_bytesel_128 # handle 2 specially on AVX2 def rcsh_vals = slice{basic_rep, special_2} rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}} @@ -107,7 +107,7 @@ def read_shuf_vecs{l, ellw:(u64), shp:*V} = { # tuple of byte selectors in 1< gen{sel{[16]u8, x, s}}, sh} @@ -192,7 +192,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { def wvb = wv << ellw def hs = (h*step) / wvb # Actual step size in argument elements def shufbase{i if hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010} - def shufbase{i if hasarch{'AARCH64'}} = load{*V~~(x+i)} + def shufbase{i if has_bytesel_128} = load{*V~~(x+i)} def shufrun{a, s} = sel{[16]i8, a, s} # happens to be the same across AVX2 & NEON i:u64 = 0 From bf69705c82c647674efccda3a3a6bcf0e1870fea Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 13 Aug 2024 20:52:30 -0400 Subject: [PATCH 17/20] =?UTF-8?q?Factor=20(8=C3=97k)/bool=20as=20bit-repli?= =?UTF-8?q?cate,=20then=20byte-replicate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 2 +- src/singeli/src/replicate.singeli | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 98810542..f108c7f8 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -768,7 +768,7 @@ B slash_c2(B t, B w, B x) { u64* xp = bitarr_ptr(x); u64* rp; r = m_bitarrv(&rp, s); #if SINGELI - if (si_constrep_bool(wv, xp, rp, s)) ; // Handles small wv + if (wv <= 64) si_constrep_bool(wv, xp, rp, s); else #endif if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) } diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 1a311fa3..c96fd248 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -59,6 +59,9 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}} # Constant replicate +def incl{a,b} = slice{iota{b+1},a} +def basic_rep = incl{2, 7} + if_inline (not (hasarch{'SSSE3'} or hasarch{'AARCH64'})) { fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { @@ -69,14 +72,11 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { def has_bytesel_128 = not hasarch{'AVX2'} -def incl{a,b} = slice{iota{b+1},a} - # 1+˝∨`⌾⌽0=div|⌜range def makefact{divisor, range} = { def t = table{{a,b}=>0==b%a, divisor, range} fold{+, 1, reverse{scan{|, reverse{t}}}} } -def basic_rep = incl{2, 7} def fact_size = 128 def fact_inds = slice{iota{fact_size},8} def fact_tab = makefact{basic_rep, fact_inds} @@ -289,17 +289,23 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean -fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { - if (wv > 64) return{0} +fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : void = { + assert{wv >= 2}; assert{wv <= 64} nw := cdiv{rlen, 64} if (wv&1 == 0) { p := ctz{wv | 8} # Power of two for second replicate wf := wv>>p if (wf == 1) { rep_const_bool_div8{wv, x, r, nw} + } else if (hasarch{'SSSE3'} and p == 3 and wv <= 8*select{basic_rep, -1}) { + # (higher wv double-factors, which doesn't work with in-place pointers) + tlen := rlen / wf + t := r + cdiv{rlen, 64} - cdiv{tlen, 64} + rep_const_bool{}(8, x, t, tlen) + rep_const{select{dat_types,0}}(promote{u64,wf}, *void~~t, *void~~r, promote{u64,tlen/8}) } else { - tlen := rlen>>p - wq := usz~~1<

> p + wq := usz~~1 << p if (p == 1 and (not hasarch{'SSSE3'} or wv>=24)) { # Expanding odd second is faster tlen = rlen / wf @@ -312,7 +318,6 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = { } else { rep_const_bool_odd{wv, x, r, nw} } - 1 } def rep_const_bool_div8{wv, x, r, nw} = { # wv in 2,4,8 From b2758d355c709f44805ef9dfbcf985a8cf9019ba Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 14 Aug 2024 09:34:19 -0400 Subject: [PATCH 18/20] k/bool cleanup and implementation comments --- src/builtins/slash.c | 9 +++++++-- src/singeli/src/replicate.singeli | 8 +++----- src/singeli/src/spaced.singeli | 2 ++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index f108c7f8..5e883113 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -38,8 +38,13 @@ // COULD consolidate refcount updates for nested 𝕩 // Replicate by constant -// Boolean uses pdep, ≠`, or overwriting -// SHOULD make a shift/mask replacement for pdep +// Boolean uses specialized small-𝕨 methods, ≠`, or overwriting +// 𝕨≤64: Singeli generic and SIMD methods +// 𝕨=2,4,8: Various shift, shuffle, and zip-based loops +// odd 𝕨: Modular permutation +// COULD use pdep or similar to avoid overhead on small results +// Otherwise, factor into power of 2 times odd +// COULD fuse 2×odd, since 2/odd/ has a larger intermediate // Other typed 𝕩 uses +`, or lots of Singeli // Fixed shuffles, factorization, partial shuffles, self-overlapping // Otherwise, cell-by-cell copying diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index c96fd248..4759908b 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -394,9 +394,9 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSE2'}} = { def __shl{x:(V), a} = V~~(H~~x << a) def __shr{x:(V), a} = V~~(H~~x >> a) - def {output, done, flush} = get_boolvec_writer{V, r, nw} + def {output, check_done, flush} = get_boolvec_writer{V, r, nw} def run24{x, get_halves} = { - i:usz = 0; while (1) { + i:usz = 0; while (1) { check_done{} xv := load{*V~~(x+i)}; ++i def getr = zip128{...get_halves{xv}, .} output{V~~getr{0}} @@ -407,7 +407,7 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSE2'}} = { {xv} => { p := pre{xv}; tup{exh{p}, exh{p>>sh}} } } def run8{rep_bytes} = { - i:usz = 0; while (1) { + i:usz = 0; while (1) { check_done{} xh := load{*[16]u8~~(*ty_u{vl}~~x + i)}; ++i xv := if (avx2) pair{xh, xh} else xh xe := rep_bytes{xv} @@ -472,8 +472,6 @@ def sel_imm{V=([16]u8), x:X, {...inds}} = { sel{V, x, make{I, cycle{n, inds}}} } -def advance_spaced_mask{k, m, sh} = m<<(k-sh) | m>>sh - # Data for the permutation that sends bit i to k*i % width{T} def modperm_dat{T, k} = { def w = width{T} diff --git a/src/singeli/src/spaced.singeli b/src/singeli/src/spaced.singeli index ff477cb3..acb926a0 100644 --- a/src/singeli/src/spaced.singeli +++ b/src/singeli/src/spaced.singeli @@ -12,3 +12,5 @@ def unaligned_spaced_mask_mod{l:T} = { def d = cast_i{T, ctz{m}} # = 64%l tup{m>>d | m<<(l-d), d} } + +def advance_spaced_mask{k, m, sh} = m<<(k-sh) | m>>sh From b801c7c1863945d7e72bea609db676592fa1b352 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 14 Aug 2024 10:20:21 -0400 Subject: [PATCH 19/20] Move to dedicated k/bool loops for k=5,7 and at least SSSE3 --- src/singeli/src/replicate.singeli | 90 +++++++++---------------------- 1 file changed, 25 insertions(+), 65 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 4759908b..6b65d7dc 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -586,7 +586,7 @@ def rep_const_bool_odd{k, x, r, nw} = { swap_data := load{swtab, k>>1} def {partperm, get_full_permute} = proc_mod_dat{swap_data} - def sp_max = if (hasarch{'SSSE3'} and not avx2) 8 else 4 + def sp_max = if (hasarch{'SSSE3'}) 8 else 4 if (k < sp_max) { rep_const_bool_small_odd{W, sp_max, k, getter{partperm}, output} } else { @@ -688,87 +688,47 @@ def rep_const_bool_small_odd{(u64), 4, wv, get_swap_x, output} = { # If 1-byte shuffle isn't available, use 4-byte units instead def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = { def ov_bytes{o} = { def V = re_el{u8, W}; v := V~~o; v += v > V**0; W~~v } - if (max_wv <= 4 or wv < 4) { - def ww = width{W} - def ew = if (hasarch{'SSSE3'}) 8 else 32 # width of shuffle-able elements - def ne = ww/ew; def se = 64/ew - def lanes = ww > 128 - def dup{v} = if (lanes) merge{v,v} else v - # 3: dedicated loop + def ww = width{W} + def ew = if (hasarch{'SSSE3'}) 8 else 32 # width of shuffle-able elements + def ne = ww/ew; def se = 64/ew + def lanes = ww > 128 + def dup{v} = if (lanes) merge{v,v} else v + def fixed_loop{k} = { + assert{wv == k} while (1) { - # 01234567 to 05316427 on each byte + # e.g. 01234567 to 05316427 on each byte for k==3, ew==8 xv := get_perm_x{} # Overhang from previous 64-bit elements - def ix = 64*slice{iota{3},1} // 3 # bits that overhang within a word + def ix = 64*slice{iota{k},1} // k # bits that overhang within a word def ib = ix // ew # byte index - def io = ew*ib + 3*ix%ew # where they are in xv + def io = ew*ib + k*ix%ew # where they are in xv def wi = split{wl, dup{tup{255, ...ib, 255, ...se+ib}}} - xo := ov_bytes{(xv & W**fold{|, 1<> (ew-3)} + xo := ov_bytes{(xv & W**fold{|, 1<> (ew-k)} # Permute and mask bytes def step{jj, oi, ind, mask} = { - def getv = if (not lanes or jj==1) ({x}=>x) - else shuf{[4]u64, ., 4b1010 + 4b2222*(jj>1)} + def hk = (k-1) / 2 + def getv = if (not lanes or jj==hk) ({x}=>x) + else sel_imm{[4]u64, ., 2*(jj>hk) + iota{4}%2} def selx{x, i} = sel_imm{[128/ew]ty_u{ew}, getv{x}, i} b := selx{xv, ind} & make{W, mask} - r := (b<<3) - b + r := (b<=vl - ind_up = mkV{iu - k*ia} - ind_inc = mkV{vl//k + ia} - mask_sh = width{V} % k - } - if (wv == 5) set_consts{5} else set_consts{7} - xv:=W**0; xo:=xv; mask:=xv; ind:=V**0 # state - q:usz = 1 - while (1) { - --q; if (q == 0) { q = wv - # Load and permute bytes - xv = get_perm_x{} - # Bytes for overhang - xo = ov_bytes{(xv & xom) >> (8 - wv)} - xo = selV{xo, xse} - # Initialize state vectors - ind = ind0 - mask = mask0 - } else { - # Update state vectors - xo = shr{V, xo, 1} - ind = selV{ind, ind_up} + ind_inc - mask = advance_spaced_mask{wv, mask, mask_sh} - } - b := selV{xv, ind} & mask - rv:= (b< Date: Wed, 14 Aug 2024 14:08:35 -0400 Subject: [PATCH 20/20] NEON k/bool support --- src/singeli/src/replicate.singeli | 44 +++++++++++++++++++------------ 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 6b65d7dc..8b80c07a 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -289,6 +289,12 @@ exportT{'si_constrep', each{rep_const, dat_types}} # Constant replicate on boolean +def any_sel = hasarch{'SSSE3'} or hasarch{'AARCH64'} +if_inline (hasarch{'AARCH64'}) { + def __shl{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,b} + def __shr{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,-b} +} + fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : void = { assert{wv >= 2}; assert{wv <= 64} nw := cdiv{rlen, 64} @@ -297,7 +303,7 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : void = { wf := wv>>p if (wf == 1) { rep_const_bool_div8{wv, x, r, nw} - } else if (hasarch{'SSSE3'} and p == 3 and wv <= 8*select{basic_rep, -1}) { + } else if (any_sel and p == 3 and wv <= 8*select{basic_rep, -1}) { # (higher wv double-factors, which doesn't work with in-place pointers) tlen := rlen / wf t := r + cdiv{rlen, 64} - cdiv{tlen, 64} @@ -306,7 +312,7 @@ fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : void = { } else { tlen := rlen >> p wq := usz~~1 << p - if (p == 1 and (not hasarch{'SSSE3'} or wv>=24)) { + if (p == 1 and (not any_sel or wv>=24)) { # Expanding odd second is faster tlen = rlen / wf t:=wf; wf=wq; wq=t @@ -383,7 +389,7 @@ def get_boolvec_writer{T=(u64), r:*T, nw} = { tup{output, {}=>{}, flush} } -def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSE2'}} = { +def rep_const_bool_div8{wv, x, r, nw if has_simd} = { def avx2 = hasarch{'AVX2'} def vl = if (avx2) 32 else 16 def V = [vl]u8 @@ -414,7 +420,7 @@ def rep_const_bool_div8{wv, x, r, nw if hasarch{'SSE2'}} = { output{(xe & mkV{1 << (iV % 8)}) > V**0} } } - if (not hasarch{'SSSE3'}) { + if (not any_sel) { if (wv == 2) { run24{*V~~x, {xv} => { def swap{x, l, m} = { def d = (x ^ x>>l) & V**m; x ^ (d|d< z{z{z{x}}}} } - } else { # hasarch{'SSSE3'} + } else { # any_sel def selH = sel{[16]u8, ., .} def makeTab{t} = selH{mkV{if (avx2) merge{t,t} else t}, .} def id{xv} = xv @@ -502,9 +508,9 @@ def swap_elts{x:V=[_]_, el_bytes} = { def rev = i + (l - 2*(i&l)) sel_imm{[n]ty_u{wd*8}, x, rev} } - if (el_bytes >= 4) { + if (hasarch{'SSE2'} and el_bytes >= 4) { selx{4, max{4, el_bytes/2}} - } else if (hasarch{'SSSE3'}) { + } else if (any_sel) { selx{16, 1} } else { def sh = el_bytes*8 @@ -515,7 +521,8 @@ def swap_elts{x:V=[_]_, el_bytes} = { # Extract swap functions from modperm_dat def extract_modperm_mask{full:W, lane, l} = { def f = l == 128 # Use full width - def w = if (l<32) 8 else if (f) 64 else 32 # Shuffle element width + def w = if (l<32 or not hasarch{'SSE2'}) 8 + else if (f) 64 else 32 # Shuffle element width def n = (if (f) 256 else 128) / w # and number of elements def o = l / w # Extract [o,2*o) def i = o + iota{n}%o @@ -556,9 +563,9 @@ def proc_mod_dat{swap_data:W} = { } def {partwidth, partperm} = { def sh{w, d} = tup{w, get_shiftperm{d, w}} - if (ww==64) sh{64, swap_data} - else if (not hasarch{'SSSE3'}) sh{32, shuf{[4]u32, swap_data, 4b0000}} - else tup{8, get_byteperm{}} + if (ww==64) sh{64, swap_data} + else if (not any_sel) sh{32, shuf{[4]u32, swap_data, 4b0000}} + else tup{8, get_byteperm{}} } # Fill in higher steps def get_mod_permuter{} = { @@ -574,7 +581,7 @@ def proc_mod_dat{swap_data:W} = { def rep_const_bool_odd{k, x, r, nw} = { def avx2 = hasarch{'AVX2'} - def W = if (hasarch{'SSE2'}) [if (avx2) 4 else 2]u64 else u64 + def W = if (has_simd) [if (avx2) 4 else 2]u64 else u64 def {output, check_done, flush} = get_boolvec_writer{W, r, nw} xp := *W~~x @@ -586,7 +593,7 @@ def rep_const_bool_odd{k, x, r, nw} = { swap_data := load{swtab, k>>1} def {partperm, get_full_permute} = proc_mod_dat{swap_data} - def sp_max = if (hasarch{'SSSE3'}) 8 else 4 + def sp_max = if (any_sel) 8 else 4 if (k < sp_max) { rep_const_bool_small_odd{W, sp_max, k, getter{partperm}, output} } else { @@ -639,7 +646,7 @@ def rep_const_bool_odd_mask4{ mr := scal{u64~~1< { - ca := if (hasarch{'SSE4.2'}) { def S = [l]i64; S~~c > S**0 } + ca := if (hasarch{'SSE4.2'} or hasarch{'AARCH64'}) { def S = [l]i64; S~~c > S**0 } else { def S = [2*l]i32; cm := S~~c; cm != shuf{S, cm, 4b2301} } a + M~~ca } @@ -648,8 +655,11 @@ def rep_const_bool_odd_mask4{ while (1) { x:M = get_modperm_x{} - def vrot1 = ifvec{{x} => if (w128{M}) shuf{[4]u32, x, 4b1032} - else shuf{M, x, 4b2103}} + def vrot1 = ifvec{{x} => { + if (w256{M}) shuf{M, x, 4b2103} + else if (any_sel) vshl{x, x, vcount{type{x}}-1} + else shuf{[4]u32, x, 4b1032} + }} xo := x<>(64-k)} # Write result word given starting bits def step{b, c} = output{sub_carry{c - b, c & mr}} @@ -689,7 +699,7 @@ def rep_const_bool_small_odd{(u64), 4, wv, get_swap_x, output} = { def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = { def ov_bytes{o} = { def V = re_el{u8, W}; v := V~~o; v += v > V**0; W~~v } def ww = width{W} - def ew = if (hasarch{'SSSE3'}) 8 else 32 # width of shuffle-able elements + def ew = if (any_sel) 8 else 32 # width of shuffle-able elements def ne = ww/ew; def se = 64/ew def lanes = ww > 128 def dup{v} = if (lanes) merge{v,v} else v