diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 7244f470..16d66846 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -145,7 +145,7 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { tfree{t0} } -def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { +def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch{'AVX2'}} = { assert{wn < 128} # Total must fit in i8 def T = i8 def vl = 32 @@ -159,64 +159,64 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { def plus_scan{tab, len} = { s:i8=0; @for_dir{up} (tab over len) { s += tab; tab = s } } + + # Convert to bit table def no_bittab = makelabel{}; def done = makelabel{} - if (hasarch{'AVX2'}) { - # Convert to bit table - def nb = 256/vl - nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques - vb := U~~make{[nb](ty_u{vl}), - @collect (t in *V~~t0 over nb) addu{homMask{t > V**0}} - } - dup := promote{u64,nu} < wn - # Unique index to w index conversion - ui := undefined{V}; ui1 := undefined{V} - if (dup) { - if (nu > vl) goto{no_bittab} - # We'll subtract 1 when indexing so the initial 0 isn't needed - tui:*i8 = copy{vl, 0}; i:T = 0 - @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } - ui = load{*V~~tui, 0} - if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} - ui = shuf{[4]u64, ui, 4b1010} - } - # Popcount on 8-bit values - def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } - def sum4 = getsel{make{H, sums{vl/2}}} - bot4 := U**0x0f - def vpopc{v} = { - def s{b} = sum4{b&bot4} - s{shr16{v,4}} + s{v} - } - # Bit table - def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness - def sel_b = getsel{swap{vb}} - # Masks for filtering bit table - def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} - # Exact values for multiples of 8 - store{*U~~t0, 0, vpopc{vb}} - plus_scan{t0, 256/8} - def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} - # Top 5 bits select bytes from tables; bottom 3 select from mask - bot3 := U**0x07 - @for_vec_overlap{vl} (j to xn) { - xv := load{*U~~(x+j), 0} - xb := xv & bot3 - xt := shr16{xv &~ bot3, 3} - ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} - if (dup) { - i0 := V~~ind # Can contain -1 - ind = sel{H, ui, i0} - if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 V**0}} } + dup := promote{u64,nu} < wn + # Unique index to w index conversion + ui := undefined{V}; ui1 := undefined{V} + if (dup) { + if (nu > vl) goto{no_bittab} + # We'll subtract 1 when indexing so the initial 0 isn't needed + tui:*i8 = copy{vl, 0}; i:T = 0 + @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } + ui = load{*V~~tui, 0} + if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} + ui = shuf{[4]u64, ui, 4b1010} + } + # Popcount on 8-bit values + def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } + def sum4 = getsel{make{H, sums{vl/2}}} + bot4 := U**0x0f + def vpopc{v} = { + def s{b} = sum4{b&bot4} + s{shr16{v,4}} + s{v} + } + # Bit table + def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness + def sel_b = getsel{swap{vb}} + # Masks for filtering bit table + def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} + # Exact values for multiples of 8 + store{*U~~t0, 0, vpopc{vb}} + plus_scan{t0, 256/8} + def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} + # Top 5 bits select bytes from tables; bottom 3 select from mask + bot3 := U**0x07 + @for_vec_overlap{vl} (j to xn) { + xv := load{*U~~(x+j), 0} + xb := xv & bot3 + xt := shr16{xv &~ bot3, 3} + ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} + if (dup) { + i0 := V~~ind # Can contain -1 + ind = sel{H, ui, i0} + if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0