Use default bins_lookup if there's no AVX2
This commit is contained in:
parent
1080236433
commit
7f5ba961c2
@ -145,7 +145,7 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
|
||||
tfree{t0}
|
||||
}
|
||||
|
||||
def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
|
||||
def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch{'AVX2'}} = {
|
||||
assert{wn < 128} # Total must fit in i8
|
||||
def T = i8
|
||||
def vl = 32
|
||||
@ -159,64 +159,64 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
|
||||
def plus_scan{tab, len} = {
|
||||
s:i8=0; @for_dir{up} (tab over len) { s += tab; tab = s }
|
||||
}
|
||||
|
||||
# Convert to bit table
|
||||
def no_bittab = makelabel{}; def done = makelabel{}
|
||||
if (hasarch{'AVX2'}) {
|
||||
# Convert to bit table
|
||||
def nb = 256/vl
|
||||
nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques
|
||||
vb := U~~make{[nb](ty_u{vl}),
|
||||
@collect (t in *V~~t0 over nb) addu{homMask{t > V**0}}
|
||||
}
|
||||
dup := promote{u64,nu} < wn
|
||||
# Unique index to w index conversion
|
||||
ui := undefined{V}; ui1 := undefined{V}
|
||||
if (dup) {
|
||||
if (nu > vl) goto{no_bittab}
|
||||
# We'll subtract 1 when indexing so the initial 0 isn't needed
|
||||
tui:*i8 = copy{vl, 0}; i:T = 0
|
||||
@for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i }
|
||||
ui = load{*V~~tui, 0}
|
||||
if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232}
|
||||
ui = shuf{[4]u64, ui, 4b1010}
|
||||
}
|
||||
# Popcount on 8-bit values
|
||||
def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} }
|
||||
def sum4 = getsel{make{H, sums{vl/2}}}
|
||||
bot4 := U**0x0f
|
||||
def vpopc{v} = {
|
||||
def s{b} = sum4{b&bot4}
|
||||
s{shr16{v,4}} + s{v}
|
||||
}
|
||||
# Bit table
|
||||
def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness
|
||||
def sel_b = getsel{swap{vb}}
|
||||
# Masks for filtering bit table
|
||||
def ms = if (up) 256-(1<<(1+iota{8})) else (1<<iota{8})-1
|
||||
def sel_m = getsel{make{H, merge{ms - 256*(ms>127), 8**0}}}
|
||||
# Exact values for multiples of 8
|
||||
store{*U~~t0, 0, vpopc{vb}}
|
||||
plus_scan{t0, 256/8}
|
||||
def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}}
|
||||
# Top 5 bits select bytes from tables; bottom 3 select from mask
|
||||
bot3 := U**0x07
|
||||
@for_vec_overlap{vl} (j to xn) {
|
||||
xv := load{*U~~(x+j), 0}
|
||||
xb := xv & bot3
|
||||
xt := shr16{xv &~ bot3, 3}
|
||||
ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}}
|
||||
if (dup) {
|
||||
i0 := V~~ind # Can contain -1
|
||||
ind = sel{H, ui, i0}
|
||||
if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0<V**(vl/2)}
|
||||
}
|
||||
store{*U~~(res+j), 0, ind}
|
||||
}
|
||||
goto{done}
|
||||
setlabel{no_bittab}
|
||||
def nb = 256/vl
|
||||
nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques
|
||||
vb := U~~make{[nb](ty_u{vl}),
|
||||
@collect (t in *V~~t0 over nb) addu{homMask{t > V**0}}
|
||||
}
|
||||
dup := promote{u64,nu} < wn
|
||||
# Unique index to w index conversion
|
||||
ui := undefined{V}; ui1 := undefined{V}
|
||||
if (dup) {
|
||||
if (nu > vl) goto{no_bittab}
|
||||
# We'll subtract 1 when indexing so the initial 0 isn't needed
|
||||
tui:*i8 = copy{vl, 0}; i:T = 0
|
||||
@for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i }
|
||||
ui = load{*V~~tui, 0}
|
||||
if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232}
|
||||
ui = shuf{[4]u64, ui, 4b1010}
|
||||
}
|
||||
# Popcount on 8-bit values
|
||||
def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} }
|
||||
def sum4 = getsel{make{H, sums{vl/2}}}
|
||||
bot4 := U**0x0f
|
||||
def vpopc{v} = {
|
||||
def s{b} = sum4{b&bot4}
|
||||
s{shr16{v,4}} + s{v}
|
||||
}
|
||||
# Bit table
|
||||
def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness
|
||||
def sel_b = getsel{swap{vb}}
|
||||
# Masks for filtering bit table
|
||||
def ms = if (up) 256-(1<<(1+iota{8})) else (1<<iota{8})-1
|
||||
def sel_m = getsel{make{H, merge{ms - 256*(ms>127), 8**0}}}
|
||||
# Exact values for multiples of 8
|
||||
store{*U~~t0, 0, vpopc{vb}}
|
||||
plus_scan{t0, 256/8}
|
||||
def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}}
|
||||
# Top 5 bits select bytes from tables; bottom 3 select from mask
|
||||
bot3 := U**0x07
|
||||
@for_vec_overlap{vl} (j to xn) {
|
||||
xv := load{*U~~(x+j), 0}
|
||||
xb := xv & bot3
|
||||
xt := shr16{xv &~ bot3, 3}
|
||||
ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}}
|
||||
if (dup) {
|
||||
i0 := V~~ind # Can contain -1
|
||||
ind = sel{H, ui, i0}
|
||||
if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0<V**(vl/2)}
|
||||
}
|
||||
store{*U~~(res+j), 0, ind}
|
||||
}
|
||||
goto{done}
|
||||
setlabel{no_bittab}
|
||||
|
||||
plus_scan{t0, 256}
|
||||
@for (res, x over xn) res = load{t, x}
|
||||
if (hasarch{'AVX2'}) setlabel{done}
|
||||
setlabel{done}
|
||||
}
|
||||
|
||||
def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user