2-byte vector binary searches
This commit is contained in:
parent
d665b90bbf
commit
1080236433
@ -145,37 +145,14 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
|
|||||||
tfree{t0}
|
tfree{t0}
|
||||||
}
|
}
|
||||||
|
|
||||||
def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = {
|
def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
|
||||||
assert{wn > 0}
|
|
||||||
def T = i8; def I = u8
|
|
||||||
def lt = if (up) <; else >
|
|
||||||
def pre = (if (up) minvalue else maxvalue){T}
|
|
||||||
def vl = 32
|
|
||||||
def V = [vl]T; def H = [vl/2]T
|
|
||||||
def U = [vl]I
|
|
||||||
if (hasarch{'AVX2'} and wn < 8) {
|
|
||||||
log := ceil_log2{wn+1}
|
|
||||||
l := 1<<log
|
|
||||||
gap := l - cast_i{u8, wn}
|
|
||||||
off := U**(gap - 1)
|
|
||||||
wv := homBlend{load{*H~~(w-gap), 0}, H**pre, maskOf{H,gap}}
|
|
||||||
def selw = getsel{wv}
|
|
||||||
h0 := U**(l/2)
|
|
||||||
@unroll (klog from 2 to 4) {
|
|
||||||
if (log==klog) @for_vec_overlap{vl} (j to n) {
|
|
||||||
xv:= load{*V~~(x+j), 0}
|
|
||||||
s := U**0
|
|
||||||
h := h0
|
|
||||||
@unroll (klog) {
|
|
||||||
m := s | h
|
|
||||||
s = homBlend{m, s, lt{xv, selw{m}}}
|
|
||||||
h = shr16{h, 1}
|
|
||||||
}
|
|
||||||
store{*U~~(res+j), 0, s - off}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
assert{wn < 128} # Total must fit in i8
|
assert{wn < 128} # Total must fit in i8
|
||||||
|
def T = i8
|
||||||
|
def vl = 32
|
||||||
|
def V = [vl]T; def H = v_half{V}
|
||||||
|
def U = [vl]u8
|
||||||
|
def res = *T~~rp
|
||||||
|
|
||||||
t0:*i8 = copy{256,0}
|
t0:*i8 = copy{256,0}
|
||||||
t:*i8 = t0 + 128
|
t:*i8 = t0 + 128
|
||||||
@for (w over wn) store{t, w, 1+load{t, w}}
|
@for (w over wn) store{t, w, 1+load{t, w}}
|
||||||
@ -222,7 +199,7 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = {
|
|||||||
def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}}
|
def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}}
|
||||||
# Top 5 bits select bytes from tables; bottom 3 select from mask
|
# Top 5 bits select bytes from tables; bottom 3 select from mask
|
||||||
bot3 := U**0x07
|
bot3 := U**0x07
|
||||||
@for_vec_overlap{vl} (j to n) {
|
@for_vec_overlap{vl} (j to xn) {
|
||||||
xv := load{*U~~(x+j), 0}
|
xv := load{*U~~(x+j), 0}
|
||||||
xb := xv & bot3
|
xb := xv & bot3
|
||||||
xt := shr16{xv &~ bot3, 3}
|
xt := shr16{xv &~ bot3, 3}
|
||||||
@ -238,8 +215,43 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = {
|
|||||||
setlabel{no_bittab}
|
setlabel{no_bittab}
|
||||||
}
|
}
|
||||||
plus_scan{t0, 256}
|
plus_scan{t0, 256}
|
||||||
@for (res, x over n) res = load{t, x}
|
@for (res, x over xn) res = load{t, x}
|
||||||
if (hasarch{'AVX2'}) setlabel{done}
|
if (hasarch{'AVX2'}) setlabel{done}
|
||||||
|
}
|
||||||
|
|
||||||
|
def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
||||||
|
assert{wn > 1}
|
||||||
|
def wd = width{T}
|
||||||
|
def bytes = wd/8; def bb = bind{base,256}
|
||||||
|
def vl = 256/wd
|
||||||
|
def V = [vl]T; def H = v_half{V}
|
||||||
|
def U = [vl](ty_u{T})
|
||||||
|
def lt = if (up) <; else >
|
||||||
|
# Number of steps
|
||||||
|
log := ceil_log2{wn+1}
|
||||||
|
l := 1<<log
|
||||||
|
gap := l - cast_i{u8, wn}
|
||||||
|
off := [vl]u8**(gap - 1)
|
||||||
|
# Fill with minimum value at the beginning
|
||||||
|
def pre = (if (up) minvalue else maxvalue){T}
|
||||||
|
wv := homBlend{load{*H~~(w-gap), 0}, H**pre, maskOf{H,gap}}
|
||||||
|
def selw = getsel{[16]u8~~wv}
|
||||||
|
# A bit in every byte
|
||||||
|
h0 := U**(bb{copy{bytes,bytes}} * (cast_i{ty_u{T},l}/2))
|
||||||
|
@unroll (klog from 2 to lb{maxwn}+1) {
|
||||||
|
if (log==klog) @for_vec_overlap{vl} (j to xn) {
|
||||||
|
xv:= load{*V~~(x+j), 0}
|
||||||
|
s := U**bb{iota{bytes}} # Select sequential bytes within each U
|
||||||
|
h := h0
|
||||||
|
@unroll (klog) {
|
||||||
|
m := s | h
|
||||||
|
s = homBlend{m, s, lt{xv, V~~selw{to_el{u8,m}}}}
|
||||||
|
h = shr16{h, 1}
|
||||||
|
}
|
||||||
|
r := if (T==i8) s
|
||||||
|
else half{narrow{u8, s>>(lb{bytes}+wd-8)}, 0}
|
||||||
|
store{*[vl]i8~~(*i8~~rp+j), 0, r - off}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -282,12 +294,13 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = {
|
|||||||
if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param}
|
if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param}
|
||||||
else if (k+1 < tuplen{rtypes}) lookup{k+1}
|
else if (k+1 < tuplen{rtypes}) lookup{k+1}
|
||||||
}
|
}
|
||||||
|
if (hasarch{'AVX2'} and T<=i16 and wn < 8 and xn >= 256/width{T}) {
|
||||||
|
bin_search_vec{T, ...param, 8}
|
||||||
# Lookup table threshold has to account for cost of
|
# Lookup table threshold has to account for cost of
|
||||||
# populating the table (proportional to wn until it's large), and
|
# populating the table (proportional to wn until it's large), and
|
||||||
# initializing the table (constant, much higher for i16)
|
# initializing the table (constant, much higher for i16)
|
||||||
if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) {
|
} else if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) {
|
||||||
if (rty==0) bin_search_vec{...slice{param,0,-1}, *i8~~rp}
|
lookup{0}
|
||||||
else lookup{1}
|
|
||||||
} else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/promote{u64,ceil_log2{wn}+2})) {
|
} else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/promote{u64,ceil_log2{wn}+2})) {
|
||||||
lookup{0}
|
lookup{0}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user