element-pointer-based vector loads & stores

This commit is contained in:
dzaima 2025-02-03 03:17:23 +02:00
parent 91800ddd1e
commit 848813331a
3 changed files with 12 additions and 7 deletions

View File

@ -198,6 +198,11 @@ def hom_to_int{...vs if length{vs}>1} = {
}
def hom_to_int{x if ktup{x}} = hom_to_int{...x}
def load{V=[_]_, ptr:*_} = assert{0, 'bad load', V, ptr}
def store{ptr:*_, val:[_]_} = assert{0, 'bad store', ptr, val}
def load{V=[k]E, ptr:*E} = load{*V~~ptr}
def store{ptr:*E, val:V=[k]E} = store{*V~~ptr, 0, val}
if_inline (hasarch{'X86_64'}) {
include 'arch/iintrinsic/basic'
include 'arch/iintrinsic/select'

View File

@ -175,13 +175,13 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
def ms = if (up) 256-(1<<(1+iota{8})) else (1<<iota{8})-1
def sel_m = getsel{make{H, merge{ms - 256*(ms>127), 8**0}}}
# Exact values for multiples of 8
store{*V~~t0, 0, vpopc{vb}}
store{t0, vpopc{vb}}
st:i8=0; @for_dir{up} (t0 over 256/8) { st += t0; t0 = st }
def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}}
def sel_c = getsel{swap{load{V, t0} - V**dup}}
# Top 5 bits select bytes from tables; bottom 3 select from mask
bot3 := U**0x07
@for_vec_overlap{vl} (j to xn) {
xv := load{*U~~(x+j), 0}
xv := U~~load{V, x+j}
xb := xv & bot3
xt := shr16{xv &~ bot3, 3}
ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}}
@ -195,7 +195,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
if (nu > 32) ind = blend_hom{blend_top{...each{isel,ui2}, b}, ind, i0 < V**vl}
}
}
store{*V~~(*T~~rp+j), 0, ind}
store{*T~~rp+j, ind}
}
goto{done}
setlabel{no_bittab}
@ -254,7 +254,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
def last = klog==lstep
def this = if (not last) log==klog else log>=klog
if (this) @for_vec_overlap{vl} (j to xn) {
xv:= load{*V~~(x+j), 0}
xv:= load{V, x+j}
s := U**bb{iota{isub}} # Select sequential bytes within each U
def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}}
def ltx = cmpx{lt}; def eqx = cmpx{==}

View File

@ -64,9 +64,9 @@ def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = {
def bufw = bufn * tw
def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw;
def R = [vc]T
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc }
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc }
assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack
store_blended_hom{*R~~r0, mask_of_first{R, end-buf}, load{*R~~buf}}
store_blended_hom{*R~~r0, mask_of_first{R, end-buf}, load{R, buf}}
} else {
@for (r0, buf over u64~~(end-buf)) r0 = buf
}