From 848813331aa58540a6910ab81a59e738e8992a80 Mon Sep 17 00:00:00 2001 From: dzaima Date: Mon, 3 Feb 2025 03:17:23 +0200 Subject: [PATCH] element-pointer-based vector loads & stores --- src/singeli/src/base.singeli | 5 +++++ src/singeli/src/bins.singeli | 10 +++++----- src/singeli/src/slash.singeli | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 595b783c..c3f1c2d7 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -198,6 +198,11 @@ def hom_to_int{...vs if length{vs}>1} = { } def hom_to_int{x if ktup{x}} = hom_to_int{...x} +def load{V=[_]_, ptr:*_} = assert{0, 'bad load', V, ptr} +def store{ptr:*_, val:[_]_} = assert{0, 'bad store', ptr, val} +def load{V=[k]E, ptr:*E} = load{*V~~ptr} +def store{ptr:*E, val:V=[k]E} = store{*V~~ptr, 0, val} + if_inline (hasarch{'X86_64'}) { include 'arch/iintrinsic/basic' include 'arch/iintrinsic/select' diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index ba14b955..5c32b02f 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -175,13 +175,13 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = { def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} # Exact values for multiples of 8 - store{*V~~t0, 0, vpopc{vb}} + store{t0, vpopc{vb}} st:i8=0; @for_dir{up} (t0 over 256/8) { st += t0; t0 = st } - def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} + def sel_c = getsel{swap{load{V, t0} - V**dup}} # Top 5 bits select bytes from tables; bottom 3 select from mask bot3 := U**0x07 @for_vec_overlap{vl} (j to xn) { - xv := load{*U~~(x+j), 0} + xv := U~~load{V, x+j} xb := xv & bot3 xt := shr16{xv &~ bot3, 3} ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} @@ -195,7 +195,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = { if (nu > 32) ind = blend_hom{blend_top{...each{isel,ui2}, b}, ind, i0 < V**vl} } } - store{*V~~(*T~~rp+j), 0, ind} + store{*T~~rp+j, ind} } goto{done} setlabel{no_bittab} @@ -254,7 +254,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = def last = klog==lstep def this = if (not last) log==klog else log>=klog if (this) @for_vec_overlap{vl} (j to xn) { - xv:= load{*V~~(x+j), 0} + xv:= load{V, x+j} s := U**bb{iota{isub}} # Select sequential bytes within each U def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}} def ltx = cmpx{lt}; def eqx = cmpx{==} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 96b2fc48..e353a291 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -64,9 +64,9 @@ def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = { def bufw = bufn * tw def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw; def R = [vc]T - @unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + @unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc } assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack - store_blended_hom{*R~~r0, mask_of_first{R, end-buf}, load{*R~~buf}} + store_blended_hom{*R~~r0, mask_of_first{R, end-buf}, load{R, buf}} } else { @for (r0, buf over u64~~(end-buf)) r0 = buf }