use outside-of-loop index widening on aarch64

slower as-is due to reduced supported ranges, but that's not an inherent restriction
This commit is contained in:
dzaima 2024-07-26 05:13:18 +03:00
parent a486e87451
commit 7cc1031f70
2 changed files with 13 additions and 3 deletions

View File

@ -81,8 +81,11 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S
def sc = width{D} / width{S}
def add = make{[k*sc]S, range{k*sc} % sc}
if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
def RV = [k*sc]S
def add = make{RV, range{k*sc} % sc}
if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) {
(RV~~sel{[16]u8, x, make{[16]u8, range{16}>>3}}<<3) + add
} else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
(sel{[8]u32, undefPromote{[8]u32, x}, make{[8]u32, range{8}>>1}}<<sc) + add
} else {
def wd = widen{[k]D, x}
@ -188,3 +191,5 @@ def lut_gen{mode, E==u64, nt, ni if nt>16 and hasarch{'AVX2'}} = 0
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and (E==u16 or E==u32)} = zip_halves{mode, E, nt, ni}
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and E==u64} = widen_inds{mode, E, nt, ni, 2}
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0

View File

@ -5,6 +5,10 @@ include './bitops'
include './lut'
include 'util/tup'
def arch_minvw = 128
def arch_minv{T=[_]E if width{T}< arch_minvw} = [arch_minvw / width{E}]E
def arch_minv{T if width{T}>=arch_minvw} = T
def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'}
def gather
@ -164,7 +168,8 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
def WV = [bulk<<sh]u8
if ((n<<sh) > inds_buf_max) return{0}
@for_backwards (i to inds_buf_max/(width{WV}/8)) {
def s = load{*[bulk]u8~~src, i}
def V = [bulk]u8
def s = loadLow{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}}
def v = raw_widen_inds{bulk, sh, s}
store{*WV~~dst, i, v}
}