use outside-of-loop index widening on aarch64
slower as-is due to reduced supported ranges, but that's not an inherent restriction
This commit is contained in:
parent
a486e87451
commit
7cc1031f70
@ -81,8 +81,11 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => {
|
||||
|
||||
def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S
|
||||
def sc = width{D} / width{S}
|
||||
def add = make{[k*sc]S, range{k*sc} % sc}
|
||||
if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
|
||||
def RV = [k*sc]S
|
||||
def add = make{RV, range{k*sc} % sc}
|
||||
if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) {
|
||||
(RV~~sel{[16]u8, x, make{[16]u8, range{16}>>3}}<<3) + add
|
||||
} else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) {
|
||||
(sel{[8]u32, undefPromote{[8]u32, x}, make{[8]u32, range{8}>>1}}<<sc) + add
|
||||
} else {
|
||||
def wd = widen{[k]D, x}
|
||||
@ -188,3 +191,5 @@ def lut_gen{mode, E==u64, nt, ni if nt>16 and hasarch{'AVX2'}} = 0
|
||||
|
||||
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and (E==u16 or E==u32)} = zip_halves{mode, E, nt, ni}
|
||||
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and E==u64} = widen_inds{mode, E, nt, ni, 2}
|
||||
|
||||
def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0
|
||||
|
||||
@ -5,6 +5,10 @@ include './bitops'
|
||||
include './lut'
|
||||
include 'util/tup'
|
||||
|
||||
def arch_minvw = 128
|
||||
def arch_minv{T=[_]E if width{T}< arch_minvw} = [arch_minvw / width{E}]E
|
||||
def arch_minv{T if width{T}>=arch_minvw} = T
|
||||
|
||||
def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'}
|
||||
|
||||
def gather
|
||||
@ -164,7 +168,8 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
|
||||
def WV = [bulk<<sh]u8
|
||||
if ((n<<sh) > inds_buf_max) return{0}
|
||||
@for_backwards (i to inds_buf_max/(width{WV}/8)) {
|
||||
def s = load{*[bulk]u8~~src, i}
|
||||
def V = [bulk]u8
|
||||
def s = loadLow{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}}
|
||||
def v = raw_widen_inds{bulk, sh, s}
|
||||
store{*WV~~dst, i, v}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user