From 7cc1031f70d5d29ba198d524f66a3a3273c3a80b Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 26 Jul 2024 05:13:18 +0300 Subject: [PATCH] use outside-of-loop index widening on aarch64 slower as-is due to reduced supported ranges, but that's not an inherent restriction --- src/singeli/src/lut.singeli | 9 +++++++-- src/singeli/src/select.singeli | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/lut.singeli b/src/singeli/src/lut.singeli index 391559e4..16ffb3e2 100644 --- a/src/singeli/src/lut.singeli +++ b/src/singeli/src/lut.singeli @@ -81,8 +81,11 @@ def blend_halves{mode, E, nt, ni} = tup{nt, ni, loader{{TG} => { def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S def sc = width{D} / width{S} - def add = make{[k*sc]S, range{k*sc} % sc} - if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) { + def RV = [k*sc]S + def add = make{RV, range{k*sc} % sc} + if (hasarch{'AARCH64'} and [k]D == [2]u64 and S==u8) { + (RV~~sel{[16]u8, x, make{[16]u8, range{16}>>3}}<<3) + add + } else if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) { (sel{[8]u32, undefPromote{[8]u32, x}, make{[8]u32, range{8}>>1}}<16 and hasarch{'AVX2'}} = 0 def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and (E==u16 or E==u32)} = zip_halves{mode, E, nt, ni} def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and E==u64} = widen_inds{mode, E, nt, ni, 2} + +def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and mode=='c' and E>=u16} = 0 diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 8145489f..5c47c853 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -5,6 +5,10 @@ include './bitops' include './lut' include 'util/tup' +def arch_minvw = 128 +def arch_minv{T=[_]E if width{T}< arch_minvw} = [arch_minvw / width{E}]E +def arch_minv{T if width{T}>=arch_minvw} = T + def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'} def gather @@ -164,7 +168,8 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max} def WV = [bulk< inds_buf_max) return{0} @for_backwards (i to inds_buf_max/(width{WV}/8)) { - def s = load{*[bulk]u8~~src, i} + def V = [bulk]u8 + def s = loadLow{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}} def v = raw_widen_inds{bulk, sh, s} store{*WV~~dst, i, v} }