More NEON functionality moved to neon_intrin/select

This commit is contained in:
Marshall Lochbaum 2024-10-13 21:25:22 -04:00
parent fa43ffd882
commit 25724b6b3e
5 changed files with 18 additions and 22 deletions

@ -1 +1 @@
Subproject commit 431d414de39973909e07cd24e98c12b5ab76d66e
Subproject commit 0620571fadc7d2e35c322d0945421ce3f0283bf2

View File

@ -178,9 +178,9 @@ def {
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow,
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
multishift,vshl,
multishift,
}
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
@ -209,14 +209,13 @@ if_inline (hasarch{'X86_64'}) {
include 'arch/neon_intrin/basic'
include 'arch/neon_intrin/select'
include './neon'
def {vec_shuffle16_lo}
} else {
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
}
if_inline (not hasarch{'X86_64'}) {
def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def {
__adds,__subs,__sqrt,vec_broadcast,vec_make,
vec_shift_left_128,vec_shift_right_128,vec_merge_shift_right,
vec_shuffle16_lo
}
}
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
@ -242,6 +241,7 @@ def make{V=[_]_, ...xs} = vec_make{V, ...xs}
def iota{V=[k]_} = make{V, ...iota{k}}
def absu{a:[_]_} = ty_u{abs{a}}
def shuf = vec_shuffle
def vshl = vec_merge_shift_right
def floor = __floor
def ceil = __ceil

View File

@ -31,8 +31,8 @@ def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, n
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
def bitBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_bit{f, t, m}
def homBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_hom{f, t, m}
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
@ -40,9 +40,7 @@ def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
def rev{(width{T}), x:[_]T} = x
def rev{16, x:T if nveci{T} and elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
def rev{32, x:T if nveci{T} and elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
def rev{64, x:T if nveci{T} and elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
def rev{w, x:[k]T if w > width{T}} = reverse_units{w/width{T}, x}
def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
@ -74,10 +72,6 @@ def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}

View File

@ -490,8 +490,11 @@ def modperm_shuf_step{x:V=[_]T, l, m if l%8==0} = {
(x &~ m) | (swap_elts{x, l/8} & m)
}
# Reverse each pair of elements
def swap_elts{x:V=[_]_, el_bytes} = {
if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
def swap_elts{x:V=[k]_, el_bytes} = {
if (hasarch{'AARCH64'}) {
if (el_bytes == 8) vshl{x, x, k/2}
else V~~rev{el_bytes*16, re_el{ty_u{8*el_bytes}, V} ~~ x}
} else if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
def wd = min{el_bytes, 8}
def l = el_bytes/wd
def i = iota{l}

View File

@ -1,6 +1,5 @@
### SSSE3 ###
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n}
### SSE4.1 ###