More NEON functionality moved to neon_intrin/select
This commit is contained in:
parent
fa43ffd882
commit
25724b6b3e
@ -1 +1 @@
|
||||
Subproject commit 431d414de39973909e07cd24e98c12b5ab76d66e
|
||||
Subproject commit 0620571fadc7d2e35c322d0945421ce3f0283bf2
|
||||
@ -178,9 +178,9 @@ def {
|
||||
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow,
|
||||
pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow,
|
||||
topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper,
|
||||
multishift,vshl,
|
||||
multishift,
|
||||
}
|
||||
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
@ -209,14 +209,13 @@ if_inline (hasarch{'X86_64'}) {
|
||||
include 'arch/neon_intrin/basic'
|
||||
include 'arch/neon_intrin/select'
|
||||
include './neon'
|
||||
def {vec_shuffle16_lo}
|
||||
} else {
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
|
||||
}
|
||||
|
||||
if_inline (not hasarch{'X86_64'}) {
|
||||
def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def {
|
||||
__adds,__subs,__sqrt,vec_broadcast,vec_make,
|
||||
vec_shift_left_128,vec_shift_right_128,vec_merge_shift_right,
|
||||
vec_shuffle16_lo
|
||||
}
|
||||
}
|
||||
|
||||
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
|
||||
@ -242,6 +241,7 @@ def make{V=[_]_, ...xs} = vec_make{V, ...xs}
|
||||
def iota{V=[k]_} = make{V, ...iota{k}}
|
||||
def absu{a:[_]_} = ty_u{abs{a}}
|
||||
def shuf = vec_shuffle
|
||||
def vshl = vec_merge_shift_right
|
||||
|
||||
def floor = __floor
|
||||
def ceil = __ceil
|
||||
|
||||
@ -31,8 +31,8 @@ def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, n
|
||||
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
||||
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
||||
|
||||
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
|
||||
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
|
||||
def bitBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_bit{f, t, m}
|
||||
def homBlend{f:T=[_]_, t:T, m:M=[_]_} = blend_hom{f, t, m}
|
||||
|
||||
def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
|
||||
def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
|
||||
@ -40,9 +40,7 @@ def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
|
||||
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
|
||||
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
|
||||
def rev{(width{T}), x:[_]T} = x
|
||||
def rev{16, x:T if nveci{T} and elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
|
||||
def rev{32, x:T if nveci{T} and elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
|
||||
def rev{64, x:T if nveci{T} and elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
|
||||
def rev{w, x:[k]T if w > width{T}} = reverse_units{w/width{T}, x}
|
||||
def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
|
||||
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
|
||||
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
|
||||
@ -74,10 +72,6 @@ def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
|
||||
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
|
||||
def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
||||
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
|
||||
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||
|
||||
def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
|
||||
def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
|
||||
def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}
|
||||
|
||||
@ -490,8 +490,11 @@ def modperm_shuf_step{x:V=[_]T, l, m if l%8==0} = {
|
||||
(x &~ m) | (swap_elts{x, l/8} & m)
|
||||
}
|
||||
# Reverse each pair of elements
|
||||
def swap_elts{x:V=[_]_, el_bytes} = {
|
||||
if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
|
||||
def swap_elts{x:V=[k]_, el_bytes} = {
|
||||
if (hasarch{'AARCH64'}) {
|
||||
if (el_bytes == 8) vshl{x, x, k/2}
|
||||
else V~~rev{el_bytes*16, re_el{ty_u{8*el_bytes}, V} ~~ x}
|
||||
} else if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
|
||||
def wd = min{el_bytes, 8}
|
||||
def l = el_bytes/wd
|
||||
def i = iota{l}
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
### SSSE3 ###
|
||||
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n}
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
|
||||
Loading…
Reference in New Issue
Block a user