From 25724b6b3e2177ec9333709d9a02755ffc9a6556 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 13 Oct 2024 21:25:22 -0400 Subject: [PATCH] More NEON functionality moved to neon_intrin/select --- build/singeliSubmodule | 2 +- src/singeli/src/base.singeli | 18 +++++++++--------- src/singeli/src/neon.singeli | 12 +++--------- src/singeli/src/replicate.singeli | 7 +++++-- src/singeli/src/sse.singeli | 1 - 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/build/singeliSubmodule b/build/singeliSubmodule index 431d414d..0620571f 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit 431d414de39973909e07cd24e98c12b5ab76d66e +Subproject commit 0620571fadc7d2e35c322d0945421ce3f0283bf2 diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 224e4593..11266938 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -178,9 +178,9 @@ def { absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half, homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, - pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow, + pext,popcRand,rbit,rev,sel,shl,shr,shufInd,storeLow, topBlend,topMask,topMaskStore,topMaskStoreF,unord,unzip,vfold,vec_select,vec_shuffle,widen,widenUpper, - multishift,vshl, + multishift, } def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times @@ -209,14 +209,13 @@ if_inline (hasarch{'X86_64'}) { include 'arch/neon_intrin/basic' include 'arch/neon_intrin/select' include './neon' + def {vec_shuffle16_lo} } else { - def {__adds,__subs,__sqrt,vec_broadcast,vec_make} -} - -if_inline (not hasarch{'X86_64'}) { - def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo} - def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} - def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} + def { + __adds,__subs,__sqrt,vec_broadcast,vec_make, + vec_shift_left_128,vec_shift_right_128,vec_merge_shift_right, + vec_shuffle16_lo + } } def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k} @@ -242,6 +241,7 @@ def make{V=[_]_, ...xs} = vec_make{V, ...xs} def iota{V=[k]_} = make{V, ...iota{k}} def absu{a:[_]_} = ty_u{abs{a}} def shuf = vec_shuffle +def vshl = vec_merge_shift_right def floor = __floor def ceil = __ceil diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index bd27ab77..2eb0a434 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -31,8 +31,8 @@ def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, n def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a< width{T}} = reverse_units{w/width{T}, x} def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x} def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x} def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x} @@ -74,10 +72,6 @@ def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b} def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si} def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i} -def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} - -def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} -def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y} def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y} diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index f492361b..9908eabb 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -490,8 +490,11 @@ def modperm_shuf_step{x:V=[_]T, l, m if l%8==0} = { (x &~ m) | (swap_elts{x, l/8} & m) } # Reverse each pair of elements -def swap_elts{x:V=[_]_, el_bytes} = { - if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) { +def swap_elts{x:V=[k]_, el_bytes} = { + if (hasarch{'AARCH64'}) { + if (el_bytes == 8) vshl{x, x, k/2} + else V~~rev{el_bytes*16, re_el{ty_u{8*el_bytes}, V} ~~ x} + } else if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) { def wd = min{el_bytes, 8} def l = el_bytes/wd def i = iota{l} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index 1af2dfb6..55239898 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -1,6 +1,5 @@ ### SSSE3 ### def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} -def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n} ### SSE4.1 ###