From 30a6de0a3154ca51937b3bee200391a3d147d77b Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 18 Dec 2022 22:30:31 +0200 Subject: [PATCH] add unrolling to aarch64 cmp.singeli --- src/singeli/src/base.singeli | 6 ++++-- src/singeli/src/cmp.singeli | 8 +++++++- src/singeli/src/sse3.singeli | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 0617c36f..eb5694b9 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -94,8 +94,10 @@ def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x def undefPromote{T, x:X & T==X} = T~~x -def andnz{x, y} = assert{'andnz not supported', show{x, y}} -def topBlend{f,t,m} = assert{'topBlend not supported', show{f,t,m}} +def andnz{...x} = assert{'andnz not supported', show{...x}} +def topBlend{...x} = assert{'topBlend not supported', show{...x}} +def topMask{...x} = assert{'topBlend not supported', show{...x}} +def pack{...x} = assert{'pack not supported', show{...x}} def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index aee3adcb..5c46b381 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -71,7 +71,13 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = { assert{am>0} while (ri < am) { r:u64 = 0 - @unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) + if (hasarch{'AARCH64'}) { + def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}} + def step{j, n & n>=2} = pack{step{j, n/2}, step{j+n/2, n/2}} + r = promote{u64, homMask{step{0, unr}}} + } else { + @unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) + } b_setBatch{bulk, dst, ri, r} xi+= unr ri+= 1 diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 58b3145a..6c72c55c 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -101,6 +101,10 @@ def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b} +def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b} +def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b} +def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b} +def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} # shuffle stuff def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}