unpack → zip & mzip; enable slash.singeli SIMD parts for NEON

This commit is contained in:
dzaima 2023-07-19 21:18:01 +03:00
parent ba2e859dce
commit 700d69e7dd
6 changed files with 33 additions and 29 deletions

View File

@ -169,9 +169,6 @@ def topMask{...x} = assert{'topMask not supported', show{...x}}
def packLo{...x} = assert{'packLo not supported', show{...x}}
def packHi{...x} = assert{'packHi not supported', show{...x}}
def mul12{...x} = assert{'mul12 not supported', show{...x}}
def unpack{...x} = assert{'unpack not supported', show{...x}}
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
def packQ{...x} = assert{'packQ not supported', show{...x}}
def shl{...x} = assert{'shl not supported', show{...x}}
@ -208,9 +205,14 @@ def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}}
def extract{...x} = assert{'extract not supported', show{...x}}
def abs{...x} = assert{'abs not supported', show{...x}}
def homBlend{...x} = assert{'homBlend not supported', show{...x}}
def zip{...x} = assert{'zip not supported', show{...x}}
def clmul{...x} = assert{'clmul not supported', show{...x}}
def zip{...x} = assert{'zip not supported', show{...x}}
def zipLo{...x} = assert{'zipLo not supported', show{...x}}
def zipHi{...x} = assert{'zipHi not supported', show{...x}}
def mzip {a:T, b:T} = each{{v}=>ty_dbl{T}~~v, zip{a, b}}
def mzipLo{a:T, b:T} = ty_dbl{T} ~~ zipLo{a, b}
def mzipHi{a:T, b:T} = ty_dbl{T} ~~ zipHi{a, b}
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
oper &~ andnot infix none 35
def widen{T, x:X & T==X} = x

View File

@ -69,7 +69,7 @@ def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
}
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
b:= [16]u8~~[8]u16**a
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
}

View File

@ -119,9 +119,9 @@ def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} =
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
def zip1{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zip2{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def zip{a:T, b:T & nvec{T}} = tup{zip1{a,b}, zip2{a,b}}
def zipLo{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zipHi{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def zip{a:T, b:T & nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }

View File

@ -17,7 +17,7 @@ fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
e := words/2
c := V**init
@for (rv, xv over e) {
rv = apply{unpackLo, (@collect (j to 2) xor64{xv, j, c})}
rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})}
}
if (words & 1) {
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}

View File

@ -152,19 +152,20 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
}
# i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper
def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32
def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16
fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'}
def thresh{c==0, T==i16 & simd128{}} = 32
def thresh{c==0, T==i32 & simd128{}} = 16
fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def I = [16]i8
j := I**(if (T==i16) 0 else cast_i{i8,x})
def {top, inctop} = topper{T, I, 8, x}
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
ind := load{itab, w}
pc := popc_alt{w, ind, 8}
v := unpackLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
def st{k, v:V} = store{*V~~r, k, v}
if (T==i16) st{0, v}
else each{st, iota{2}, unpack{v, tupsel{1,top}}}
else each{st, iota{2}, mzip{v, tupsel{1,top}}}
r += pc
j += I**8
inctop{i, top}
@ -172,15 +173,16 @@ fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T
}
# i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle
def thresh{c==1, T==i8 & hasarch{'SSSE3'}} = 64
def thresh{c==1, T==i16 & hasarch{'SSSE3'}} = 32
fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'}
def thresh{c==1, T==i8 & shufb128{}} = 64
def thresh{c==1, T==i16 & shufb128{}} = 32
fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def V = [16]i8
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
ind := load{itab, w}
pc := popc_alt{w, ind, 8}
s := V~~make{[2]u64, ind,0}
if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} }
if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} }
res := sel{V, load{*V~~(x+8*i)}, s}
if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}}
else store{*V~~r, 0, res}
@ -190,8 +192,8 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64,
# i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle
i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B
def thresh{c==1, T==i32 & hasarch{'SSSE3'}} = 8
fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c==1, T==i32 & shufb128{}} = 8
fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def V = [16]i8
expander := make{V, iota{16}>>2}
trail := make{V, tail{2,iota{16}}}

View File

@ -128,7 +128,7 @@ def __mul{a:T,b:T & [4]i32==T} = {
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
lo:= mu{a, b}
hi:= mu{...sw{4b2301, a, b}}
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
}
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
@ -186,13 +186,13 @@ def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b}
def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b}
def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
@ -207,7 +207,7 @@ def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) |
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = {
def s{v} = s{unpackLo{v, v}}
def s{v} = s{mzipLo{v, v}}
def s{v:V & V==T} = v
s{x} >> (elwidth{T} - elwidth{X})
}