unpack → zip & mzip; enable slash.singeli SIMD parts for NEON
This commit is contained in:
parent
ba2e859dce
commit
700d69e7dd
@ -169,9 +169,6 @@ def topMask{...x} = assert{'topMask not supported', show{...x}}
|
||||
def packLo{...x} = assert{'packLo not supported', show{...x}}
|
||||
def packHi{...x} = assert{'packHi not supported', show{...x}}
|
||||
def mul12{...x} = assert{'mul12 not supported', show{...x}}
|
||||
def unpack{...x} = assert{'unpack not supported', show{...x}}
|
||||
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
|
||||
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
|
||||
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
|
||||
def packQ{...x} = assert{'packQ not supported', show{...x}}
|
||||
def shl{...x} = assert{'shl not supported', show{...x}}
|
||||
@ -208,9 +205,14 @@ def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}}
|
||||
def extract{...x} = assert{'extract not supported', show{...x}}
|
||||
def abs{...x} = assert{'abs not supported', show{...x}}
|
||||
def homBlend{...x} = assert{'homBlend not supported', show{...x}}
|
||||
def zip{...x} = assert{'zip not supported', show{...x}}
|
||||
def clmul{...x} = assert{'clmul not supported', show{...x}}
|
||||
def zip{...x} = assert{'zip not supported', show{...x}}
|
||||
def zipLo{...x} = assert{'zipLo not supported', show{...x}}
|
||||
def zipHi{...x} = assert{'zipHi not supported', show{...x}}
|
||||
|
||||
def mzip {a:T, b:T} = each{{v}=>ty_dbl{T}~~v, zip{a, b}}
|
||||
def mzipLo{a:T, b:T} = ty_dbl{T} ~~ zipLo{a, b}
|
||||
def mzipHi{a:T, b:T} = ty_dbl{T} ~~ zipHi{a, b}
|
||||
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
|
||||
oper &~ andnot infix none 35
|
||||
def widen{T, x:X & T==X} = x
|
||||
|
||||
@ -69,7 +69,7 @@ def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
|
||||
}
|
||||
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
|
||||
b:= [16]u8~~[8]u16**a
|
||||
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
|
||||
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
|
||||
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
|
||||
}
|
||||
|
||||
|
||||
@ -119,9 +119,9 @@ def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} =
|
||||
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
||||
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||
|
||||
def zip1{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zip2{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
def zip{a:T, b:T & nvec{T}} = tup{zip1{a,b}, zip2{a,b}}
|
||||
def zipLo{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zipHi{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
def zip{a:T, b:T & nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
||||
|
||||
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
||||
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
|
||||
|
||||
@ -17,7 +17,7 @@ fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||
e := words/2
|
||||
c := V**init
|
||||
@for (rv, xv over e) {
|
||||
rv = apply{unpackLo, (@collect (j to 2) xor64{xv, j, c})}
|
||||
rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})}
|
||||
}
|
||||
if (words & 1) {
|
||||
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
|
||||
|
||||
@ -152,19 +152,20 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
}
|
||||
|
||||
# i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper
|
||||
def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32
|
||||
def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16
|
||||
fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'}
|
||||
def thresh{c==0, T==i16 & simd128{}} = 32
|
||||
def thresh{c==0, T==i32 & simd128{}} = 16
|
||||
fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def I = [16]i8
|
||||
j := I**(if (T==i16) 0 else cast_i{i8,x})
|
||||
def {top, inctop} = topper{T, I, 8, x}
|
||||
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
|
||||
ind := load{itab, w}
|
||||
pc := popc_alt{w, ind, 8}
|
||||
v := unpackLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
|
||||
v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
|
||||
def st{k, v:V} = store{*V~~r, k, v}
|
||||
if (T==i16) st{0, v}
|
||||
else each{st, iota{2}, unpack{v, tupsel{1,top}}}
|
||||
else each{st, iota{2}, mzip{v, tupsel{1,top}}}
|
||||
r += pc
|
||||
j += I**8
|
||||
inctop{i, top}
|
||||
@ -172,15 +173,16 @@ fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T
|
||||
}
|
||||
|
||||
# i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle
|
||||
def thresh{c==1, T==i8 & hasarch{'SSSE3'}} = 64
|
||||
def thresh{c==1, T==i16 & hasarch{'SSSE3'}} = 32
|
||||
fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'}
|
||||
def thresh{c==1, T==i8 & shufb128{}} = 64
|
||||
def thresh{c==1, T==i16 & shufb128{}} = 32
|
||||
fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def V = [16]i8
|
||||
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
|
||||
ind := load{itab, w}
|
||||
pc := popc_alt{w, ind, 8}
|
||||
s := V~~make{[2]u64, ind,0}
|
||||
if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} }
|
||||
if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} }
|
||||
res := sel{V, load{*V~~(x+8*i)}, s}
|
||||
if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}}
|
||||
else store{*V~~r, 0, res}
|
||||
@ -190,8 +192,8 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64,
|
||||
|
||||
# i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle
|
||||
i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B
|
||||
def thresh{c==1, T==i32 & hasarch{'SSSE3'}} = 8
|
||||
fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def thresh{c==1, T==i32 & shufb128{}} = 8
|
||||
fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||
def V = [16]i8
|
||||
expander := make{V, iota{16}>>2}
|
||||
trail := make{V, tail{2,iota{16}}}
|
||||
|
||||
@ -128,7 +128,7 @@ def __mul{a:T,b:T & [4]i32==T} = {
|
||||
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
||||
lo:= mu{a, b}
|
||||
hi:= mu{...sw{4b2301, a, b}}
|
||||
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
|
||||
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
|
||||
}
|
||||
|
||||
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
|
||||
@ -186,13 +186,13 @@ def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
|
||||
|
||||
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
|
||||
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
|
||||
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
|
||||
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
|
||||
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
|
||||
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
|
||||
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
|
||||
|
||||
def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b}
|
||||
def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b}
|
||||
def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
||||
|
||||
|
||||
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
|
||||
|
||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
||||
@ -207,7 +207,7 @@ def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) |
|
||||
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = {
|
||||
def s{v} = s{unpackLo{v, v}}
|
||||
def s{v} = s{mzipLo{v, v}}
|
||||
def s{v:V & V==T} = v
|
||||
s{x} >> (elwidth{T} - elwidth{X})
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user