unpack → zip & mzip; enable slash.singeli SIMD parts for NEON
This commit is contained in:
parent
ba2e859dce
commit
700d69e7dd
@ -169,9 +169,6 @@ def topMask{...x} = assert{'topMask not supported', show{...x}}
|
|||||||
def packLo{...x} = assert{'packLo not supported', show{...x}}
|
def packLo{...x} = assert{'packLo not supported', show{...x}}
|
||||||
def packHi{...x} = assert{'packHi not supported', show{...x}}
|
def packHi{...x} = assert{'packHi not supported', show{...x}}
|
||||||
def mul12{...x} = assert{'mul12 not supported', show{...x}}
|
def mul12{...x} = assert{'mul12 not supported', show{...x}}
|
||||||
def unpack{...x} = assert{'unpack not supported', show{...x}}
|
|
||||||
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
|
|
||||||
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
|
|
||||||
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
|
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
|
||||||
def packQ{...x} = assert{'packQ not supported', show{...x}}
|
def packQ{...x} = assert{'packQ not supported', show{...x}}
|
||||||
def shl{...x} = assert{'shl not supported', show{...x}}
|
def shl{...x} = assert{'shl not supported', show{...x}}
|
||||||
@ -208,9 +205,14 @@ def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}}
|
|||||||
def extract{...x} = assert{'extract not supported', show{...x}}
|
def extract{...x} = assert{'extract not supported', show{...x}}
|
||||||
def abs{...x} = assert{'abs not supported', show{...x}}
|
def abs{...x} = assert{'abs not supported', show{...x}}
|
||||||
def homBlend{...x} = assert{'homBlend not supported', show{...x}}
|
def homBlend{...x} = assert{'homBlend not supported', show{...x}}
|
||||||
def zip{...x} = assert{'zip not supported', show{...x}}
|
|
||||||
def clmul{...x} = assert{'clmul not supported', show{...x}}
|
def clmul{...x} = assert{'clmul not supported', show{...x}}
|
||||||
|
def zip{...x} = assert{'zip not supported', show{...x}}
|
||||||
|
def zipLo{...x} = assert{'zipLo not supported', show{...x}}
|
||||||
|
def zipHi{...x} = assert{'zipHi not supported', show{...x}}
|
||||||
|
|
||||||
|
def mzip {a:T, b:T} = each{{v}=>ty_dbl{T}~~v, zip{a, b}}
|
||||||
|
def mzipLo{a:T, b:T} = ty_dbl{T} ~~ zipLo{a, b}
|
||||||
|
def mzipHi{a:T, b:T} = ty_dbl{T} ~~ zipHi{a, b}
|
||||||
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
|
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
|
||||||
oper &~ andnot infix none 35
|
oper &~ andnot infix none 35
|
||||||
def widen{T, x:X & T==X} = x
|
def widen{T, x:X & T==X} = x
|
||||||
|
|||||||
@ -69,7 +69,7 @@ def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
|
|||||||
}
|
}
|
||||||
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
|
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
|
||||||
b:= [16]u8~~[8]u16**a
|
b:= [16]u8~~[8]u16**a
|
||||||
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
|
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
|
||||||
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
|
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -119,9 +119,9 @@ def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} =
|
|||||||
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
||||||
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||||
|
|
||||||
def zip1{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
def zipLo{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||||
def zip2{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
def zipHi{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||||
def zip{a:T, b:T & nvec{T}} = tup{zip1{a,b}, zip2{a,b}}
|
def zip{a:T, b:T & nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
||||||
|
|
||||||
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
||||||
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
|
def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
|
||||||
|
|||||||
@ -17,7 +17,7 @@ fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
|||||||
e := words/2
|
e := words/2
|
||||||
c := V**init
|
c := V**init
|
||||||
@for (rv, xv over e) {
|
@for (rv, xv over e) {
|
||||||
rv = apply{unpackLo, (@collect (j to 2) xor64{xv, j, c})}
|
rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})}
|
||||||
}
|
}
|
||||||
if (words & 1) {
|
if (words & 1) {
|
||||||
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
|
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
|
||||||
|
|||||||
@ -152,19 +152,20 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper
|
# i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper
|
||||||
def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32
|
def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'}
|
||||||
def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16
|
def thresh{c==0, T==i16 & simd128{}} = 32
|
||||||
fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
def thresh{c==0, T==i32 & simd128{}} = 16
|
||||||
|
fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||||
def I = [16]i8
|
def I = [16]i8
|
||||||
j := I**(if (T==i16) 0 else cast_i{i8,x})
|
j := I**(if (T==i16) 0 else cast_i{i8,x})
|
||||||
def {top, inctop} = topper{T, I, 8, x}
|
def {top, inctop} = topper{T, I, 8, x}
|
||||||
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
|
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
|
||||||
ind := load{itab, w}
|
ind := load{itab, w}
|
||||||
pc := popc_alt{w, ind, 8}
|
pc := popc_alt{w, ind, 8}
|
||||||
v := unpackLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
|
v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
|
||||||
def st{k, v:V} = store{*V~~r, k, v}
|
def st{k, v:V} = store{*V~~r, k, v}
|
||||||
if (T==i16) st{0, v}
|
if (T==i16) st{0, v}
|
||||||
else each{st, iota{2}, unpack{v, tupsel{1,top}}}
|
else each{st, iota{2}, mzip{v, tupsel{1,top}}}
|
||||||
r += pc
|
r += pc
|
||||||
j += I**8
|
j += I**8
|
||||||
inctop{i, top}
|
inctop{i, top}
|
||||||
@ -172,15 +173,16 @@ fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T
|
|||||||
}
|
}
|
||||||
|
|
||||||
# i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle
|
# i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle
|
||||||
def thresh{c==1, T==i8 & hasarch{'SSSE3'}} = 64
|
def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'}
|
||||||
def thresh{c==1, T==i16 & hasarch{'SSSE3'}} = 32
|
def thresh{c==1, T==i8 & shufb128{}} = 64
|
||||||
fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
def thresh{c==1, T==i16 & shufb128{}} = 32
|
||||||
|
fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||||
def V = [16]i8
|
def V = [16]i8
|
||||||
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
|
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
|
||||||
ind := load{itab, w}
|
ind := load{itab, w}
|
||||||
pc := popc_alt{w, ind, 8}
|
pc := popc_alt{w, ind, 8}
|
||||||
s := V~~make{[2]u64, ind,0}
|
s := V~~make{[2]u64, ind,0}
|
||||||
if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} }
|
if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} }
|
||||||
res := sel{V, load{*V~~(x+8*i)}, s}
|
res := sel{V, load{*V~~(x+8*i)}, s}
|
||||||
if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}}
|
if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}}
|
||||||
else store{*V~~r, 0, res}
|
else store{*V~~r, 0, res}
|
||||||
@ -190,8 +192,8 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64,
|
|||||||
|
|
||||||
# i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle
|
# i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle
|
||||||
i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B
|
i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B
|
||||||
def thresh{c==1, T==i32 & hasarch{'SSSE3'}} = 8
|
def thresh{c==1, T==i32 & shufb128{}} = 8
|
||||||
fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
|
||||||
def V = [16]i8
|
def V = [16]i8
|
||||||
expander := make{V, iota{16}>>2}
|
expander := make{V, iota{16}>>2}
|
||||||
trail := make{V, tail{2,iota{16}}}
|
trail := make{V, tail{2,iota{16}}}
|
||||||
|
|||||||
@ -128,7 +128,7 @@ def __mul{a:T,b:T & [4]i32==T} = {
|
|||||||
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
||||||
lo:= mu{a, b}
|
lo:= mu{a, b}
|
||||||
hi:= mu{...sw{4b2301, a, b}}
|
hi:= mu{...sw{4b2301, a, b}}
|
||||||
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
|
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
|
||||||
}
|
}
|
||||||
|
|
||||||
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
|
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
|
||||||
@ -186,13 +186,13 @@ def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
|||||||
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||||
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
|
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
|
||||||
|
|
||||||
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
|
|
||||||
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
|
def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b}
|
||||||
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
|
def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b}
|
||||||
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
|
def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
||||||
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
|
|
||||||
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
|
|
||||||
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
|
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
|
||||||
|
|
||||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||||
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
||||||
@ -207,7 +207,7 @@ def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) |
|
|||||||
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||||
|
|
||||||
def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = {
|
def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = {
|
||||||
def s{v} = s{unpackLo{v, v}}
|
def s{v} = s{mzipLo{v, v}}
|
||||||
def s{v:V & V==T} = v
|
def s{v:V & V==T} = v
|
||||||
s{x} >> (elwidth{T} - elwidth{X})
|
s{x} >> (elwidth{T} - elwidth{X})
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user