diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index e9854284..d3e2fcb4 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -169,9 +169,6 @@ def topMask{...x} = assert{'topMask not supported', show{...x}} def packLo{...x} = assert{'packLo not supported', show{...x}} def packHi{...x} = assert{'packHi not supported', show{...x}} def mul12{...x} = assert{'mul12 not supported', show{...x}} -def unpack{...x} = assert{'unpack not supported', show{...x}} -def unpackLo{...x} = assert{'unpackLo not supported', show{...x}} -def unpackHi{...x} = assert{'unpackHi not supported', show{...x}} def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} def packQ{...x} = assert{'packQ not supported', show{...x}} def shl{...x} = assert{'shl not supported', show{...x}} @@ -208,9 +205,14 @@ def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}} def extract{...x} = assert{'extract not supported', show{...x}} def abs{...x} = assert{'abs not supported', show{...x}} def homBlend{...x} = assert{'homBlend not supported', show{...x}} -def zip{...x} = assert{'zip not supported', show{...x}} def clmul{...x} = assert{'clmul not supported', show{...x}} +def zip{...x} = assert{'zip not supported', show{...x}} +def zipLo{...x} = assert{'zipLo not supported', show{...x}} +def zipHi{...x} = assert{'zipHi not supported', show{...x}} +def mzip {a:T, b:T} = each{{v}=>ty_dbl{T}~~v, zip{a, b}} +def mzipLo{a:T, b:T} = ty_dbl{T} ~~ zipLo{a, b} +def mzipHi{a:T, b:T} = ty_dbl{T} ~~ zipHi{a, b} def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b oper &~ andnot infix none 35 def widen{T, x:X & T==X} = x diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index bc08b317..9d74a627 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -69,7 +69,7 @@ def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = { } def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = { b:= [16]u8~~[8]u16**a - exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100} + exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} (exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0 } diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 54b75b3e..15032c8d 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -119,9 +119,9 @@ def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} = def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} -def zip1{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} -def zip2{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} -def zip{a:T, b:T & nvec{T}} = tup{zip1{a,b}, zip2{a,b}} +def zipLo{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} +def zipHi{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} +def zip{a:T, b:T & nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}} def packLo{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} } def packHi{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} } diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli index 729a383c..11aef7d6 100644 --- a/src/singeli/src/neq.singeli +++ b/src/singeli/src/neq.singeli @@ -17,7 +17,7 @@ fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { e := words/2 c := V**init @for (rv, xv over e) { - rv = apply{unpackLo, (@collect (j to 2) xor64{xv, j, c})} + rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})} } if (words & 1) { storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 3044972b..80d40a94 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -152,19 +152,20 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { } # i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper -def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32 -def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16 -fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'} +def thresh{c==0, T==i16 & simd128{}} = 32 +def thresh{c==0, T==i32 & simd128{}} = 16 +fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def I = [16]i8 j := I**(if (T==i16) 0 else cast_i{i8,x}) def {top, inctop} = topper{T, I, 8, x} @for_special_buffered{r,8} (w in *u8~~w over i to sum) { ind := load{itab, w} pc := popc_alt{w, ind, 8} - v := unpackLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}} + v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}} def st{k, v:V} = store{*V~~r, k, v} if (T==i16) st{0, v} - else each{st, iota{2}, unpack{v, tupsel{1,top}}} + else each{st, iota{2}, mzip{v, tupsel{1,top}}} r += pc j += I**8 inctop{i, top} @@ -172,15 +173,16 @@ fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T } # i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle -def thresh{c==1, T==i8 & hasarch{'SSSE3'}} = 64 -def thresh{c==1, T==i16 & hasarch{'SSSE3'}} = 32 -fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'} +def thresh{c==1, T==i8 & shufb128{}} = 64 +def thresh{c==1, T==i16 & shufb128{}} = 32 +fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def V = [16]i8 @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { ind := load{itab, w} pc := popc_alt{w, ind, 8} s := V~~make{[2]u64, ind,0} - if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} } + if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} } res := sel{V, load{*V~~(x+8*i)}, s} if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}} else store{*V~~r, 0, res} @@ -190,8 +192,8 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, # i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B -def thresh{c==1, T==i32 & hasarch{'SSSE3'}} = 8 -fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==1, T==i32 & shufb128{}} = 8 +fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def V = [16]i8 expander := make{V, iota{16}>>2} trail := make{V, tail{2,iota{16}}} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 94c6d350..b9c8e49d 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -128,7 +128,7 @@ def __mul{a:T,b:T & [4]i32==T} = { def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs} lo:= mu{a, b} hi:= mu{...sw{4b2301, a, b}} - [4]i32~~unpackLo{...sw{4b3120, lo, hi}} + [4]i32~~mzipLo{...sw{4b3120, lo, hi}} } def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b} @@ -186,13 +186,13 @@ def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b} def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b} def packQ{a:T,b:T & w128i{T}} = packs{a,b} -def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}} -def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}} -def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}} -def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}} -def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}} -def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}} -def unpackQ{a:T, b:T & w128{T}} = unpack{a, b} + +def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b} +def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b} +def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}} + + +def unpackQ{a:T, b:T & w128{T}} = mzip{a, b} def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x} @@ -207,7 +207,7 @@ def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = { - def s{v} = s{unpackLo{v, v}} + def s{v} = s{mzipLo{v, v}} def s{v:V & V==T} = v s{x} >> (elwidth{T} - elwidth{X}) }