make /(m?zip(128)?|pack)(Lo|Hi)/ not have Lo/Hi as part of name
instead being an optional 3rg arg
This commit is contained in:
parent
a671ec9462
commit
f79e4c6d0a
@ -146,13 +146,13 @@ Some may also support one scalar argument or arguments with different widths.
|
||||
- `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]`
|
||||
- `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values
|
||||
<!-- -->
|
||||
- `zip{a:V,b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}`
|
||||
- `mzip{a:V,b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}`
|
||||
- `zipLo{a:V,b:V} : V` - `select{zip{a, b}, 0}`
|
||||
- `zipHi{a:V,b:V} : V` - `select{zip{a, b}, 1}`
|
||||
- `mzipLo{a:V,b:V} : V` - `select{mzip{a, b}, 0}`
|
||||
- `mzipHi{a:V,b:V} : V` - `select{mzip{a, b}, 1}`
|
||||
- `zip128{,Lo,Hi}`, `mzip128{,Lo,Hi}` - within-128-bit-lane versions of the above
|
||||
- `zip{a:V, b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}`
|
||||
- `mzip{a:V, b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}`
|
||||
- `pack{a:V, b:V} : tup{}`
|
||||
- `zip{a:V, b:V, k} : V` - `select{zip{a, b}, k}`
|
||||
- `mzip{a:V, b:V, k} : V` - `select{mzip{a, b}, k}`
|
||||
- `pack{a:V, b:V, k} : V` - `select{pack{a, b}, k}`
|
||||
- `zip128`, `mzip128` - within-128-bit-lane versions of the above
|
||||
|
||||
## Mask stuff
|
||||
|
||||
@ -242,7 +242,6 @@ For float conversions, the used rounding mode is unspecified.
|
||||
- `mla` - multiply-add
|
||||
- `mls` - multiply-subtract
|
||||
- `ornot` - a|~b
|
||||
- `packLo`, `packHi`
|
||||
- `popc` - popcount of each byte
|
||||
- `rbit` - reverse bits in bytes
|
||||
- `rev` - reverse elements in lanes
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
def zip128Lo{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip128Hi{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
# questionable pack; these saturate the argument
|
||||
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
||||
|
||||
@ -161,10 +161,9 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
|
||||
def {
|
||||
absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packHi,packLo,packQ,pair,pdep,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,shufInd,storeLow,
|
||||
topBlend,topMask,topMaskStore,topMaskStoreF,unord,vfold,widen,widenUpper,
|
||||
zipHi,zipLo
|
||||
}
|
||||
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
@ -192,18 +191,15 @@ if_inline (hasarch{'X86_64'}) {
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
|
||||
}
|
||||
|
||||
def zip128Lo{a:T, b:T if width{T}==128} = zipLo{a, b}
|
||||
def zip128Hi{a:T, b:T if width{T}==128} = zipHi{a, b}
|
||||
def zip {a:T, b:T} = tup{zipLo {a,b}, zipHi {a,b}}
|
||||
def zip128{a:T, b:T} = tup{zip128Lo{a,b}, zip128Hi{a,b}}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
|
||||
local def make_mzip{l, h} = tup{
|
||||
{a:T, b:T} => el_m{T} ~~ l{a, b},
|
||||
{a:T, b:T} => el_m{T} ~~ h{a, b},
|
||||
{a:T, b:T} => each{{z} => el_m{T}~~z{a, b}, tup{l, h}}
|
||||
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
|
||||
def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k}
|
||||
local def extend kpair{op} = {
|
||||
def op{a:T, b:T} = tup{op{a,b,0}, op{a,b,1}}
|
||||
}
|
||||
def {mzipLo, mzipHi, mzip } = make_mzip{zipLo, zipHi}
|
||||
def {mzip128Lo, mzip128Hi, mzip128} = make_mzip{zip128Lo, zip128Hi}
|
||||
extend ({...f}=>each{kpair,f}){pack, zip, zip128, mzip, mzip128}
|
||||
|
||||
def packQ{{a, b}} = packQ{a, b}
|
||||
def pair{{a, b}} = pair{a, b}
|
||||
|
||||
@ -74,7 +74,7 @@ def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
|
||||
}
|
||||
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
|
||||
b:= T~~[8]u16**a
|
||||
exp:= T~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
|
||||
exp:= T~~shuf{[4]i32, shuf16Lo{mzip{b,b,0}, 4b1100}, 4b1100}
|
||||
(exp & make{T, 1<<(iota{16}&7)}) != T**0
|
||||
}
|
||||
|
||||
|
||||
@ -70,8 +70,8 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
|
||||
|
||||
def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = {
|
||||
def r12 = mulw{w, x}
|
||||
rl:= packLo{r12}
|
||||
rh:= packHi{r12}
|
||||
rl:= pack{...r12, 0}
|
||||
rh:= pack{...r12, 1}
|
||||
tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}}
|
||||
}
|
||||
|
||||
|
||||
@ -70,20 +70,18 @@ def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, nt
|
||||
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
|
||||
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||
|
||||
def zipLo{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zipHi{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
||||
def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
||||
|
||||
def unzipLo{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
|
||||
def unzipHi{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}
|
||||
def packLo{x:T, y:T if nvec{T}} = unzipLo{el_s{T}~~x, el_s{T}~~y}
|
||||
def packHi{x:T, y:T if nvec{T}} = unzipHi{el_s{T}~~x, el_s{T}~~y}
|
||||
def packLo{{x, y}} = packLo{x, y}
|
||||
def packHi{{x, y}} = packHi{x, y}
|
||||
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzipLo{x,y}
|
||||
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzipHi{x,y}
|
||||
def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
|
||||
def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}
|
||||
def pack{x:T, y:T, 0 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 0}
|
||||
def pack{x:T, y:T, 1 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 1}
|
||||
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzip{x,y,0}
|
||||
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzip{x,y,1}
|
||||
|
||||
def trn1{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
|
||||
def trn2{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
|
||||
def trn{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
|
||||
def trn{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
|
||||
|
||||
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
|
||||
def sel{{x0:T}, i:I} = sel{T, x0, i}
|
||||
@ -112,7 +110,7 @@ def narrow {T, x:X=[_]E if w128{X} and isfloat{T}!=isfloat{E} and width{T}<w
|
||||
|
||||
def narrowUpper{lowRes:L=[k]E, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[k*2]E, ntyp0{'vmovn_high', X}, lowRes, x}
|
||||
def narrowPair{a:T=[_]E, b:T} = narrowUpper{narrow{w_h{E}, a}, b}
|
||||
def narrowPair{a:T=[_]E, b:T if isint{E}} = packLo{a, b}
|
||||
def narrowPair{a:T=[_]E, b:T if isint{E}} = pack{a, b, 0}
|
||||
|
||||
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
|
||||
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
|
||||
|
||||
@ -97,7 +97,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
|
||||
e := words/2
|
||||
c := V**init
|
||||
@for (rv, xv over e) {
|
||||
rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})}
|
||||
rv = zip{...(@collect (j to 2) xor64{xv, j, c}), 0}
|
||||
}
|
||||
if ((words & 1) == 1) {
|
||||
storeLow{*u64~~(rv+e), 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
|
||||
|
||||
@ -9,7 +9,7 @@ local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} }
|
||||
local def rev{up,t} = if (up) t else rev{t}
|
||||
def sel8{v, t, up} = sel8{v, rev{up,t}}
|
||||
|
||||
def zip{up, x} = (if (up) zipHi else zipLo){x,x}
|
||||
def zip{up, x if knum{up}} = zip{x, x, up}
|
||||
|
||||
# Fill last 4 bytes with last element, in each lane
|
||||
def spread{a:[_]T, ...up} = {
|
||||
|
||||
@ -149,7 +149,7 @@ fn slash{c==0, T if simd128{} and i16<=T and T<=i32}(w:*u64, x:arg{c,T}, r:*T, l
|
||||
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
|
||||
ind := load{itab, w}
|
||||
pc := popc_alt{w, ind, 8}
|
||||
v := mzipLo{j + I~~make{[2]u64, ind, 0}, select{top,0}}
|
||||
v := mzip{j + I~~make{[2]u64, ind, 0}, select{top,0}, 0}
|
||||
def st{k, v:V} = store{*V~~r, k, v}
|
||||
if (T==i16) st{0, v}
|
||||
else each{st, iota{2}, mzip{v, select{top,1}}}
|
||||
@ -169,7 +169,7 @@ fn slash{c==1, T if T<=i16 and shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum
|
||||
ind := load{itab, w}
|
||||
pc := popc_alt{w, ind, 8}
|
||||
s := V~~make{[2]u64, ind,0}
|
||||
if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} }
|
||||
if (T==i16) { s+=s; s = V~~mzip{s, s+V**1, 0} }
|
||||
res := sel{V, load{*V~~(x+8*i)}, s}
|
||||
if (T==i8) storeu{*u64~~r, extract{[2]u64~~res, 0}}
|
||||
else store{*V~~r, 0, res}
|
||||
@ -342,7 +342,7 @@ def pext_popc{x:T, m:T} = {
|
||||
def pext_width {if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2
|
||||
def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32
|
||||
def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = {
|
||||
def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}}
|
||||
def clmul{a, b} = zip{...@collect (j to 2) clmul{a,b,j}, 0}
|
||||
m := m0
|
||||
x := x0 & m
|
||||
d := ~m << 1 # One bit of the position difference at x
|
||||
|
||||
@ -88,7 +88,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
if (B) {
|
||||
if (any_nonC32{M, ...xs}) return{3}
|
||||
} else {
|
||||
def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) packHi{...xs} else packHi{orx,orx}}
|
||||
def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}}
|
||||
else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}}
|
||||
if (bad) return{lb{width{X}}-3}
|
||||
}
|
||||
@ -140,7 +140,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
assert{hasarch{'AARCH64'}}
|
||||
def k = length{xs}
|
||||
def int = each{cvt{i64,.}, xs}
|
||||
def int32 = if (k==2) packLo{...int} else packLo{...int, [bulk]i64**0}
|
||||
def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0}
|
||||
def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}}
|
||||
|
||||
if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}}
|
||||
|
||||
@ -36,7 +36,7 @@ def __mul{a:T,b:T if [4]i32==T} = {
|
||||
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
||||
lo:= mu{a, b}
|
||||
hi:= mu{...sw{4b2301, a, b}}
|
||||
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
|
||||
[4]i32~~mzip{...sw{4b3120, lo, hi}, 0}
|
||||
}
|
||||
|
||||
# float arith
|
||||
@ -71,8 +71,8 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b}
|
||||
def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}}
|
||||
def x86_postfix{(f32)} = '_ps'
|
||||
def x86_postfix{(f64)} = '_pd'
|
||||
def zipLo{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zipHi{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
|
||||
@ -84,7 +84,7 @@ def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m
|
||||
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
|
||||
def s{v} = s{mzipLo{v, v}}
|
||||
def s{v} = s{mzip{v,v,0}}
|
||||
def s{v:(T)} = v
|
||||
s{x} >> (elwidth{T} - elwidth{X})
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user