make /(m?zip(128)?|pack)(Lo|Hi)/ not have Lo/Hi as part of name

instead being an optional 3rg arg
This commit is contained in:
dzaima 2024-07-12 05:23:46 +03:00
parent a671ec9462
commit f79e4c6d0a
11 changed files with 42 additions and 49 deletions

View File

@ -146,13 +146,13 @@ Some may also support one scalar argument or arguments with different widths.
- `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]` - `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]`
- `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values - `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values
<!-- --> <!-- -->
- `zip{a:V,b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` - `zip{a:V, b:V} : tup{V, V}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}`
- `mzip{a:V,b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}` - `mzip{a:V, b:V} : tup{el_dbl{V}, el_dbl{V}}` - reinterpreted `zip{a, b}`
- `zipLo{a:V,b:V} : V` - `select{zip{a, b}, 0}` - `pack{a:V, b:V} : tup{}`
- `zipHi{a:V,b:V} : V` - `select{zip{a, b}, 1}` - `zip{a:V, b:V, k} : V` - `select{zip{a, b}, k}`
- `mzipLo{a:V,b:V} : V` - `select{mzip{a, b}, 0}` - `mzip{a:V, b:V, k} : V` - `select{mzip{a, b}, k}`
- `mzipHi{a:V,b:V} : V` - `select{mzip{a, b}, 1}` - `pack{a:V, b:V, k} : V` - `select{pack{a, b}, k}`
- `zip128{,Lo,Hi}`, `mzip128{,Lo,Hi}` - within-128-bit-lane versions of the above - `zip128`, `mzip128` - within-128-bit-lane versions of the above
## Mask stuff ## Mask stuff
@ -242,7 +242,6 @@ For float conversions, the used rounding mode is unspecified.
- `mla` - multiply-add - `mla` - multiply-add
- `mls` - multiply-subtract - `mls` - multiply-subtract
- `ornot` - a|~b - `ornot` - a|~b
- `packLo`, `packHi`
- `popc` - popcount of each byte - `popc` - popcount of each byte
- `rbit` - reverse bits in bytes - `rbit` - reverse bits in bytes
- `rev` - reverse elements in lanes - `rev` - reverse elements in lanes

View File

@ -1,5 +1,5 @@
def zip128Lo{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b} def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
def zip128Hi{a:T, b:T if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b} def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
# questionable pack; these saturate the argument # questionable pack; these saturate the argument
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b} def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}

View File

@ -161,10 +161,9 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
def { def {
absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half, absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half,
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packHi,packLo,packQ,pair,pdep, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,shufInd,storeLow, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,shufInd,storeLow,
topBlend,topMask,topMaskStore,topMaskStoreF,unord,vfold,widen,widenUpper, topBlend,topMask,topMaskStore,topMaskStoreF,unord,vfold,widen,widenUpper,
zipHi,zipLo
} }
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
@ -192,18 +191,15 @@ if_inline (hasarch{'X86_64'}) {
def {__adds,__subs,__sqrt,vec_broadcast,vec_make} def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
} }
def zip128Lo{a:T, b:T if width{T}==128} = zipLo{a, b} def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def zip128Hi{a:T, b:T if width{T}==128} = zipHi{a, b} def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def zip {a:T, b:T} = tup{zipLo {a,b}, zipHi {a,b}}
def zip128{a:T, b:T} = tup{zip128Lo{a,b}, zip128Hi{a,b}}
local def make_mzip{l, h} = tup{ def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
{a:T, b:T} => el_m{T} ~~ l{a, b}, def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k}
{a:T, b:T} => el_m{T} ~~ h{a, b}, local def extend kpair{op} = {
{a:T, b:T} => each{{z} => el_m{T}~~z{a, b}, tup{l, h}} def op{a:T, b:T} = tup{op{a,b,0}, op{a,b,1}}
} }
def {mzipLo, mzipHi, mzip } = make_mzip{zipLo, zipHi} extend ({...f}=>each{kpair,f}){pack, zip, zip128, mzip, mzip128}
def {mzip128Lo, mzip128Hi, mzip128} = make_mzip{zip128Lo, zip128Hi}
def packQ{{a, b}} = packQ{a, b} def packQ{{a, b}} = packQ{a, b}
def pair{{a, b}} = pair{a, b} def pair{{a, b}} = pair{a, b}

View File

@ -74,7 +74,7 @@ def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
} }
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = { def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
b:= T~~[8]u16**a b:= T~~[8]u16**a
exp:= T~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} exp:= T~~shuf{[4]i32, shuf16Lo{mzip{b,b,0}, 4b1100}, 4b1100}
(exp & make{T, 1<<(iota{16}&7)}) != T**0 (exp & make{T, 1<<(iota{16}&7)}) != T**0
} }

View File

@ -70,8 +70,8 @@ def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = {
def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = { def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = {
def r12 = mulw{w, x} def r12 = mulw{w, x}
rl:= packLo{r12} rl:= pack{...r12, 0}
rh:= packHi{r12} rh:= pack{...r12, 1}
tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}} tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}}
} }

View File

@ -70,20 +70,18 @@ def copyLane{dst:D=[_]E, di, src:S=[_]E, si if w128{D} and nvec{S}} = emit{D, nt
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i} def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
def zipLo{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b} def zip{a:T, b:T, 0 if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zipHi{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b} def zip{a:T, b:T, 1 if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def unzipLo{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y} def unzip{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vuzp1', T}, T~~x, T~~y}
def unzipHi{x:T, y:T if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y} def unzip{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vuzp2', T}, T~~x, T~~y}
def packLo{x:T, y:T if nvec{T}} = unzipLo{el_s{T}~~x, el_s{T}~~y} def pack{x:T, y:T, 0 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 0}
def packHi{x:T, y:T if nvec{T}} = unzipHi{el_s{T}~~x, el_s{T}~~y} def pack{x:T, y:T, 1 if nvec{T}} = unzip{el_s{T}~~x, el_s{T}~~y, 1}
def packLo{{x, y}} = packLo{x, y} def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzip{x,y,0}
def packHi{{x, y}} = packHi{x, y} def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzip{x,y,1}
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 2*range{vcount{T}}}} = T~~unzipLo{x,y}
def shufInd{x:T, y:T, {...is} if nvec{T,32} and same{is, 1+2*range{vcount{T}}}} = T~~unzipHi{x,y}
def trn1{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y} def trn{x:T, y:T, 0 if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
def trn2{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y} def trn{x:T, y:T, 1 if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}} def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
def sel{{x0:T}, i:I} = sel{T, x0, i} def sel{{x0:T}, i:I} = sel{T, x0, i}
@ -112,7 +110,7 @@ def narrow {T, x:X=[_]E if w128{X} and isfloat{T}!=isfloat{E} and width{T}<w
def narrowUpper{lowRes:L=[k]E, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[k*2]E, ntyp0{'vmovn_high', X}, lowRes, x} def narrowUpper{lowRes:L=[k]E, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[k*2]E, ntyp0{'vmovn_high', X}, lowRes, x}
def narrowPair{a:T=[_]E, b:T} = narrowUpper{narrow{w_h{E}, a}, b} def narrowPair{a:T=[_]E, b:T} = narrowUpper{narrow{w_h{E}, a}, b}
def narrowPair{a:T=[_]E, b:T if isint{E}} = packLo{a, b} def narrowPair{a:T=[_]E, b:T if isint{E}} = pack{a, b, 0}
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x} def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}} def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}

View File

@ -97,7 +97,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
e := words/2 e := words/2
c := V**init c := V**init
@for (rv, xv over e) { @for (rv, xv over e) {
rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})} rv = zip{...(@collect (j to 2) xor64{xv, j, c}), 0}
} }
if ((words & 1) == 1) { if ((words & 1) == 1) {
storeLow{*u64~~(rv+e), 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} storeLow{*u64~~(rv+e), 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}

View File

@ -9,7 +9,7 @@ local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} }
local def rev{up,t} = if (up) t else rev{t} local def rev{up,t} = if (up) t else rev{t}
def sel8{v, t, up} = sel8{v, rev{up,t}} def sel8{v, t, up} = sel8{v, rev{up,t}}
def zip{up, x} = (if (up) zipHi else zipLo){x,x} def zip{up, x if knum{up}} = zip{x, x, up}
# Fill last 4 bytes with last element, in each lane # Fill last 4 bytes with last element, in each lane
def spread{a:[_]T, ...up} = { def spread{a:[_]T, ...up} = {

View File

@ -149,7 +149,7 @@ fn slash{c==0, T if simd128{} and i16<=T and T<=i32}(w:*u64, x:arg{c,T}, r:*T, l
@for_special_buffered{r,8} (w in *u8~~w over i to sum) { @for_special_buffered{r,8} (w in *u8~~w over i to sum) {
ind := load{itab, w} ind := load{itab, w}
pc := popc_alt{w, ind, 8} pc := popc_alt{w, ind, 8}
v := mzipLo{j + I~~make{[2]u64, ind, 0}, select{top,0}} v := mzip{j + I~~make{[2]u64, ind, 0}, select{top,0}, 0}
def st{k, v:V} = store{*V~~r, k, v} def st{k, v:V} = store{*V~~r, k, v}
if (T==i16) st{0, v} if (T==i16) st{0, v}
else each{st, iota{2}, mzip{v, select{top,1}}} else each{st, iota{2}, mzip{v, select{top,1}}}
@ -169,7 +169,7 @@ fn slash{c==1, T if T<=i16 and shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum
ind := load{itab, w} ind := load{itab, w}
pc := popc_alt{w, ind, 8} pc := popc_alt{w, ind, 8}
s := V~~make{[2]u64, ind,0} s := V~~make{[2]u64, ind,0}
if (T==i16) { s+=s; s = V~~mzipLo{s, s+V**1} } if (T==i16) { s+=s; s = V~~mzip{s, s+V**1, 0} }
res := sel{V, load{*V~~(x+8*i)}, s} res := sel{V, load{*V~~(x+8*i)}, s}
if (T==i8) storeu{*u64~~r, extract{[2]u64~~res, 0}} if (T==i8) storeu{*u64~~r, extract{[2]u64~~res, 0}}
else store{*V~~r, 0, res} else store{*V~~r, 0, res}
@ -342,7 +342,7 @@ def pext_popc{x:T, m:T} = {
def pext_width {if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 def pext_width {if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2
def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32 def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32
def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = { def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = {
def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} def clmul{a, b} = zip{...@collect (j to 2) clmul{a,b,j}, 0}
m := m0 m := m0
x := x0 & m x := x0 & m
d := ~m << 1 # One bit of the position difference at x d := ~m << 1 # One bit of the position difference at x

View File

@ -88,7 +88,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
if (B) { if (B) {
if (any_nonC32{M, ...xs}) return{3} if (any_nonC32{M, ...xs}) return{3}
} else { } else {
def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) packHi{...xs} else packHi{orx,orx}} def bad = if (hasarch{'AARCH64'}) bitAny{if (length{xs}==2) pack{...xs,1} else pack{orx,orx,1}}
else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}} else ~andAllZero{orx, ~XV**maxvalue{w_h{X}}}
if (bad) return{lb{width{X}}-3} if (bad) return{lb{width{X}}-3}
} }
@ -140,7 +140,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
assert{hasarch{'AARCH64'}} assert{hasarch{'AARCH64'}}
def k = length{xs} def k = length{xs}
def int = each{cvt{i64,.}, xs} def int = each{cvt{i64,.}, xs}
def int32 = if (k==2) packLo{...int} else packLo{...int, [bulk]i64**0} def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0}
def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}} def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widenUpper}, 0, k}}
if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} if (~homAll{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}}

View File

@ -36,7 +36,7 @@ def __mul{a:T,b:T if [4]i32==T} = {
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs} def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
lo:= mu{a, b} lo:= mu{a, b}
hi:= mu{...sw{4b2301, a, b}} hi:= mu{...sw{4b2301, a, b}}
[4]i32~~mzipLo{...sw{4b3120, lo, hi}} [4]i32~~mzip{...sw{4b3120, lo, hi}, 0}
} }
# float arith # float arith
@ -71,8 +71,8 @@ def packQ{a:T,b:T if w128i{T}} = packs{a,b}
def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}} def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}}
def x86_postfix{(f32)} = '_ps' def x86_postfix{(f32)} = '_ps'
def x86_postfix{(f64)} = '_pd' def x86_postfix{(f64)} = '_pd'
def zipLo{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b} def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
def zipHi{a:T, b:T if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b} def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n} def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
@ -84,7 +84,7 @@ def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m
def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} def homMaskStoreF{p:*T, m:M, v:T if w128i{M} and w128{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}}
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = { def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
def s{v} = s{mzipLo{v, v}} def s{v} = s{mzip{v,v,0}}
def s{v:(T)} = v def s{v:(T)} = v
s{x} >> (elwidth{T} - elwidth{X}) s{x} >> (elwidth{T} - elwidth{X})
} }