158 lines
8.1 KiB
Plaintext
158 lines
8.1 KiB
Plaintext
def nvec{T} = 0
|
|
def nvec{T if isvec{T}} = (width{T}==64) | (width{T}==128)
|
|
def nvec{T,w} = 0
|
|
def nvec{T,w if nvec{T}} = elwidth{T}==w
|
|
def nveci = genchk{nvec, isint}
|
|
def nvecs = genchk{nvec, issigned}
|
|
def nvecu = genchk{nvec, isunsigned}
|
|
def nvecf = genchk{nvec, isfloat}
|
|
|
|
|
|
def reinterpret{T, v if same{'pointer',typekind{T}} and ktup{v}} = { tmp:T=v }
|
|
|
|
def nty{T} = {
|
|
def q = quality{T}
|
|
merge{if (q=='i') 's' else q, fmtnat{width{T}}}
|
|
}
|
|
def nty{T if isvec{T}} = nty{eltype{T}}
|
|
def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}}
|
|
def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}}
|
|
def ntyp0{S, T} = merge{S, '_', nty{T}}
|
|
|
|
def addwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b}
|
|
def subwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b}
|
|
def mulwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b}
|
|
def mulwHi{a:T,b:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b}
|
|
def mulw {a:T,b:T if w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}}
|
|
|
|
def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
|
|
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
|
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
|
|
|
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
|
|
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
|
|
|
|
def addpw { x:T if nveci{T} and elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
|
|
def addpwa{a:D, x:T if nveci{T} and elwidth{T}<=32 and D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
|
|
def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
|
|
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
|
|
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
|
|
def rev{w, x:T if w==elwidth{T}} = x
|
|
def rev{w==16, x:T if elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
|
|
def rev{w==32, x:T if elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
|
|
def rev{w==64, x:T if elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
|
|
def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
|
|
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
|
|
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
|
|
|
|
def fold_add {a:T if nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
|
|
def fold_addw{a:T if nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a}
|
|
def fold_min {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
|
|
def fold_max {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
|
|
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x}
|
|
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x}
|
|
def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x}
|
|
|
|
def storeLow{ptr:P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
|
|
def storeLow{ptr:P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
|
|
|
|
def loadLow{ptr:P, w if w<=64} = { # a broadcast load
|
|
def T=eltype{P}
|
|
def L=re_el{ty_u{w}, T}
|
|
T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
|
|
}
|
|
def loadLow{ptr:P, w if w==elwidth{P}} = load{ptr}
|
|
|
|
|
|
|
|
|
|
def undefPromote{T, x:X if w64{X} and w128{T} and eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯
|
|
def half{x:T, n==0 if w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x}
|
|
def half{x:T, n==1 if w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x}
|
|
def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
|
|
def copyLane{dst:D, di, src:S, si if w64{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
|
|
def copyLane{dst:D, di, src:S, si if w128{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
|
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
|
|
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
|
|
|
def zipLo{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
|
|
def zipHi{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
|
|
def zip{a:T, b:T if nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
|
|
|
def packLo{x:T, y:T if nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
|
|
def packHi{x:T, y:T if nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
|
|
def packLo{{x, y}} = packLo{x, y}
|
|
def packHi{{x, y}} = packHi{x, y}
|
|
|
|
def trn1{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
|
|
def trn2{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
|
|
|
|
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
|
|
|
|
|
|
|
|
local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers
|
|
|
|
def cvt{T==f64, x:X if nveci{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_f64', X}, x}
|
|
def cvt{T==i64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_s64', X}, x}
|
|
def cvt{T==u64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_u64', X}, x}
|
|
|
|
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x}
|
|
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}}
|
|
def widen{T, x:X if w64{X} and isfloat{eltype{T}}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}}
|
|
def widen{T, x:X if w128{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}}
|
|
|
|
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}}
|
|
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x}
|
|
def narrow{T, x:X if w128{X} and isfloat{T}!=isfloat{eltype{X}} and width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
|
|
|
|
def narrowUpper{lowRes:L, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
|
|
def narrowPair{a:T, b:T} = narrowUpper{narrow{w_h{eltype{T}}, a}, b}
|
|
def narrowPair{a:T, b:T if isint{eltype{T}}} = packLo{a, b}
|
|
|
|
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
|
|
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
|
|
|
|
def bitAny{x:T} = fold_max{re_el{u32, x}}!=0
|
|
def bitAll{x:T} = fold_min{re_el{u32, x}}==0xffff_ffff
|
|
|
|
def topAny{x:T if nvec{T}} = fold_min{ty_s{x}}<0
|
|
def topAll{x:T if nvec{T}} = fold_max{ty_s{x}}<0
|
|
def homAny{x:T if nvec{T}} = bitAny{x}
|
|
def homAll{x:T if nvec{T}} = bitAll{x}
|
|
|
|
|
|
def homMask{x:T if nvecu{T} and elwidth{T}>=vcount{T}} = {
|
|
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
|
|
}
|
|
def homMask{x:T if nvecu{T} and T==[16]u8} = {
|
|
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, 0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15}}
|
|
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
|
|
}
|
|
def homMask{a:T,b:T if T==[16]u8} = {
|
|
m:= make{[16]u8, 1<<(iota{16}&7)}
|
|
fold_add{addpw{addpw{addp{a&m, b&m}}}<<make{[4]u32,iota{4}*8}}
|
|
}
|
|
def homMask{a:T,b:T,c:T,d:T if T==[16]u8} = {
|
|
m:= make{[16]u8, 1<<(iota{16}&7)}
|
|
t1:= addp{a&m, b&m}
|
|
t2:= addp{c&m, d&m}
|
|
t3:= addp{t1, t2}
|
|
extract{[2]u64~~addp{t3,t3},0}
|
|
}
|
|
def homMask{...as if length{as}>1 and elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
|
|
def homMask{a:T,b:T if vcount{T}*2<=elwidth{T}} = {
|
|
def n = vcount{T}
|
|
truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
|
|
}
|
|
|
|
def andAllZero{x:T, y:T if nveci{T}} = ~bitAny{x&y}
|
|
|
|
def homMaskX{a:T if eltype{T}!=u64} = {
|
|
def h = elwidth{T}/2
|
|
tup{h, truncBits{vcount{T}*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
|
|
}
|
|
|
|
|
|
def homMaskStoreF{p:P, m:M, v:T if nveci{M} and nvec{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|