width{eltype{T}} → elwidth{T}
This commit is contained in:
parent
2a1611ac81
commit
444268f051
@ -5,12 +5,12 @@ local def wgen256{F} = {
|
||||
def r{T} = 0
|
||||
def r{T & w256{T}} = F{eltype{T}}
|
||||
def r{T,w} = 0
|
||||
def r{T,w & w256{T}} = F{eltype{T}} & (width{eltype{T}}==w)
|
||||
def r{T,w & w256{T}} = F{eltype{T}} & (elwidth{T}==w)
|
||||
def r{T & ~isvec{T}} = 0
|
||||
r
|
||||
}
|
||||
def w256{T,w} = 0
|
||||
def w256{T,w & w256{T}} = width{eltype{T}}==w
|
||||
def w256{T,w & w256{T}} = elwidth{T}==w
|
||||
def w256i = wgen256{{T} => isint{T}}
|
||||
def w256s = wgen256{{T} => isint{T} & issigned{T}}
|
||||
def w256u = wgen256{{T} => isint{T} & isunsigned{T}}
|
||||
@ -104,7 +104,7 @@ def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_m
|
||||
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
|
||||
|
||||
def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a}
|
||||
def cvt{F, T, a:A & w256{A} & width{F}<width{eltype{T}}} = cvt{F, T, half{a, 0}}
|
||||
def cvt{F, T, a:A & w256{A} & width{F}<elwidth{T}} = cvt{F, T, half{a, 0}}
|
||||
|
||||
# structural operations
|
||||
def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i}
|
||||
|
||||
@ -105,7 +105,7 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void,
|
||||
|
||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
||||
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
|
||||
def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x}
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
@ -118,7 +118,7 @@ def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm2
|
||||
def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
|
||||
# assumes all bits are the same in each mask item
|
||||
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & width{eltype{M}}!=16} = blend{f, t, m}
|
||||
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & elwidth{M}!=16} = blend{f, t, m}
|
||||
def blendF{f:T, t:T, m:M & w256{T} & w256{M,16}} = blend{f, t, [32]i8 ~~ m}
|
||||
|
||||
def shuf{L==[4]u32, x:T, n & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
|
||||
@ -182,7 +182,7 @@ def ucvt{T, x:X & w256u{X,64} & T==u32} = to_el{T, sel{[8]i32, x, make{[8]i32, 2
|
||||
def ucvt{T, x:X & w256u{X,64} & T==u16} = to_el{T, sel{[16]i8, ucvt{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}}
|
||||
def ucvt{T, x:X & w256u{X,64} & T== u8} = to_el{T, sel{[16]i8, ucvt{u32,x}, make{[32]i8, 4*iota{32}}}}
|
||||
|
||||
def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO check for not being f64/i64
|
||||
def ucvt{T, x:X & w256{X} & width{T}==elwidth{X}} = to_el{T, x} # TODO check for not being f64/i64
|
||||
|
||||
|
||||
def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
|
||||
|
||||
@ -7,6 +7,8 @@ oper ** broadcast infix right 55
|
||||
def Size = u64
|
||||
def load{x} = *x
|
||||
|
||||
def elwidth{T} = width{eltype{T}}
|
||||
|
||||
# ceiling divide
|
||||
def cdiv{a,b} = (a+b-1)/b
|
||||
|
||||
@ -20,12 +22,12 @@ def bit {k,x} = x & (1<<k) # get the k-th bit
|
||||
# Convert tuple to number in little-endian base b
|
||||
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
|
||||
|
||||
# hints
|
||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||
def assert{x & x==0} = assert{'failed assertion'}
|
||||
def assert{x & x==1} = 1
|
||||
def assert{x:u1} = { if (not x) emit{void, '__builtin_unreachable'} }
|
||||
|
||||
|
||||
# various checks
|
||||
def isreg = kreg
|
||||
def isconst = kcon
|
||||
@ -49,6 +51,7 @@ def anyInt{x & isreg{x}|isconst{x}} = isint{x}
|
||||
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
||||
def trunc{T, x & knum{x}} = cast{T, x}
|
||||
|
||||
# reinterpret vector element type to signed/unsigned
|
||||
def ty_s{w} = primtype{'i', w}
|
||||
def ty_u{w} = primtype{'u', w}
|
||||
|
||||
@ -60,6 +63,7 @@ def ty_s{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def ty_u{x:T & isvec{T}} = ty_u{T}~~x
|
||||
def ty_s{x:T & isvec{T}} = ty_s{T}~~x
|
||||
|
||||
# reinterpret vector as one with element type E
|
||||
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
|
||||
def to_el{E, x:T} = to_el{E,T} ~~ x
|
||||
|
||||
|
||||
@ -56,7 +56,7 @@ def spreadBits{T==[32]u8, a:u32} = {
|
||||
e:= make{[32]u8, 1<<tail{3, idxs}}
|
||||
e == (d&e)
|
||||
}
|
||||
def spreadBits{T, a & vcount{T} <= width{eltype{T}} & w256u{T}} = {
|
||||
def spreadBits{T, a & vcount{T} <= elwidth{T} & w256u{T}} = {
|
||||
b:= make{T, 1<<iota{vcount{T}}}
|
||||
b == (b & T ~~ to_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
|
||||
}
|
||||
|
||||
@ -24,8 +24,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
|
||||
# + & -
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = anyneg{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = anyneg{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
|
||||
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ def runner{u, R, F} = {
|
||||
|
||||
def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation
|
||||
|
||||
def toggleTop{x:X} = x ^ X**(1<<(width{eltype{X}}-1))
|
||||
def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1))
|
||||
def run{F==__sub, OO, M, w:VU, x:VU & is_u{VU}} = { # 'b'-'a'
|
||||
def VS = ty_s{VU}
|
||||
run{F, OO, M, VS~~toggleTop{w}, VS~~toggleTop{x}}
|
||||
|
||||
@ -7,7 +7,7 @@ mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~
|
||||
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
||||
|
||||
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
||||
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(width{eltype{T}}/8))}
|
||||
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
|
||||
|
||||
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
||||
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
||||
@ -46,7 +46,7 @@ def storeLow{ptr:P, w, x:T & w128{T} & w==64} = emit{void, '_mm_storeu_si64', pt
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==16} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==32} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/width{eltype{T}}](eltype{T})~~ptr, 0, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/elwidth{T}](eltype{T})~~ptr, 0, half{x, 0}}
|
||||
|
||||
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = eltype{P} ~~ load{*[16]u8 ~~ ptr}
|
||||
@ -55,9 +55,9 @@ def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_m
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
|
||||
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w<256} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}}
|
||||
def loadLow{ptr:P, w & width{eltype{P}} == w} = load{*eltype{P} ~~ ptr}
|
||||
def loadLow{ptr:P, w & elwidth{P} == w} = load{*eltype{P} ~~ ptr}
|
||||
|
||||
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/width{eltype{P}})), w}
|
||||
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
|
||||
|
||||
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
|
||||
def storeBatch{ptr:P, n, x:T, M} = {
|
||||
|
||||
@ -5,12 +5,12 @@ local def ngen128{F} = {
|
||||
def r{T} = 0
|
||||
def r{T & n128{T}} = F{eltype{T}}
|
||||
def r{T,w} = 0
|
||||
def r{T,w & n128{T}} = F{eltype{T}} & (width{eltype{T}}==w)
|
||||
def r{T,w & n128{T}} = F{eltype{T}} & (elwidth{T}==w)
|
||||
def r{T & ~isvec{T}} = 0
|
||||
r
|
||||
}
|
||||
def n128{T,w} = 0
|
||||
def n128{T,w & n128{T}} = width{eltype{T}}==w
|
||||
def n128{T,w & n128{T}} = elwidth{T}==w
|
||||
def n128i = ngen128{{T} => isint{T}}
|
||||
def n128s = ngen128{{T} => isint{T} & issigned{T}}
|
||||
def n128u = ngen128{{T} => isint{T} & isunsigned{T}}
|
||||
@ -44,11 +44,11 @@ def andnot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vbicq', T}, a, b}
|
||||
def ornot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vornq', T}, a, b}
|
||||
def min{a:T,b:T & n128i{T}} = emit{T, ntyp{'vminq', T}, a, b} # TODO float - there are multiple options
|
||||
def max{a:T,b:T & n128i{T}} = emit{T, ntyp{'vmaxq', T}, a, b}
|
||||
def __shl{a:T,b:S & n128i{T} & n128i{S} & width{eltype{T}}==width{eltype{S}}} = emit{T, ntyp{'vshlq', T}, a, ty_s{b}}
|
||||
def __shl{a:T,b:S & n128i{T} & n128i{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshlq', T}, a, ty_s{b}}
|
||||
|
||||
def __shl{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshlq_n', T}, a, b}; def __shl{a:T,b==0 & n128i{T}} = a
|
||||
def __shr{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshrq_n', T}, a, b}; def __shr{a:T,b==0 & n128i{T}} = a
|
||||
def bblend{f:T, t:T, m:M & n128{T} & n128u{M} & width{eltype{T}}==width{eltype{M}}} = emit{T, ntyp{'vbslq', T}, m, t, f}
|
||||
def bblend{f:T, t:T, m:M & n128{T} & n128u{M} & elwidth{T}==elwidth{M}} = emit{T, ntyp{'vbslq', T}, m, t, f}
|
||||
|
||||
def __neg{a:T & (n128s{T}|n128f{T})} = emit{T, ntyp{'vnegq', T}, a}
|
||||
def __not{a:T & n128u{T}} = emit{T, ntyp{'vmvnq', T}, a}
|
||||
|
||||
@ -11,7 +11,7 @@ def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}}
|
||||
|
||||
# Fill last 4 bytes with last element, in each lane
|
||||
def spread{a:VT} = {
|
||||
def w = width{eltype{VT}}
|
||||
def w = elwidth{VT}
|
||||
def b = w/8
|
||||
if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}}; else a
|
||||
}
|
||||
|
||||
@ -10,15 +10,15 @@ include 'util/tup'
|
||||
oper &~ andnot infix none 35
|
||||
|
||||
# def:T - masked original content
|
||||
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
|
||||
# b:B - pointer to data to index; if width{B}<elwidth{T}, padding bytes are garbage read after wanted position
|
||||
# idx - actual (unscaled) index list
|
||||
def gather{def:T, b:B, idx:[8]i32, M & w256{T,32}} = {
|
||||
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', def, *void~~b, idx, M{T,'to sign bits'}, width{eltype{B}}/8}
|
||||
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, width{eltype{B}}/8}
|
||||
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
|
||||
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, elwidth{B}/8}
|
||||
}
|
||||
def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
||||
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', def, *void~~b, idx, M{T,'to sign bits'}, width{eltype{B}}/8}
|
||||
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, width{eltype{B}}/8}
|
||||
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
|
||||
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, elwidth{B}/8}
|
||||
}
|
||||
|
||||
def wrapChk{cw0, VI,xlf, M} = {
|
||||
|
||||
@ -14,7 +14,7 @@ def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = {
|
||||
}
|
||||
def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = {
|
||||
def TS = ty_s{TU}
|
||||
def h = 1 << (width{eltype{TU}}-1)
|
||||
def h = 1 << (elwidth{TU}-1)
|
||||
(TS~~(x-TU**(start-h))) < TS**(count-h)
|
||||
}
|
||||
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1}
|
||||
@ -32,7 +32,7 @@ def fold{F, x:T} = {
|
||||
}
|
||||
def fold{F, x:T & w128{T}} = {
|
||||
c:= x
|
||||
def EW = width{eltype{T}}
|
||||
def EW = elwidth{T}
|
||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
||||
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
|
||||
|
||||
@ -5,12 +5,12 @@ local def wgen128{F} = {
|
||||
def r{T} = 0
|
||||
def r{T & w128{T}} = F{eltype{T}}
|
||||
def r{T,w} = 0
|
||||
def r{T,w & w128{T}} = F{eltype{T}} & (width{eltype{T}}==w)
|
||||
def r{T,w & w128{T}} = F{eltype{T}} & (elwidth{T}==w)
|
||||
def r{T & ~isvec{T}} = 0
|
||||
r
|
||||
}
|
||||
def w128{T,w} = 0
|
||||
def w128{T,w & w128{T}} = width{eltype{T}}==w
|
||||
def w128{T,w & w128{T}} = elwidth{T}==w
|
||||
def w128i = wgen128{{T} => isint{T}}
|
||||
def w128s = wgen128{{T} => isint{T} & issigned{T}}
|
||||
def w128u = wgen128{{T} => isint{T} & isunsigned{T}}
|
||||
@ -159,7 +159,7 @@ def blend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv
|
||||
def blend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
|
||||
# assumes all bits are the same in each mask item
|
||||
def blendF{f:T, t:T, m:M & w128{T} & w128{M} & width{eltype{M}}!=16} = blend{f, t, m}
|
||||
def blendF{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = blend{f, t, m}
|
||||
def blendF{f:T, t:T, m:M & w128{T} & w128{M,16}} = blend{f, t, [16]i8,m}
|
||||
|
||||
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user