width{eltype{T}} → elwidth{T}

This commit is contained in:
dzaima 2022-12-12 00:07:50 +02:00
parent 2a1611ac81
commit 444268f051
11 changed files with 34 additions and 30 deletions

View File

@ -5,12 +5,12 @@ local def wgen256{F} = {
def r{T} = 0
def r{T & w256{T}} = F{eltype{T}}
def r{T,w} = 0
def r{T,w & w256{T}} = F{eltype{T}} & (width{eltype{T}}==w)
def r{T,w & w256{T}} = F{eltype{T}} & (elwidth{T}==w)
def r{T & ~isvec{T}} = 0
r
}
def w256{T,w} = 0
def w256{T,w & w256{T}} = width{eltype{T}}==w
def w256{T,w & w256{T}} = elwidth{T}==w
def w256i = wgen256{{T} => isint{T}}
def w256s = wgen256{{T} => isint{T} & issigned{T}}
def w256u = wgen256{{T} => isint{T} & isunsigned{T}}
@ -104,7 +104,7 @@ def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_m
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a}
def cvt{F, T, a:A & w256{A} & width{F}<width{eltype{T}}} = cvt{F, T, half{a, 0}}
def cvt{F, T, a:A & w256{A} & width{F}<elwidth{T}} = cvt{F, T, half{a, 0}}
# structural operations
def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i}

View File

@ -105,7 +105,7 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void,
# maskstore with all cases defined, at the cost of not being a single instruction
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x}
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
@ -118,7 +118,7 @@ def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm2
def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & width{eltype{M}}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & elwidth{M}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w256{T} & w256{M,16}} = blend{f, t, [32]i8 ~~ m}
def shuf{L==[4]u32, x:T, n & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
@ -182,7 +182,7 @@ def ucvt{T, x:X & w256u{X,64} & T==u32} = to_el{T, sel{[8]i32, x, make{[8]i32, 2
def ucvt{T, x:X & w256u{X,64} & T==u16} = to_el{T, sel{[16]i8, ucvt{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}}
def ucvt{T, x:X & w256u{X,64} & T== u8} = to_el{T, sel{[16]i8, ucvt{u32,x}, make{[32]i8, 4*iota{32}}}}
def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO check for not being f64/i64
def ucvt{T, x:X & w256{X} & width{T}==elwidth{X}} = to_el{T, x} # TODO check for not being f64/i64
def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}

View File

@ -7,6 +7,8 @@ oper ** broadcast infix right 55
def Size = u64
def load{x} = *x
def elwidth{T} = width{eltype{T}}
# ceiling divide
def cdiv{a,b} = (a+b-1)/b
@ -20,12 +22,12 @@ def bit {k,x} = x & (1<<k) # get the k-th bit
# Convert tuple to number in little-endian base b
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
# hints
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
def assert{x & x==0} = assert{'failed assertion'}
def assert{x & x==1} = 1
def assert{x:u1} = { if (not x) emit{void, '__builtin_unreachable'} }
# various checks
def isreg = kreg
def isconst = kcon
@ -49,6 +51,7 @@ def anyInt{x & isreg{x}|isconst{x}} = isint{x}
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
def trunc{T, x & knum{x}} = cast{T, x}
# reinterpret vector element type to signed/unsigned
def ty_s{w} = primtype{'i', w}
def ty_u{w} = primtype{'u', w}
@ -60,6 +63,7 @@ def ty_s{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
def ty_u{x:T & isvec{T}} = ty_u{T}~~x
def ty_s{x:T & isvec{T}} = ty_s{T}~~x
# reinterpret vector as one with element type E
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
def to_el{E, x:T} = to_el{E,T} ~~ x

View File

@ -56,7 +56,7 @@ def spreadBits{T==[32]u8, a:u32} = {
e:= make{[32]u8, 1<<tail{3, idxs}}
e == (d&e)
}
def spreadBits{T, a & vcount{T} <= width{eltype{T}} & w256u{T}} = {
def spreadBits{T, a & vcount{T} <= elwidth{T} & w256u{T}} = {
b:= make{T, 1<<iota{vcount{T}}}
b == (b & T ~~ to_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}

View File

@ -24,8 +24,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
# + & -
def arithChk1{F==__add, M, w:T, x:T, r:T} = anyneg{M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T} = anyneg{M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
@ -83,7 +83,7 @@ def runner{u, R, F} = {
def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation
def toggleTop{x:X} = x ^ X**(1<<(width{eltype{X}}-1))
def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1))
def run{F==__sub, OO, M, w:VU, x:VU & is_u{VU}} = { # 'b'-'a'
def VS = ty_s{VU}
run{F, OO, M, VS~~toggleTop{w}, VS~~toggleTop{x}}

View File

@ -7,7 +7,7 @@ mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
mask256:*i64 = merge{4 ** -1, 4 ** 0}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(width{eltype{T}}/8))}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
# get mask of first n items; 0 ≤ n ≤ vcount{T}
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
@ -46,7 +46,7 @@ def storeLow{ptr:P, w, x:T & w128{T} & w==64} = emit{void, '_mm_storeu_si64', pt
def storeLow{ptr:P, w, x:T & w256{T} & w==16} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==32} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/width{eltype{T}}](eltype{T})~~ptr, 0, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/elwidth{T}](eltype{T})~~ptr, 0, half{x, 0}}
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = eltype{P} ~~ load{*[16]u8 ~~ ptr}
@ -55,9 +55,9 @@ def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_m
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:P, w & w256{eltype{P}} & w<256} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}}
def loadLow{ptr:P, w & width{eltype{P}} == w} = load{*eltype{P} ~~ ptr}
def loadLow{ptr:P, w & elwidth{P} == w} = load{*eltype{P} ~~ ptr}
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/width{eltype{P}})), w}
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
def storeBatch{ptr:P, n, x:T, M} = {

View File

@ -5,12 +5,12 @@ local def ngen128{F} = {
def r{T} = 0
def r{T & n128{T}} = F{eltype{T}}
def r{T,w} = 0
def r{T,w & n128{T}} = F{eltype{T}} & (width{eltype{T}}==w)
def r{T,w & n128{T}} = F{eltype{T}} & (elwidth{T}==w)
def r{T & ~isvec{T}} = 0
r
}
def n128{T,w} = 0
def n128{T,w & n128{T}} = width{eltype{T}}==w
def n128{T,w & n128{T}} = elwidth{T}==w
def n128i = ngen128{{T} => isint{T}}
def n128s = ngen128{{T} => isint{T} & issigned{T}}
def n128u = ngen128{{T} => isint{T} & isunsigned{T}}
@ -44,11 +44,11 @@ def andnot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vbicq', T}, a, b}
def ornot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vornq', T}, a, b}
def min{a:T,b:T & n128i{T}} = emit{T, ntyp{'vminq', T}, a, b} # TODO float - there are multiple options
def max{a:T,b:T & n128i{T}} = emit{T, ntyp{'vmaxq', T}, a, b}
def __shl{a:T,b:S & n128i{T} & n128i{S} & width{eltype{T}}==width{eltype{S}}} = emit{T, ntyp{'vshlq', T}, a, ty_s{b}}
def __shl{a:T,b:S & n128i{T} & n128i{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshlq', T}, a, ty_s{b}}
def __shl{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshlq_n', T}, a, b}; def __shl{a:T,b==0 & n128i{T}} = a
def __shr{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshrq_n', T}, a, b}; def __shr{a:T,b==0 & n128i{T}} = a
def bblend{f:T, t:T, m:M & n128{T} & n128u{M} & width{eltype{T}}==width{eltype{M}}} = emit{T, ntyp{'vbslq', T}, m, t, f}
def bblend{f:T, t:T, m:M & n128{T} & n128u{M} & elwidth{T}==elwidth{M}} = emit{T, ntyp{'vbslq', T}, m, t, f}
def __neg{a:T & (n128s{T}|n128f{T})} = emit{T, ntyp{'vnegq', T}, a}
def __not{a:T & n128u{T}} = emit{T, ntyp{'vmvnq', T}, a}

View File

@ -11,7 +11,7 @@ def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}}
# Fill last 4 bytes with last element, in each lane
def spread{a:VT} = {
def w = width{eltype{VT}}
def w = elwidth{VT}
def b = w/8
if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}}; else a
}

View File

@ -10,15 +10,15 @@ include 'util/tup'
oper &~ andnot infix none 35
# def:T - masked original content
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
# b:B - pointer to data to index; if width{B}<elwidth{T}, padding bytes are garbage read after wanted position
# idx - actual (unscaled) index list
def gather{def:T, b:B, idx:[8]i32, M & w256{T,32}} = {
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', def, *void~~b, idx, M{T,'to sign bits'}, width{eltype{B}}/8}
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, width{eltype{B}}/8}
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, elwidth{B}/8}
}
def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', def, *void~~b, idx, M{T,'to sign bits'}, width{eltype{B}}/8}
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, width{eltype{B}}/8}
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, elwidth{B}/8}
}
def wrapChk{cw0, VI,xlf, M} = {

View File

@ -14,7 +14,7 @@ def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = {
}
def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = {
def TS = ty_s{TU}
def h = 1 << (width{eltype{TU}}-1)
def h = 1 << (elwidth{TU}-1)
(TS~~(x-TU**(start-h))) < TS**(count-h)
}
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1}
@ -32,7 +32,7 @@ def fold{F, x:T} = {
}
def fold{F, x:T & w128{T}} = {
c:= x
def EW = width{eltype{T}}
def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}

View File

@ -5,12 +5,12 @@ local def wgen128{F} = {
def r{T} = 0
def r{T & w128{T}} = F{eltype{T}}
def r{T,w} = 0
def r{T,w & w128{T}} = F{eltype{T}} & (width{eltype{T}}==w)
def r{T,w & w128{T}} = F{eltype{T}} & (elwidth{T}==w)
def r{T & ~isvec{T}} = 0
r
}
def w128{T,w} = 0
def w128{T,w & w128{T}} = width{eltype{T}}==w
def w128{T,w & w128{T}} = elwidth{T}==w
def w128i = wgen128{{T} => isint{T}}
def w128s = wgen128{{T} => isint{T} & issigned{T}}
def w128u = wgen128{{T} => isint{T} & isunsigned{T}}
@ -159,7 +159,7 @@ def blend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv
def blend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def blendF{f:T, t:T, m:M & w128{T} & w128{M} & width{eltype{M}}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w128{T} & w128{M,16}} = blend{f, t, [16]i8,m}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}