99 lines
3.9 KiB
Plaintext
99 lines
3.9 KiB
Plaintext
local def maskInit1{w} = {
|
|
apply{merge, each{{x} => {
|
|
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
|
|
}, iota{8}}}
|
|
}
|
|
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
|
|
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
|
|
|
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
|
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(width{eltype{T}}/8))}
|
|
|
|
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
|
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
|
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
|
|
|
|
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y}
|
|
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}}
|
|
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
|
|
def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
|
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
|
|
|
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
|
|
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T }} = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
|
|
def anynePositive{x:T, y:T, M & M{0}==1 & w256{T,16}} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
|
|
|
def maskNone{x} = x
|
|
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
|
|
def maskAfter{n} = {
|
|
def mask{x:X & isvec{X}} = x & (X~~maskOf{X,n})
|
|
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
|
|
def mask{x:X, mode=='all bits zeroes'} = andIsZero{x, X~~maskOfBit{X,n}}
|
|
def mask{X, mode=='to sign bits'} = maskOf{X,n}
|
|
def mask{mode=='count'} = n
|
|
def mask{x==0} = 1
|
|
}
|
|
|
|
|
|
|
|
# store low packed elements of x to P
|
|
def storeLow{ptr:P, w, x:T & width{T}==w} = store{*T~~ptr, 0, x}
|
|
|
|
def storeLow{ptr:P, w, x:T & w128{T} & w==16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
|
|
def storeLow{ptr:P, w, x:T & w128{T} & w==32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
|
|
def storeLow{ptr:P, w, x:T & w128{T} & w==64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
|
|
|
|
def storeLow{ptr:P, w, x:T & w256{T} & w==16} = storeLow{ptr, w, half{x, 0}}
|
|
def storeLow{ptr:P, w, x:T & w256{T} & w==32} = storeLow{ptr, w, half{x, 0}}
|
|
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = storeLow{ptr, w, half{x, 0}}
|
|
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/width{eltype{T}}](eltype{T})~~ptr, 0, half{x, 0}}
|
|
|
|
|
|
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = eltype{P} ~~ load{*[16]u8 ~~ ptr}
|
|
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
|
|
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
|
|
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
|
|
|
|
def loadLow{ptr:P, w & w256{eltype{P}} & w<256} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}}
|
|
def loadLow{ptr:P, w & width{eltype{P}} == w} = load{*eltype{P} ~~ ptr}
|
|
|
|
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/width{eltype{P}})), w}
|
|
|
|
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
|
|
def storeBatch{ptr:P, n, x:T, M} = {
|
|
def rpos = ptr + n*vcount{T}
|
|
def E0 = eltype{P}
|
|
xu:= ucvt{E0, x}
|
|
def TF = to_el{E0, T}
|
|
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, 0, xu}
|
|
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
|
}
|
|
|
|
# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T
|
|
def loadBatch{ptr:P, n, T} = {
|
|
def rpos = ptr + n*vcount{T}
|
|
def E0 = eltype{P}
|
|
|
|
cvt{E0, T, loadLow{*to_el{E0, T} ~~ rpos, vcount{T}*width{E0}}}
|
|
}
|
|
|
|
|
|
|
|
def maskedLoop{bulk, l, step} = maskedLoop{bulk, 0, l, step}
|
|
|
|
def maskedLoop{bulk, i0, l, step} = {
|
|
m:u64 = l/bulk
|
|
@for (i from i0 to m) step{i, maskNone}
|
|
|
|
left:= l & (bulk-1)
|
|
if (left!=0) step{m, maskAfter{left}}
|
|
}
|
|
|
|
def maskedLoopPositive{bulk, l:L, step} = {
|
|
i:L = 0
|
|
while(i < (l-1)/bulk) {
|
|
step{i, maskNone}
|
|
++i
|
|
}
|
|
step{i, maskAfter{l - i*bulk}}
|
|
} |