remove offset argument from maskstore; don't use w256 in mask.singeli
This commit is contained in:
parent
ddf1dca0b4
commit
6f2d6bfb0b
@ -100,12 +100,12 @@ def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
||||
|
||||
|
||||
# structural operations
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, [8]i32~~v}
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, [4]f64~~v}
|
||||
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
|
||||
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
|
||||
|
||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
||||
def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x}
|
||||
def maskstoreF{p, m, x:T} = store{p, 0, blendF{load{p}, x, m}}
|
||||
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
|
||||
@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
|
||||
}}
|
||||
setlabel{end}
|
||||
q := nr & (step-1)
|
||||
if (q) maskstoreF{rv, maskOf{V, q}, e, s}
|
||||
if (q) maskstoreF{rv+e, maskOf{V, q}, s}
|
||||
}
|
||||
}
|
||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
||||
@ -122,7 +122,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
}
|
||||
setlabel{end}
|
||||
q := (re+step) - r
|
||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s}
|
||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
|
||||
}
|
||||
|
||||
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
|
||||
@ -3,15 +3,15 @@ local def maskInit1{w} = {
|
||||
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
|
||||
}, iota{8}}}
|
||||
}
|
||||
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
|
||||
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
||||
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
|
||||
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
||||
|
||||
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
||||
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
|
||||
|
||||
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
||||
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
|
||||
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
|
||||
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
|
||||
@ -20,8 +20,8 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
|
||||
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T }} = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & w256{T,16}} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
||||
|
||||
def maskNone{x} = x
|
||||
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
|
||||
@ -46,7 +46,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
|
||||
def E0 = eltype{P}
|
||||
xu:= ucvt{E0, x}
|
||||
def TF = to_el{E0, T}
|
||||
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, 0, xu}
|
||||
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, xu}
|
||||
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
||||
}
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
|
||||
e:= len/step
|
||||
@for (xv, rv over e) rv = scan{xv,p}
|
||||
q:= len & (step-1)
|
||||
if (q) maskstoreF{rv, maskOf{V, q}, e, scan_last{load{xv,e}, p}}
|
||||
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
|
||||
}
|
||||
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||
def last{v, p} = op{pre{v}, p}
|
||||
@ -141,7 +141,7 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
if (jv+vl <= l) {
|
||||
store{p, j, v}
|
||||
} else {
|
||||
if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v}
|
||||
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v}
|
||||
return{}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user