remove offset argument from maskstore; don't use w256 in mask.singeli

This commit is contained in:
dzaima 2022-12-15 15:48:16 +02:00
parent ddf1dca0b4
commit 6f2d6bfb0b
4 changed files with 15 additions and 15 deletions

View File

@ -100,12 +100,12 @@ def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
# structural operations
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, [8]i32~~v}
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, [4]f64~~v}
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
# maskstore with all cases defined, at the cost of not being a single instruction
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x}
def maskstoreF{p, m, x:T} = store{p, 0, blendF{load{p}, x, m}}
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}

View File

@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
}}
setlabel{end}
q := nr & (step-1)
if (q) maskstoreF{rv, maskOf{V, q}, e, s}
if (q) maskstoreF{rv+e, maskOf{V, q}, s}
}
}
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
@ -122,7 +122,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
}
setlabel{end}
q := (re+step) - r
if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s}
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
}
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {

View File

@ -3,15 +3,15 @@ local def maskInit1{w} = {
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
}, iota{8}}}
}
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
mask256:*i64 = merge{4 ** -1, 4 ** 0}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
# get mask of first n items; 0 ≤ n ≤ vcount{T}
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
@ -20,8 +20,8 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T }} = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & w256{T,16}} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def maskNone{x} = x
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
@ -46,7 +46,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
def E0 = eltype{P}
xu:= ucvt{E0, x}
def TF = to_el{E0, T}
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, 0, xu}
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, xu}
else storeLow{rpos, vcount{T}*width{E0}, xu}
}

View File

@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
e:= len/step
@for (xv, rv over e) rv = scan{xv,p}
q:= len & (step-1)
if (q) maskstoreF{rv, maskOf{V, q}, e, scan_last{load{xv,e}, p}}
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
}
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
def last{v, p} = op{pre{v}, p}
@ -141,7 +141,7 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
if (jv+vl <= l) {
store{p, j, v}
} else {
if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v}
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v}
return{}
}
}