remove offset argument from maskstore; don't use w256 in mask.singeli
This commit is contained in:
parent
ddf1dca0b4
commit
6f2d6bfb0b
@ -100,12 +100,12 @@ def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
|||||||
|
|
||||||
|
|
||||||
# structural operations
|
# structural operations
|
||||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, [8]i32~~v}
|
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
|
||||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, [4]f64~~v}
|
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
|
||||||
|
|
||||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
def maskstoreF{p, m, x:T} = store{p, 0, blendF{load{p}, x, m}}
|
||||||
def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x}
|
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
|
||||||
|
|
||||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||||
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||||
|
|||||||
@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
|
|||||||
}}
|
}}
|
||||||
setlabel{end}
|
setlabel{end}
|
||||||
q := nr & (step-1)
|
q := nr & (step-1)
|
||||||
if (q) maskstoreF{rv, maskOf{V, q}, e, s}
|
if (q) maskstoreF{rv+e, maskOf{V, q}, s}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
||||||
@ -122,7 +122,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
|||||||
}
|
}
|
||||||
setlabel{end}
|
setlabel{end}
|
||||||
q := (re+step) - r
|
q := (re+step) - r
|
||||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s}
|
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
|
||||||
}
|
}
|
||||||
|
|
||||||
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||||
|
|||||||
@ -3,15 +3,15 @@ local def maskInit1{w} = {
|
|||||||
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
|
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
|
||||||
}, iota{8}}}
|
}, iota{8}}}
|
||||||
}
|
}
|
||||||
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & w256{T}} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
|
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
|
||||||
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
|
||||||
|
|
||||||
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
||||||
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
|
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
|
||||||
|
|
||||||
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
||||||
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
|
||||||
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
|
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
|
||||||
|
|
||||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
|
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
|
||||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
|
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
|
||||||
@ -21,7 +21,7 @@ def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
|||||||
|
|
||||||
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
|
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
|
||||||
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
|
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
|
||||||
def anynePositive{x:T, y:T, M & M{0}==1 & w256{T,16}} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
||||||
|
|
||||||
def maskNone{x} = x
|
def maskNone{x} = x
|
||||||
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
|
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
|
||||||
@ -46,7 +46,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
|
|||||||
def E0 = eltype{P}
|
def E0 = eltype{P}
|
||||||
xu:= ucvt{E0, x}
|
xu:= ucvt{E0, x}
|
||||||
def TF = to_el{E0, T}
|
def TF = to_el{E0, T}
|
||||||
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, 0, xu}
|
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, xu}
|
||||||
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
|
|||||||
e:= len/step
|
e:= len/step
|
||||||
@for (xv, rv over e) rv = scan{xv,p}
|
@for (xv, rv over e) rv = scan{xv,p}
|
||||||
q:= len & (step-1)
|
q:= len & (step-1)
|
||||||
if (q) maskstoreF{rv, maskOf{V, q}, e, scan_last{load{xv,e}, p}}
|
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
|
||||||
}
|
}
|
||||||
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||||
def last{v, p} = op{pre{v}, p}
|
def last{v, p} = op{pre{v}, p}
|
||||||
@ -141,7 +141,7 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
|||||||
if (jv+vl <= l) {
|
if (jv+vl <= l) {
|
||||||
store{p, j, v}
|
store{p, j, v}
|
||||||
} else {
|
} else {
|
||||||
if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v}
|
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v}
|
||||||
return{}
|
return{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user