diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 521e39fb..6501d4aa 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -100,12 +100,12 @@ def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} # structural operations -def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, [8]i32~~v} -def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, [4]f64~~v} +def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v} +def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v} # maskstore with all cases defined, at the cost of not being a single instruction -def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}} -def maskstoreF{p, m, n, x:T & elwidth{T}>=32} = maskstore{p,m,n,x} +def maskstoreF{p, m, x:T} = store{p, 0, blendF{load{p}, x, m}} +def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x} def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} diff --git a/src/singeli/src/constrep.singeli b/src/singeli/src/constrep.singeli index 1b68a430..5364fc0c 100644 --- a/src/singeli/src/constrep.singeli +++ b/src/singeli/src/constrep.singeli @@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { }} setlabel{end} q := nr & (step-1) - if (q) maskstoreF{rv, maskOf{V, q}, e, s} + if (q) maskstoreF{rv+e, maskOf{V, q}, s} } } def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} @@ -122,7 +122,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { } setlabel{end} q := (re+step) - r - if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} + if (q) maskstoreF{*V~~r, maskOf{V, q}, s} } rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index ea7ea755..c9e2c4ea 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -3,15 +3,15 @@ local def maskInit1{w} = { merge{(w/8-1)**255, (1<>3)^31 + 64*(n&7))} -mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & w128{T}} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} +mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))} +mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} mask256:*i64 = merge{4 ** -1, 4 ** 0} local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} # get mask of first n items; 0 ≤ n ≤ vcount{T} -def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256} -def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128} +def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256} +def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128} def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y} def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}} @@ -20,8 +20,8 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M} -def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T }} = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0 -def anynePositive{x:T, y:T, M & M{0}==1 & w256{T,16}} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 +def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0 +def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 def maskNone{x} = x def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x} @@ -46,7 +46,7 @@ def storeBatch{ptr:P, n, x:T, M} = { def E0 = eltype{P} xu:= ucvt{E0, x} def TF = to_el{E0, T} - if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, 0, xu} + if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, xu} else storeLow{rpos, vcount{T}*width{E0}, xu} } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index aa5de496..7b305297 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = { e:= len/step @for (xv, rv over e) rv = scan{xv,p} q:= len & (step-1) - if (q) maskstoreF{rv, maskOf{V, q}, e, scan_last{load{xv,e}, p}} + if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}} } def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { def last{v, p} = op{pre{v}, p} @@ -141,7 +141,7 @@ avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = { if (jv+vl <= l) { store{p, j, v} } else { - if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v} + if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v} return{} } }