neg(Any|All) → top(Any|All), msk(Any|All) → hom(Any|All)
This commit is contained in:
parent
336da996b3
commit
9fea5f1e13
@ -121,8 +121,8 @@ def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
||||
def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
||||
def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
||||
|
||||
def mskAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} != 0
|
||||
def mskAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff
|
||||
def homAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} != 0
|
||||
def homAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff
|
||||
|
||||
def negAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0
|
||||
def negAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<<vcount{T})-1
|
||||
def topAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0
|
||||
def topAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<<vcount{T})-1
|
||||
|
||||
@ -139,13 +139,13 @@ def getmask{x:T & w256{T, 16}} = {
|
||||
(msk&255) | (msk>>8)
|
||||
}
|
||||
|
||||
def mskAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0
|
||||
def mskAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
def homAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0
|
||||
def homAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
|
||||
def negAny{x:T & w256i{T}} = getmask{x} != 0
|
||||
def negAll{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def negAny{x:T & w256i{T,16}} = mskAny{[16]i16~~x < [16]i16**0}
|
||||
def negAll{x:T & w256i{T,16}} = mskAll{[16]i16~~x < [16]i16**0}
|
||||
def topAny{x:T & w256i{T}} = getmask{x} != 0
|
||||
def topAll{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def topAny{x:T & w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0}
|
||||
def topAll{x:T & w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}
|
||||
|
||||
|
||||
# conversion
|
||||
|
||||
@ -69,8 +69,8 @@ def ty_s{T & isprim{T}} = ty_s{width{T}}
|
||||
|
||||
def ty_u{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
|
||||
def ty_s{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def ty_u{x:T & isvec{T}} = ty_u{T}~~x
|
||||
def ty_s{x:T & isvec{T}} = ty_s{T}~~x
|
||||
def ty_u{x:T} = ty_u{T}~~x
|
||||
def ty_s{x:T} = ty_s{T}~~x
|
||||
|
||||
# reinterpret vector as one with element type E
|
||||
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
|
||||
|
||||
@ -22,8 +22,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
|
||||
|
||||
|
||||
# + & -
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = negAny{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = negAny{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = topAny{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = topAny{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
|
||||
|
||||
@ -41,9 +41,9 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
|
||||
def rp = each{__mul, wp, xp}
|
||||
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp}
|
||||
if (M{0}) { # masked check
|
||||
tup{packQ{rp}, mskAny{M{packQ{bad}}}}
|
||||
tup{packQ{rp}, homAny{M{packQ{bad}}}}
|
||||
} else { # unmasked check; can do check in a simpler way
|
||||
tup{packQ{rp}, mskAny{tupsel{0,bad}|tupsel{1,bad}}}
|
||||
tup{packQ{rp}, homAny{tupsel{0,bad}|tupsel{1,bad}}}
|
||||
}
|
||||
}
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
|
||||
@ -55,7 +55,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
max:= [8]f32 ~~ [8]u32**0x4efffffe
|
||||
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
tup{w*x, mskAny{M{abs{f32mul} >= max}}}
|
||||
tup{w*x, homAny{M{abs{f32mul} >= max}}}
|
||||
# TODO fallback to the below if the above fails
|
||||
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half
|
||||
# def wp = unpackQ{w, T**0}
|
||||
@ -65,7 +65,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
# def bad = each{{v} => {
|
||||
# ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}
|
||||
# }, rp}
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, mskAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, homAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
}
|
||||
|
||||
|
||||
@ -95,7 +95,7 @@ def runner{u, R, F} = {
|
||||
|
||||
def run{F, OO, M, w:VW, x:VX & c & R==u32 & (match{F,__add} | match{F,__sub})} = { # 'a'+1, 'a'-1
|
||||
r:= F{ty_u{w}, ty_u{x}}
|
||||
if (mskAny{M{r > type{r}**1114111}}) OO{}
|
||||
if (homAny{M{r > type{r}**1114111}}) OO{}
|
||||
to_el{R, VW}~~r
|
||||
}
|
||||
run
|
||||
|
||||
@ -13,8 +13,8 @@ local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth
|
||||
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
|
||||
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~mskAll{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = mskAny{M{x!=y}}
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
|
||||
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
|
||||
def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
|
||||
@ -70,10 +70,10 @@ def fold_max {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
|
||||
def bitAny{x:T} = fold_max{to_el{u32, x}}!=0
|
||||
def bitAll{x:T} = fold_min{to_el{u32, x}}==0xffff_ffff
|
||||
|
||||
def negAny{x:T & nvec{T}} = fold_min{ty_s{x}}<0
|
||||
def negAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0
|
||||
def mskAny{x:T & nvec{T}} = bitAny{x}
|
||||
def mskAll{x:T & nvec{T}} = bitAll{x}
|
||||
def topAny{x:T & nvec{T}} = fold_min{ty_s{x}}<0
|
||||
def topAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0
|
||||
def homAny{x:T & nvec{T}} = bitAny{x}
|
||||
def homAll{x:T & nvec{T}} = bitAll{x}
|
||||
|
||||
def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x}
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
||||
|
||||
def wrapChk{cw0, VI,xlf, M} = {
|
||||
cw:= cw0 + (xlf & VI~~(cw0<VI**0))
|
||||
if (mskAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
if (homAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
cw
|
||||
}
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
c:= EV~~loadBatch{xp, i, XV}
|
||||
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
|
||||
if (B) {
|
||||
if (mskAny{M{~q_chr{c}}}) return{3}
|
||||
if (homAny{M{~q_chr{c}}}) return{3}
|
||||
} else {
|
||||
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
|
||||
}
|
||||
@ -89,7 +89,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
maskedLoop{bulk, iCont, len, {i, M} => {
|
||||
def XU = [bulk]u64
|
||||
v:= XU ~~ loadBatch{xp, i, XV}
|
||||
if (mskAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number
|
||||
if (homAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number
|
||||
}}
|
||||
return{0xffff_ffff} # float
|
||||
}}
|
||||
|
||||
@ -115,13 +115,13 @@ def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u
|
||||
def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
||||
def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
||||
|
||||
def mskAny{x:T & w128i{T}} = getmask{[16]u8 ~~ x} != 0
|
||||
def mskAll{x:T & w128i{T}} = getmask{[16]u8 ~~ x} == 0xffff
|
||||
def homAny{x:T & w128i{T}} = getmask{[16]u8 ~~ x} != 0
|
||||
def homAll{x:T & w128i{T}} = getmask{[16]u8 ~~ x} == 0xffff
|
||||
|
||||
def negAny{x:T & w128i{T}} = getmask{x} != 0
|
||||
def negAll{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def negAny{x:T & w128i{T, 16}} = mskAny{[8]i16~~x < [8]i16**0}
|
||||
def negAll{x:T & w128i{T, 16}} = mskAll{[8]i16~~x < [8]i16**0}
|
||||
def topAny{x:T & w128i{T}} = getmask{x} != 0
|
||||
def topAll{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
|
||||
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
|
||||
|
||||
|
||||
# the lone SSE2 extract
|
||||
|
||||
Loading…
Reference in New Issue
Block a user