any/all vector check renaming
This commit is contained in:
parent
7e13bf7f45
commit
336da996b3
@ -116,9 +116,13 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_ble
|
||||
def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
|
||||
# mask stuff
|
||||
def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
||||
|
||||
def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
||||
def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
||||
def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
||||
def any{x:T & w256i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits)
|
||||
def all{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1 # same assumption
|
||||
def anyneg{x:T & w256s{T}} = getmask{x}!=0
|
||||
|
||||
def mskAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} != 0
|
||||
def mskAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff
|
||||
|
||||
def negAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0
|
||||
def negAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<<vcount{T})-1
|
||||
|
||||
@ -138,9 +138,14 @@ def getmask{x:T & w256{T, 16}} = {
|
||||
msk:u32 = getmask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
|
||||
(msk&255) | (msk>>8)
|
||||
}
|
||||
def any{x:T & w256i{T, 16}} = getmask{[32]u8~~x}!=0
|
||||
def all{x:T & w256i{T, 16}} = getmask{[32]u8~~x} == (1<<32)-1
|
||||
def anyneg{x:T & w256s{T, 16}} = getmask{[32]u8 ~~ ([16]i16~~x < [16]i16**0)}!=0
|
||||
|
||||
def mskAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0
|
||||
def mskAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff
|
||||
|
||||
def negAny{x:T & w256i{T}} = getmask{x} != 0
|
||||
def negAll{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def negAny{x:T & w256i{T,16}} = mskAny{[16]i16~~x < [16]i16**0}
|
||||
def negAll{x:T & w256i{T,16}} = mskAll{[16]i16~~x < [16]i16**0}
|
||||
|
||||
|
||||
# conversion
|
||||
|
||||
@ -136,8 +136,6 @@ def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
|
||||
|
||||
# tuple operations
|
||||
def broadcast{T, v & isprim{T}} = v
|
||||
def any{v:T & isprim{T}} = v
|
||||
def anyneg{v:T & isprim{T}} = v<0
|
||||
def iota{n & knum{n}} = @collect(i to n) i
|
||||
def broadcast{n, v & knum{n}} = @collect(n) v
|
||||
|
||||
|
||||
@ -22,8 +22,8 @@ def bqn_or{a, b} = (a+b)-(a*b)
|
||||
|
||||
|
||||
# + & -
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = anyneg{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = anyneg{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T} = negAny{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T} = negAny{M{(w^x) & (w^r)}}
|
||||
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M}
|
||||
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M}
|
||||
|
||||
@ -41,9 +41,9 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
|
||||
def rp = each{__mul, wp, xp}
|
||||
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp}
|
||||
if (M{0}) { # masked check
|
||||
tup{packQ{rp}, any{M{packQ{bad}}}}
|
||||
tup{packQ{rp}, mskAny{M{packQ{bad}}}}
|
||||
} else { # unmasked check; can do check in a simpler way
|
||||
tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}}
|
||||
tup{packQ{rp}, mskAny{tupsel{0,bad}|tupsel{1,bad}}}
|
||||
}
|
||||
}
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
|
||||
@ -55,7 +55,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
max:= [8]f32 ~~ [8]u32**0x4efffffe
|
||||
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
tup{w*x, any{M{abs{f32mul} >= max}}}
|
||||
tup{w*x, mskAny{M{abs{f32mul} >= max}}}
|
||||
# TODO fallback to the below if the above fails
|
||||
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half
|
||||
# def wp = unpackQ{w, T**0}
|
||||
@ -65,7 +65,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
|
||||
# def bad = each{{v} => {
|
||||
# ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}
|
||||
# }, rp}
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, mskAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
}
|
||||
|
||||
|
||||
@ -95,7 +95,7 @@ def runner{u, R, F} = {
|
||||
|
||||
def run{F, OO, M, w:VW, x:VX & c & R==u32 & (match{F,__add} | match{F,__sub})} = { # 'a'+1, 'a'-1
|
||||
r:= F{ty_u{w}, ty_u{x}}
|
||||
if (any{M{r > type{r}**1114111}}) OO{}
|
||||
if (mskAny{M{r > type{r}**1114111}}) OO{}
|
||||
to_el{R, VW}~~r
|
||||
}
|
||||
run
|
||||
|
||||
@ -13,8 +13,8 @@ local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth
|
||||
def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128}
|
||||
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}}
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~mskAll{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = mskAny{M{x!=y}}
|
||||
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
|
||||
def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
|
||||
@ -67,6 +67,14 @@ def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a}
|
||||
def fold_min {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vminv', T}, a}
|
||||
def fold_max {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
|
||||
|
||||
def bitAny{x:T} = fold_max{to_el{u32, x}}!=0
|
||||
def bitAll{x:T} = fold_min{to_el{u32, x}}==0xffff_ffff
|
||||
|
||||
def negAny{x:T & nvec{T}} = fold_min{ty_s{x}}<0
|
||||
def negAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0
|
||||
def mskAny{x:T & nvec{T}} = bitAny{x}
|
||||
def mskAll{x:T & nvec{T}} = bitAll{x}
|
||||
|
||||
def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x}
|
||||
|
||||
def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = {
|
||||
|
||||
@ -23,7 +23,7 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
||||
|
||||
def wrapChk{cw0, VI,xlf, M} = {
|
||||
cw:= cw0 + (xlf & VI~~(cw0<VI**0))
|
||||
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
if (mskAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
cw
|
||||
}
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
c:= EV~~loadBatch{xp, i, XV}
|
||||
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
|
||||
if (B) {
|
||||
if (any{M{~q_chr{c}}}) return{3}
|
||||
if (mskAny{M{~q_chr{c}}}) return{3}
|
||||
} else {
|
||||
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
|
||||
}
|
||||
@ -89,7 +89,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
maskedLoop{bulk, iCont, len, {i, M} => {
|
||||
def XU = [bulk]u64
|
||||
v:= XU ~~ loadBatch{xp, i, XV}
|
||||
if (any{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number
|
||||
if (mskAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number
|
||||
}}
|
||||
return{0xffff_ffff} # float
|
||||
}}
|
||||
|
||||
@ -108,16 +108,21 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
|
||||
|
||||
|
||||
# mask stuff
|
||||
def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
||||
def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}
|
||||
def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
||||
def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
||||
def any{x:T & w128i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits)
|
||||
def all{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1 # same assumption
|
||||
def anyneg{x:T & w128s{T}} = getmask{x}!=0
|
||||
def any{x:T & w128i{T, 16}} = getmask{[32]u8 ~~ x}!=0
|
||||
def all{x:T & w128i{T, 16}} = getmask{[32]u8 ~~ x} == (1<<32)-1
|
||||
def anyneg{x:T & w128s{T, 16}} = getmask{[32]u8 ~~ (([16]i16~~x) < [16]i16**0)}!=0
|
||||
|
||||
def mskAny{x:T & w128i{T}} = getmask{[16]u8 ~~ x} != 0
|
||||
def mskAll{x:T & w128i{T}} = getmask{[16]u8 ~~ x} == 0xffff
|
||||
|
||||
def negAny{x:T & w128i{T}} = getmask{x} != 0
|
||||
def negAll{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1
|
||||
def negAny{x:T & w128i{T, 16}} = mskAny{[8]i16~~x < [8]i16**0}
|
||||
def negAll{x:T & w128i{T, 16}} = mskAll{[8]i16~~x < [8]i16**0}
|
||||
|
||||
|
||||
# the lone SSE2 extract
|
||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||
@ -141,9 +146,6 @@ def cvt{F==i8, T==[8]i16, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi16', a}
|
||||
def cvt{F==i8, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi32', a}
|
||||
def cvt{F==i16, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi16_epi32', a}
|
||||
|
||||
# mask stuff
|
||||
def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
# blend by sign bit; no 16-bit case
|
||||
def blend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def blend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user