From 336da996b3e506587b480bec3c782f47452175cb Mon Sep 17 00:00:00 2001 From: dzaima Date: Tue, 13 Dec 2022 02:03:11 +0200 Subject: [PATCH] any/all vector check renaming --- src/singeli/src/avx.singeli | 12 ++++++++---- src/singeli/src/avx2.singeli | 11 ++++++++--- src/singeli/src/base.singeli | 2 -- src/singeli/src/dyarith.singeli | 14 +++++++------- src/singeli/src/mask.singeli | 4 ++-- src/singeli/src/neon.singeli | 8 ++++++++ src/singeli/src/select.singeli | 2 +- src/singeli/src/squeeze.singeli | 4 ++-- src/singeli/src/sse3.singeli | 20 +++++++++++--------- 9 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 08967f80..dcd735d3 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -116,9 +116,13 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_ble def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} # mask stuff +def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} + def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} -def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} -def any{x:T & w256i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits) -def all{x:T & w256i{T}} = getmask{x} == (1<=32} = getmask{[8]u32 ~~ x} != 0 +def mskAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff + +def negAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0 +def negAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<>8) } -def any{x:T & w256i{T, 16}} = getmask{[32]u8~~x}!=0 -def all{x:T & w256i{T, 16}} = getmask{[32]u8~~x} == (1<<32)-1 -def anyneg{x:T & w256s{T, 16}} = getmask{[32]u8 ~~ ([16]i16~~x < [16]i16**0)}!=0 + +def mskAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0 +def mskAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff + +def negAny{x:T & w256i{T}} = getmask{x} != 0 +def negAll{x:T & w256i{T}} = getmask{x} == (1<b, a, b} # tuple operations def broadcast{T, v & isprim{T}} = v -def any{v:T & isprim{T}} = v -def anyneg{v:T & isprim{T}} = v<0 def iota{n & knum{n}} = @collect(i to n) i def broadcast{n, v & knum{n}} = @collect(n) v diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index cb76b537..e92900df 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -22,8 +22,8 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - -def arithChk1{F==__add, M, w:T, x:T, r:T} = anyneg{M{(w^r) & (x^r)}} -def arithChk1{F==__sub, M, w:T, x:T, r:T} = anyneg{M{(w^x) & (w^r)}} +def arithChk1{F==__add, M, w:T, x:T, r:T} = negAny{M{(w^r) & (x^r)}} +def arithChk1{F==__sub, M, w:T, x:T, r:T} = negAny{M{(w^x) & (w^r)}} def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__adds{w,x}, r, M} def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & elwidth{T}<=16} = anyne{__subs{w,x}, r, M} @@ -41,9 +41,9 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = { def rp = each{__mul, wp, xp} def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp} if (M{0}) { # masked check - tup{packQ{rp}, any{M{packQ{bad}}}} + tup{packQ{rp}, mskAny{M{packQ{bad}}}} } else { # unmasked check; can do check in a simpler way - tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}} + tup{packQ{rp}, mskAny{tupsel{0,bad}|tupsel{1,bad}}} } } def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = { @@ -55,7 +55,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { max:= [8]f32 ~~ [8]u32**0x4efffffe def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} f32mul:= cf32{w} * cf32{x} - tup{w*x, any{M{abs{f32mul} >= max}}} + tup{w*x, mskAny{M{abs{f32mul} >= max}}} # TODO fallback to the below if the above fails # TODO don't do this, but instead shuffle one half, do math, unshuffle that half # def wp = unpackQ{w, T**0} @@ -65,7 +65,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { # def bad = each{{v} => { # ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF} # }, rp} - # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M + # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, mskAny{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M } @@ -95,7 +95,7 @@ def runner{u, R, F} = { def run{F, OO, M, w:VW, x:VX & c & R==u32 & (match{F,__add} | match{F,__sub})} = { # 'a'+1, 'a'-1 r:= F{ty_u{w}, ty_u{x}} - if (any{M{r > type{r}**1114111}}) OO{} + if (mskAny{M{r > type{r}**1114111}}) OO{} to_el{R, VW}~~r } run diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 2fb88110..9dd487f2 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -13,8 +13,8 @@ local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth def maskOf{T,n & w256{T}} = maskOfImpl{T, n, 256} def maskOf{T,n & w128{T}} = maskOfImpl{T, n, 128} -def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y} -def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}} +def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~mskAll{x==y} +def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = mskAny{M{x!=y}} def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index c5fe650f..b658b77f 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -67,6 +67,14 @@ def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} def fold_min {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vminv', T}, a} def fold_max {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vmaxv', T}, a} +def bitAny{x:T} = fold_max{to_el{u32, x}}!=0 +def bitAll{x:T} = fold_min{to_el{u32, x}}==0xffff_ffff + +def negAny{x:T & nvec{T}} = fold_min{ty_s{x}}<0 +def negAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0 +def mskAny{x:T & nvec{T}} = bitAny{x} +def mskAll{x:T & nvec{T}} = bitAll{x} + def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x} def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = { diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 429ec97d..a83a90ea 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -23,7 +23,7 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = { def wrapChk{cw0, VI,xlf, M} = { cw:= cw0 + (xlf & VI~~(cw0= ty_u{xlf}}}) return{0} + if (mskAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0} cw } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 20937698..9c714b38 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -67,7 +67,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { c:= EV~~loadBatch{xp, i, XV} if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end if (B) { - if (any{M{~q_chr{c}}}) return{3} + if (mskAny{M{~q_chr{c}}}) return{3} } else { if (anynePositive{EV**((1< { def XU = [bulk]u64 v:= XU ~~ loadBatch{xp, i, XV} - if (any{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number + if (mskAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number }} return{0xffff_ffff} # float }} diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 1a95bcf1..f74ab71e 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -108,16 +108,21 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} # mask stuff +def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} + def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}} def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} -def any{x:T & w128i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits) -def all{x:T & w128i{T}} = getmask{x} == (1<