more general equal.singeli, remove trailing semicolons
This commit is contained in:
parent
2a32f52f8d
commit
a07b94b889
@ -95,6 +95,7 @@ def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def andnz{x, y} = assert{'andnz not supported', show{x, y}}
|
||||
def topBlend{f,t,m} = assert{'topBlend not supported', show{f,t,m}}
|
||||
|
||||
|
||||
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
|
||||
|
||||
@ -4,10 +4,15 @@ def b_get{x:*u64, n:(Size)} = {
|
||||
((load{x,n>>6}>>(n&63)) & 1) != 0
|
||||
}
|
||||
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) & 3
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15
|
||||
def b_getBatchLo{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
|
||||
def b_getBatchLo{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
|
||||
def b_getBatchLo{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
|
||||
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = b_getBatchLo{sz, x, n} & 3
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = b_getBatchLo{sz, x, n} & 15
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
|
||||
|
||||
|
||||
def b_set{x:*u64, n:(Size), v:u1} = {
|
||||
m:u64 = cast{u64,1}<<(n&63)
|
||||
p:u64 = load{x,n>>6}
|
||||
@ -17,7 +22,7 @@ def b_set{x:*u64, n:(Size), v:u1} = {
|
||||
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v} = {
|
||||
vc:u64 = promote{u64,v}
|
||||
am:u64 = 64/sz;
|
||||
am:u64 = 64/sz
|
||||
w:u64 = load{x,n/am}
|
||||
sh:u64 = (n&(am-1)) * sz
|
||||
w&= ~(ones{u64,sz}<<sh)
|
||||
@ -69,5 +74,5 @@ def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
|
||||
}
|
||||
|
||||
def loadBatchBit{T, x:*u64, n:(Size)} = { # vector with type T with each element being either all 0s or 1s
|
||||
spreadBits{T, b_getBatch{vcount{T}, x, n}}
|
||||
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
|
||||
}
|
||||
|
||||
@ -45,7 +45,7 @@ def pathAS{dst, len, T, op, x & issigned{T}} = {
|
||||
|
||||
def pathAS{dst, len, T, op, x & T==f64} = {
|
||||
if (rare{~q_f64{x}}) {
|
||||
if (~eqne{op}) if (~q_chr{x}) cmp_err{x};
|
||||
if (~eqne{op}) if (~q_chr{x}) cmp_err{x}
|
||||
fillbits{dst, len, op{0,1}, x}
|
||||
}
|
||||
from_B{T,x}
|
||||
@ -53,7 +53,7 @@ def pathAS{dst, len, T, op, x & T==f64} = {
|
||||
|
||||
def pathAS{dst, len, T, op, x & isunsigned{T}} = {
|
||||
if (rare{~q_chr{x}}) {
|
||||
if (~eqne{op}) if (~q_f64{x}) cmp_err{x};
|
||||
if (~eqne{op}) if (~q_f64{x}) cmp_err{x}
|
||||
fillbits{dst, len, op{1,0}, x}
|
||||
}
|
||||
xc32:u32 = from_B{u32,x}
|
||||
|
||||
@ -105,7 +105,7 @@ def runner{u, R, F} = {
|
||||
def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
|
||||
# show{F, mode, W, X, R}
|
||||
if (R==u1) {
|
||||
def bulk = vw/64;
|
||||
def bulk = vw/64
|
||||
def TY = [bulk]u64
|
||||
maskedLoop{bulk, cdiv{len, 64}, {i, M} => {
|
||||
cw:= loadBatch{*u64~~w, i, TY}
|
||||
|
||||
@ -5,6 +5,7 @@ include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
include './bitops'
|
||||
|
||||
def swap{w,x} = {
|
||||
t:= w
|
||||
@ -28,27 +29,33 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
} else if (X==f64) { # bitarr ≡ f64arr
|
||||
def TF = [vw/64]f64
|
||||
def TU = [vw/64]u64
|
||||
def bulk = 4
|
||||
f0:= TF**0.0
|
||||
f1:= TF**1.0
|
||||
maskedLoopPositive{bulk, l, {i, M} => {
|
||||
cw:= load{*u8 ~~ w, i>>1} >> cast_i{u8, 4*(i&1)}
|
||||
cx:= load{*TF ~~ x, i}
|
||||
wu:= topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}}
|
||||
wu:= (if (hasarch{'X86_64'}) {
|
||||
cw:= b_getBatchLo{bulk, *u64~~w, i}
|
||||
topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}}
|
||||
} else {
|
||||
cw:= loadBatchBit{TU, *u64~~w, i}
|
||||
homBlend{f0, f1, cw}
|
||||
})
|
||||
cx:= load{*TF ~~ x, i}
|
||||
if (anynePositive{wu, cx, M}) return{0}
|
||||
}}
|
||||
} else { # bitarr ≡ i8/i16/i32arr
|
||||
def T = [vw/width{X}]X
|
||||
def T = [bulk]X
|
||||
def sh{c} = c << (width{X}-1)
|
||||
def sh{c & X==u8} = T ~~ (to_el{u16,c}<<7)
|
||||
def topMask{x:X & hasarch{'X86_64'}} = getmask{x}
|
||||
def topMask{x:X & hasarch{'AARCH64'}} = getmask{andnz{x, ~T**0}}
|
||||
|
||||
# TODO compare with doing the comparison in vector registers
|
||||
badBits:= T ** ~cast{X,1}
|
||||
badBits:= T ** ~(X~~1)
|
||||
maskedLoop{bulk, l, {i, M} => {
|
||||
cw:= load{*ty_u{bulk} ~~ w, i}
|
||||
cw:= b_getBatch{bulk, *u64~~w, i}
|
||||
cx:= load{*T ~~ x, i}
|
||||
if (~andAllZero{M{cx}, badBits}) return{0}
|
||||
if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0}
|
||||
if (anyne{promote{u64,topMask{sh{cx}}}, promote{u64,cw}, M}) return{0}
|
||||
}}
|
||||
1
|
||||
}
|
||||
|
||||
@ -77,8 +77,8 @@ def storeLow{ptr:P, w, x:T & nvec{T} & w<=64} = { def E=ty_u{w}; storeu{*E~~ptr,
|
||||
def storeLow{ptr:P, w, x:T & nvec{T} & w==width{T}} = store{*T~~ptr, 0, x}
|
||||
|
||||
def loadLow{ptr:P, w & w<=64} = { # a broadcast load
|
||||
def T=eltype{P};
|
||||
def L=to_el{ty_u{w}, T};
|
||||
def T=eltype{P}
|
||||
def L=to_el{ty_u{w}, T}
|
||||
T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
|
||||
}
|
||||
def loadLow{ptr:P, w & w==elwidth{P}} = load{ptr}
|
||||
@ -98,8 +98,11 @@ def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, em
|
||||
|
||||
local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers
|
||||
|
||||
def cvt{T, x:X & width{T}==width{X} & nvecf{T,64} & nveci{X}} = emit{T, ntyp{'vcvt', '_f64', X}, x}
|
||||
|
||||
def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x}
|
||||
def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}> elwidth{X}*2} = widen{T, widen{ty_half{T}, x}}
|
||||
def widen{T, x:X & n64{X} & isfloat{eltype{T}}!=isfloat{eltype{X}} & elwidth{T}>elwidth{X}} = cvt{T, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}}
|
||||
def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
|
||||
|
||||
def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}}
|
||||
|
||||
@ -19,7 +19,7 @@ clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||
}
|
||||
xv := *V ~~ x
|
||||
rv := *V ~~ r
|
||||
e := words/2;
|
||||
e := words/2
|
||||
c := V**init
|
||||
@for (rv, xv over e) {
|
||||
rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})}
|
||||
|
||||
@ -31,7 +31,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||
def last{v, p} = op{pre{v}, p}
|
||||
def scan{v, p} = {
|
||||
n:= last{v, p}
|
||||
p = (if (width{T}<=32) sel{[8]i32, spread{n}, [8]i32**7};
|
||||
p = (if (width{T}<=32) sel{[8]i32, spread{n}, [8]i32**7}
|
||||
else shuf{[4]u64, n, 4b3333})
|
||||
n
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user