From a07b94b8893ad71b786c852d75c40a1bea1d60f1 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 18 Dec 2022 21:53:25 +0200 Subject: [PATCH] more general equal.singeli, remove trailing semicolons --- src/singeli/src/base.singeli | 1 + src/singeli/src/bitops.singeli | 13 +++++++++---- src/singeli/src/cmp.singeli | 4 ++-- src/singeli/src/dyarith.singeli | 2 +- src/singeli/src/equal.singeli | 23 +++++++++++++++-------- src/singeli/src/neon.singeli | 7 +++++-- src/singeli/src/neq.singeli | 2 +- src/singeli/src/scan.singeli | 2 +- 8 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index d1129191..0617c36f 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -95,6 +95,7 @@ def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x def undefPromote{T, x:X & T==X} = T~~x def andnz{x, y} = assert{'andnz not supported', show{x, y}} +def topBlend{f,t,m} = assert{'topBlend not supported', show{f,t,m}} def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index c7685e1d..9b0f9aea 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -4,10 +4,15 @@ def b_get{x:*u64, n:(Size)} = { ((load{x,n>>6}>>(n&63)) & 1) != 0 } -def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) & 3 -def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15 +def b_getBatchLo{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) +def b_getBatchLo{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) +def b_getBatchLo{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n} + +def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = b_getBatchLo{sz, x, n} & 3 +def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = b_getBatchLo{sz, x, n} & 15 def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n} + def b_set{x:*u64, n:(Size), v:u1} = { m:u64 = cast{u64,1}<<(n&63) p:u64 = load{x,n>>6} @@ -17,7 +22,7 @@ def b_set{x:*u64, n:(Size), v:u1} = { def b_setBatch{sz, x:*u64, n:(Size), v} = { vc:u64 = promote{u64,v} - am:u64 = 64/sz; + am:u64 = 64/sz w:u64 = load{x,n/am} sh:u64 = (n&(am-1)) * sz w&= ~(ones{u64,sz}< { cw:= loadBatch{*u64~~w, i, TY} diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 3e42cfc6..dd6c48e8 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -5,6 +5,7 @@ include './sse3' include './avx' include './avx2' include './mask' +include './bitops' def swap{w,x} = { t:= w @@ -28,27 +29,33 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { } else if (X==f64) { # bitarr ≡ f64arr def TF = [vw/64]f64 def TU = [vw/64]u64 - def bulk = 4 f0:= TF**0.0 f1:= TF**1.0 maskedLoopPositive{bulk, l, {i, M} => { - cw:= load{*u8 ~~ w, i>>1} >> cast_i{u8, 4*(i&1)} - cx:= load{*TF ~~ x, i} - wu:= topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}} + wu:= (if (hasarch{'X86_64'}) { + cw:= b_getBatchLo{bulk, *u64~~w, i} + topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}} + } else { + cw:= loadBatchBit{TU, *u64~~w, i} + homBlend{f0, f1, cw} + }) + cx:= load{*TF ~~ x, i} if (anynePositive{wu, cx, M}) return{0} }} } else { # bitarr ≡ i8/i16/i32arr - def T = [vw/width{X}]X + def T = [bulk]X def sh{c} = c << (width{X}-1) def sh{c & X==u8} = T ~~ (to_el{u16,c}<<7) + def topMask{x:X & hasarch{'X86_64'}} = getmask{x} + def topMask{x:X & hasarch{'AARCH64'}} = getmask{andnz{x, ~T**0}} # TODO compare with doing the comparison in vector registers - badBits:= T ** ~cast{X,1} + badBits:= T ** ~(X~~1) maskedLoop{bulk, l, {i, M} => { - cw:= load{*ty_u{bulk} ~~ w, i} + cw:= b_getBatch{bulk, *u64~~w, i} cx:= load{*T ~~ x, i} if (~andAllZero{M{cx}, badBits}) return{0} - if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0} + if (anyne{promote{u64,topMask{sh{cx}}}, promote{u64,cw}, M}) return{0} }} 1 } diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index f3080723..9de18b23 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -77,8 +77,8 @@ def storeLow{ptr:P, w, x:T & nvec{T} & w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, def storeLow{ptr:P, w, x:T & nvec{T} & w==width{T}} = store{*T~~ptr, 0, x} def loadLow{ptr:P, w & w<=64} = { # a broadcast load - def T=eltype{P}; - def L=to_el{ty_u{w}, T}; + def T=eltype{P} + def L=to_el{ty_u{w}, T} T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} } def loadLow{ptr:P, w & w==elwidth{P}} = load{ptr} @@ -98,8 +98,11 @@ def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, em local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers +def cvt{T, x:X & width{T}==width{X} & nvecf{T,64} & nveci{X}} = emit{T, ntyp{'vcvt', '_f64', X}, x} + def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x} def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}> elwidth{X}*2} = widen{T, widen{ty_half{T}, x}} +def widen{T, x:X & n64{X} & isfloat{eltype{T}}!=isfloat{eltype{X}} & elwidth{T}>elwidth{X}} = cvt{T, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}} def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}} diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli index 3a68c4d8..2b511616 100644 --- a/src/singeli/src/neq.singeli +++ b/src/singeli/src/neq.singeli @@ -19,7 +19,7 @@ clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { } xv := *V ~~ x rv := *V ~~ r - e := words/2; + e := words/2 c := V**init @for (rv, xv over e) { rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 7b305297..fdfa28ee 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -31,7 +31,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { def last{v, p} = op{pre{v}, p} def scan{v, p} = { n:= last{v, p} - p = (if (width{T}<=32) sel{[8]i32, spread{n}, [8]i32**7}; + p = (if (width{T}<=32) sel{[8]i32, spread{n}, [8]i32**7} else shuf{[4]u64, n, 4b3333}) n }