From 526d9bbebe4e5f5539cc6a4ca4978927fbb5feab Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 10 May 2024 08:30:08 -0400 Subject: [PATCH 1/4] Wrap non-destructuring type parameters in parens --- src/singeli/src/avx.singeli | 4 +-- src/singeli/src/avx2.singeli | 8 +++--- src/singeli/src/base.singeli | 16 +++++------ src/singeli/src/bins.singeli | 2 +- src/singeli/src/bitops.singeli | 44 +++++++++++++++---------------- src/singeli/src/bmi2.singeli | 8 +++--- src/singeli/src/cmp.singeli | 8 +++--- src/singeli/src/copy.singeli | 2 +- src/singeli/src/debug.singeli | 2 +- src/singeli/src/f64.singeli | 20 +++++++------- src/singeli/src/fold.singeli | 8 +++--- src/singeli/src/hashtab.singeli | 6 ++--- src/singeli/src/mask.singeli | 4 +-- src/singeli/src/replicate.singeli | 14 +++++----- src/singeli/src/scan.singeli | 12 ++++----- src/singeli/src/search.singeli | 8 +++--- src/singeli/src/select.singeli | 6 ++--- src/singeli/src/sse2.singeli | 24 ++++++++--------- 18 files changed, 98 insertions(+), 98 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index c3c645b0..b4b6894f 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -20,8 +20,8 @@ def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3} def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3} # f32 arith -def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a} -def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a} +def rsqrtE{a:([8]f32)} = emit{[8]f32, '_mm256_rsqrt_ps', a} +def rcpE{a:([8]f32)} = emit{[8]f32, '_mm256_rcp_ps', a} # conversion def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 3e317ace..d50e1543 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,8 +1,8 @@ # questionable pack -def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}} -def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}} -def unpackQ{a:[ 8]i32, b:[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}} -def unpackQ{a:[ 4]i64, b:[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}} +def unpackQ{a:T,b:T & T==[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}} +def unpackQ{a:T,b:T & T==[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}} +def unpackQ{a:T,b:T & T==[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}} +def unpackQ{a:T,b:T & T==[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}} # inverse of questionable pack; these saturate the argument def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b} def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index d5e11120..7bb4f8e2 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -32,12 +32,12 @@ def load{p:P, n & isvec{eltype{P}}} = assert{0} def store{p:P, n, v & isvec{eltype{P}}} = assert{0} def load{p:P & isptr{P}} = load{p, 0} # def store{p:P, v & isptr{P}} = store{p, 0, v} -def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} -def storeu{p:T, v:eltype{T} & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} -def loadu{p:T & issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p} -def storeu{p:T, v:eltype{T} & issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}} -def loadu{p:T & elwidth{T}==8} = load{p} -def storeu{p:T, v:eltype{T} & elwidth{T}==8} = store{p, v} +def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} +def storeu{p:T, v:(eltype{T}) & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} +def loadu{p:T & issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p} +def storeu{p:T, v:(eltype{T}) & issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}} +def loadu{p:T & elwidth{T}==8} = load{p} +def storeu{p:T, v:(eltype{T}) & elwidth{T}==8} = store{p, v} def reinterpret{T, x:X & T==X} = x @@ -47,11 +47,11 @@ def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} } # hints def rare{x & knum{x}} = x -def rare{x:u1} = emit{u1, '__builtin_expect', x, 0} +def rare{x:(u1)} = emit{u1, '__builtin_expect', x, 0} def assert{x & x==0} = assert{'failed assertion'} def assert{x & x==1} = 1 def unreachable{} = emit{void, 'si_unreachable'} -def assert{x:u1} = { if (not x) emit{void, 'si_unreachable'} } +def assert{x:(u1)} = { if (not x) emit{void, 'si_unreachable'} } # various checks def oneVal{{h, ...t}} = { diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index adc56ef7..8aade258 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -111,7 +111,7 @@ fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = { fn write_indices{I,T & width{I}==8}(t:*I, w:*T, n:u64) : void = { @for (w over j to n) store{t, w, cast_i{I, j+1}} } -def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { +def bins_lookup{I, T, up, w:*T, wn:(u64), x:*T, xn:(u64), rp:(*void)} = { # Build table def tc = 1<>6}>>(n&63)) & 1) != 0 } -def b_getBatchLo{sz, x:*u64, n:(ux) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) -def b_getBatchLo{sz, x:*u64, n:(ux) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) -def b_getBatchLo{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatchLo{sz, x:(*u64), n:(ux) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) +def b_getBatchLo{sz, x:(*u64), n:(ux) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) +def b_getBatchLo{sz, x:(*u64), n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} -def b_getBatch{sz, x:*u64, n:(ux) & sz==2} = b_getBatchLo{sz, x, n} & 3 -def b_getBatch{sz, x:*u64, n:(ux) & sz==4} = b_getBatchLo{sz, x, n} & 15 -def b_getBatch{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatch{sz, x:(*u64), n:(ux) & sz==2} = b_getBatchLo{sz, x, n} & 3 +def b_getBatch{sz, x:(*u64), n:(ux) & sz==4} = b_getBatchLo{sz, x, n} & 15 +def b_getBatch{sz, x:(*u64), n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} -def b_set{x:*u64, n:(ux), v:u1} = { +def b_set{x:(*u64), n:(ux), v:(u1)} = { m:u64 = cast{u64,1}<<(n&63) p:u64 = load{x,n>>6} if (v) store{x,n>>6,p | m} else store{x,n>>6,p & ~m} } -def b_setBatch{sz, x:*u64, n:(ux), v} = { +def b_setBatch{sz, x:(*u64), n:(ux), v} = { vc:u64 = promote{u64,v} am:u64 = 64/sz w:u64 = load{x,n/am} @@ -30,7 +30,7 @@ def b_setBatch{sz, x:*u64, n:(ux), v} = { store{x, n/am, w} } -def b_setBatch{sz, x:*u64, n:(ux), v & sz==4} = { +def b_setBatch{sz, x:(*u64), n:(ux), v & sz==4} = { x8:= *u8 ~~ x #w:u64 = cast_i{u64, load{x8,n/2}} @@ -49,12 +49,12 @@ def b_setBatch{sz, x:*u64, n:(ux), v & sz==4} = { store{x8, n/2, cast_i{u8,w}} } -def b_setBatch{sz, x:*u64, n:(ux), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}} -def b_setBatch{sz, x:*u64, n:(ux), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}} -def b_setBatch{sz, x:*u64, n:(ux), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}} -def b_setBatch{sz, x:*u64, n:(ux), v & sz==64} = store{ x, n, cast_i{u64,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}} +def b_setBatch{sz, x:(*u64), n:(ux), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v & sz==64} = store{ x, n, cast_i{u64,v}} -def spreadBits{T==[32]u8, a:u32} = { +def spreadBits{T==[32]u8, a:(u32)} = { def idxs = iota{32} b:= [8]u32**a c:= [32]u8~~b @@ -63,11 +63,11 @@ def spreadBits{T==[32]u8, a:u32} = { e == (d&e) } -def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = { +def spreadBits{T==[16]u8, a:(u16) & hasarch{'AARCH64'}} = { b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}} andnz{b, make{[16]u8, 1<<(iota{16}&7)}} } -def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = { +def spreadBits{T==[16]u8, a:(u16) & hasarch{'X86_64'}} = { b:= [16]u8~~[8]u16**a exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} (exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0 @@ -78,22 +78,22 @@ def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { b == (b & T ~~ re_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter } -def loadBatchBit{T, x:*u64, n:(ux)} = { # vector with type T with each element being either all 0s or 1s +def loadBatchBit{T, x:(*u64), n:(ux)} = { # vector with type T with each element being either all 0s or 1s spreadBits{T, b_getBatchLo{vcount{T}, x, n}} } # load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it -def loaduBitRaw{x:*u64, i} = { +def loaduBitRaw{x:(*u64), i} = { loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7) } -def loaduBit{x:*u64, i, n} = { +def loaduBit{x:(*u64), i, n} = { assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))} loaduBitRaw{x, i} } -def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}} +def loaduBitTrunc{x:(*u64), i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}} -def loadBatchBit{T, x:*u64, is & ktup{is}} = { +def loadBatchBit{T, x:(*u64), is & ktup{is}} = { # def len = tuplen{is} # def count = vcount{T} # assert{count*len <= 64} diff --git a/src/singeli/src/bmi2.singeli b/src/singeli/src/bmi2.singeli index cf13020e..640ca3bd 100644 --- a/src/singeli/src/bmi2.singeli +++ b/src/singeli/src/bmi2.singeli @@ -1,4 +1,4 @@ -def pdep{x:u64, m:u64} = emit{u64, '_pdep_u64', x, m} -def pdep{x:u32, m:u32} = emit{u32, '_pdep_u32', x, m} -def pext{x:u64, m:u64} = emit{u64, '_pext_u64', x, m} -def pext{x:u32, m:u32} = emit{u32, '_pext_u32', x, m} +def pdep{x:(u64), m:(u64)} = emit{u64, '_pdep_u64', x, m} +def pdep{x:(u32), m:(u32)} = emit{u32, '_pdep_u32', x, m} +def pext{x:(u64), m:(u64)} = emit{u64, '_pext_u64', x, m} +def pext{x:(u32), m:(u32)} = emit{u32, '_pext_u32', x, m} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 23d1aeca..e47db6ca 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -4,11 +4,11 @@ include './f64' include './bitops' -def fillbits{dst:*u64, len:(ux), v } = { emit{void, 'fillBits', dst, len, v }; return{}; } -def fillbits{dst:*u64, len:(ux), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; } +def fillbits{dst:(*u64), len:(ux), v } = { emit{void, 'fillBits', dst, len, v }; return{}; } +def fillbits{dst:(*u64), len:(ux), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; } def cmp_err{x} = { emit{void, 'cmp_err'}; return{}; } -fn cmpIX(dst:*u64, len:ux, x:u64, v:u1) : void = { +fn cmpIX(dst:(*u64), len:ux, x:(u64), v:(u1)) : void = { nan:u1 = q_f64{x} if (~(nan | q_chr{x})) cmp_err{x} fillbits{dst, len, v&~nan, x} @@ -60,7 +60,7 @@ def pathAS{dst, len, T, op, x & isunsigned{T}} = { -def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:*u64, len:(ux)} = { +def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = { def bulk = vcount{VT}*unr xi:ux = 0 diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index cf70423d..450cc1aa 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -3,7 +3,7 @@ include './mask' include './cbqnDefs' include './bitops' -def copyFromBits{T, loadFn, rp, l:u64} = { +def copyFromBits{T, loadFn, rp, l:(u64)} = { def bulk = vcount{T} def TU = ty_u{T} diff --git a/src/singeli/src/debug.singeli b/src/singeli/src/debug.singeli index 4614fb4a..12578928 100644 --- a/src/singeli/src/debug.singeli +++ b/src/singeli/src/debug.singeli @@ -1,4 +1,4 @@ include 'debug/printf' # printf & lprintf -def assert{x:u1} = { if (not x) emit{void, '__builtin_trap'} } +def assert{x:(u1)} = { if (not x) emit{void, '__builtin_trap'} } def test_assert = assert # test_assert is guaranteed to either not exist, or always trap on bad input diff --git a/src/singeli/src/f64.singeli b/src/singeli/src/f64.singeli index 646e54c4..00dda936 100644 --- a/src/singeli/src/f64.singeli +++ b/src/singeli/src/f64.singeli @@ -1,15 +1,15 @@ -def ceil{x:f64} = emit{f64, 'ceil', x} -def floor{x:f64} = emit{f64, 'floor', x} -def abs{x:f64} = emit{f64, 'fabs', x} +def ceil{x:(f64)} = emit{f64, 'ceil', x} +def floor{x:(f64)} = emit{f64, 'floor', x} +def abs{x:(f64)} = emit{f64, 'fabs', x} def NaN = 0.0/0.0 -def isNaN{x:f64} = x!=x -def qNaN{x:u64} = (x<<1) == (cast{u64, 0x8ff8} << 49) +def isNaN{x:(f64)} = x!=x +def qNaN{x:(u64)} = (x<<1) == (cast{u64, 0x8ff8} << 49) -def ftrunc{T, x:f64 & i8==T} = emit{i8, '', x} -def ftrunc{T, x:f64 & i16==T} = emit{i16, '', x} -def ftrunc{T, x:f64 & i32==T} = emit{i32, '', x} # maybe explicitly use _mm_cvtsd_si32? -def ftrunc{T, x:f64 & i64==T} = emit{i64, '', x} +def ftrunc{T, x:(f64) & i8==T} = emit{i8, '', x} +def ftrunc{T, x:(f64) & i16==T} = emit{i16, '', x} +def ftrunc{T, x:(f64) & i32==T} = emit{i32, '', x} # maybe explicitly use _mm_cvtsd_si32? +def ftrunc{T, x:(f64) & i64==T} = emit{i64, '', x} def fext{x} = emit{f64, '', x} -def interp_f64{x:u64} = emit{f64, 'interp_f64', x} \ No newline at end of file +def interp_f64{x:(u64)} = emit{f64, 'interp_f64', x} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index f513360d..bb132d4e 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,10 +1,10 @@ include './base' include './mask' -def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}} -def opsh32{op}{v:[2]f64, perm} = op{v, shuf{[4]u32, v, perm}} -def mix{op, v:[4]f64 & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} } -def mix{op, v:[2]f64 & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032} +def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}} +def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}} +def mix{op, v:([4]f64) & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} } +def mix{op, v:([2]f64) & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032} def reduce_pairwise{op, plog, x:*T, len, init:T} = { # Pairwise combination to shorten dependency chains diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 411a8d5d..a14cc5dd 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -15,13 +15,13 @@ def memset{p:pT, v, l} = { # These hashes are stored in tables and must be invertible! # Murmur3 -def hash_val{x0:u32} = { +def hash_val{x0:(u32)} = { x := x0 x ^= x >> 16; x *= 0x85ebca6b x ^= x >> 13; x *= 0xc2b2ae35 x ^= x >> 16; x } -def hash_val{x0:u64} = { +def hash_val{x0:(u64)} = { x := x0 x ^= x >> 33; x *= 0xff51afd7ed558ccd x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53 @@ -29,7 +29,7 @@ def hash_val{x0:u64} = { } # CRC32 if (hasarch{'SSE4.2'}) require{'x86intrin.h'} -def hash_val{x:u32 & hasarch{'SSE4.2'}} = { +def hash_val{x:(u32) & hasarch{'SSE4.2'}} = { emit{u32, '_mm_crc32_u32', 0x973afb51, x} } diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 60232f3a..598bb439 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -70,8 +70,8 @@ def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n, # "harmless" pointer cast that'll only cast void* def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got ',p}} -def hCast{T,p:P & same{T,eltype{P}}} = p -def hCast{T,p:P & same{P,*void}} = *T~~p +def hCast{T,p:*T} = p +def hCast{T,p:(*void)} = *T~~p def mlExec{i, iter, vars0, bulk, M} = { def vproc{p:P & isptr{P}} = p diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 98a9f428..45074114 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -20,7 +20,7 @@ def scan_core{upd, set, scan, rp:pT, wp:W, s:(usz)} = { k = e } } -def indrep_by_sum{T, rp:*T, wp, s:(usz), js, inc} = { +def indrep_by_sum{T, rp:(*T), wp, s:(usz), js, inc} = { def scan{ptr, len} = @for (ptr over len) js=ptr+=js def scan{ptr, len & width{T}<=32} = { def scanfn = merge{'si_scan_pluswrap_u',fmtnat{width{T}}} @@ -45,7 +45,7 @@ fn ind_by_scan_i32{W}(xv:*void, rp:*i32, s:usz) : void = { } } -def rep_by_scan{T, wp, xv:*void, rv:*void, s} = { +def rep_by_scan{T, wp, xv:(*void), rv:(*void), s} = { xp := *T~~xv; js := *xp; px := js def inc{j} = {sx:=px; px=load{xp,j}; px-sx} indrep_by_sum{T, *T~~rv, wp, s, js, inc} @@ -101,7 +101,7 @@ def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} rcsh4_dat:*i8 = join{join{each{get_shuf_data{., 4}, rcsh4_dom}}} rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} -def read_shuf_vecs{l, ellw:u64, shp:P} = { # tuple of byte selectors in 1< ...} +def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:(u64)} = { # onreps{inputVector, {nextOutputVector} => ...} def step = vcount{V} nv := n / step j:u64 = 0 @@ -168,7 +168,7 @@ if (hasarch{'AVX2'}) { {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } - def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} + def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} } else if (hasarch{'AARCH64'}) { @@ -176,7 +176,7 @@ if (hasarch{'AVX2'}) { each{{s} => gen{sel{[16]u8, x, s}}, sh} } - def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:u64} = { + def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:(u64)} = { def E = ty_u{eltype{V}} rv:= *E~~rv0 @for (x in *E~~xv0 over i to n) { # autovectorized well enough, probably @@ -240,7 +240,7 @@ fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { each{try, rcsh_vals} } -def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { +def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = { assert{kv > 0} def V = [arch_defvw/width{T}]T @for (x over n) { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index fed3f288..dcf50ec4 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -12,7 +12,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = { @for (x, r over len) r = m = op{m, x} } -def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = { +def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = { def step = arch_defvw/width{T} def V = [step]T p:= V**init @@ -23,7 +23,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = { q:= len & (step-1) if (q!=0) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}} } -def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { +def scan_post{T, init, x:*T, r:*T, len:(u64), op, pre} = { def last{v, p} = op{pre{v}, p} def scan{v, p} = { n:= last{v, p} @@ -149,7 +149,7 @@ fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { def sums{n} = (if (n==0) tup{0}; else { def s=sums{n-1}; merge{s,s+1} }) def widen{v:T} = unpackQ{shuf{[4]u64, v, 4b3120}, T**0} - def sumlanes{x:u32} = { + def sumlanes{x:(u32)} = { b:= [8]u32**x >> make{[8]u32, 4*tail{1, iota{8}}} s:= sel8{[32]u8~~b, ii32>>3 + bit{2}} p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes @@ -157,12 +157,12 @@ fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { d+= sel8{d, bit{2}*(1+bit{3}>>2)-1} d + sel8{d, bit{3}-1} } - def step{x:u32, i, store1} = { + def step{x:(u32), i, store1} = { d:= sumlanes{x} if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100} j:= (w/8)*i def out{v, k} = each{out, widen{v}, 2*k+iota{2}} - def out{v0:[vl]T, k} = { + def out{v0:([vl]T), k} = { v := V~~v0 + c # Update carry at the lane boundary if (w!=32 or tail{1,k}) { @@ -239,7 +239,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { len } # Sum as many vector registers as possible; modifies c and i -def simd_plus_scan_part{X, R}{x:*X, c:(R), r:*R, len:u64, i:u64} = { +def simd_plus_scan_part{X, R}{x:(*X), c:(R), r:(*R), len:(u64), i:(u64)} = { def b = max{width{R}/2, width{X}} def bulk = arch_defvw/b diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 0e4d0287..bb7c22df 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -18,7 +18,7 @@ def findFirst{C, M, F, ...v1} = { F{...args} } -def search{E, x, n:u64, OP} = { +def search{E, x, n:(u64), OP} = { def bulk = arch_defvw/width{E} def VT = [bulk]E def end = makeBranch{ @@ -111,7 +111,7 @@ def readbytes{vtab}{} = { } # Look up bits from table -def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = { +def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void)} = { x:= *u8~~x0 t:= *TI~~tab r:= *u64~~r0 @@ -124,7 +124,7 @@ def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = { x+=k; rem-=k; ++r } } -def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & simd_bittab} = { +def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void) & simd_bittab} = { def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}} def k = vcount{VI} @for (x in *VI~~x0, r in *ty_u{k}~~r0 over cdiv{n,k}) r = bitsel{x} @@ -139,7 +139,7 @@ def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & simd_bittab} = { # - 'mask': Mark Firsts of x0 # - 'unique': Deduplicate of x0 # - 'index': First index of value x at r0+x -def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { +def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = { def rbit = mode == 'mask' def rval = mode == 'unique' def rind = mode == 'index' diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 87724f94..a290baa5 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -9,11 +9,11 @@ include 'util/tup' # def:T - masked original content # b:B - pointer to data to index; if width{B} Date: Fri, 10 May 2024 08:36:24 -0400 Subject: [PATCH 2/4] Replace top-level if with if_inline --- src/singeli/src/base.singeli | 4 ++-- src/singeli/src/bins.singeli | 4 ++-- src/singeli/src/count.singeli | 2 +- src/singeli/src/hashtab.singeli | 2 +- src/singeli/src/replicate.singeli | 4 ++-- src/singeli/src/scan.singeli | 6 +++--- src/singeli/src/search.singeli | 2 +- src/singeli/src/select.singeli | 2 +- src/singeli/src/slash.singeli | 8 ++++---- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 7bb4f8e2..c62bcd2c 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -185,13 +185,13 @@ def homMask{...vs & tuplen{vs}>1} = { (b<<(n/2 * vcount{T})) | a } -if (hasarch{'X86_64'}) { +if_inline (hasarch{'X86_64'}) { include 'arch/iintrinsic/basic' include './sse2' include './sse' include './avx' include './avx2' -} else if (hasarch{'AARCH64'}) { +} else if_inline (hasarch{'AARCH64'}) { include 'arch/neon_intrin/basic' include './neon' } else { diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 8aade258..c5bf9aba 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -40,7 +40,7 @@ fn max_scan{T, up}(x:*T, len:u64) : void = { } def getsel{...x} = assert{'shuffling not supported', show{...x}} -if (hasarch{'AVX2'}) { +if_inline (hasarch{'AVX2'}) { def getsel{h:H & lvec{H, 16, 8}} = { sel{H, pair{h,h}, .} } @@ -295,7 +295,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { } } -if (hasarch{'AVX2'}) { +if_inline (hasarch{'AVX2'}) { fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = { bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn} } diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index bc69303e..3283be4d 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -2,7 +2,7 @@ include './base' include 'util/tup' include './vecfold' -if (hasarch{'SSE2'}) { +if_inline (hasarch{'SSE2'}) { fn sum_vec{T}(v:T) = vfold{+, fold{+, unpackQ{v, T**0}}} def fold_addw{v:T & eltype{T}==i8} = sum_vec{T}(v) } diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index a14cc5dd..c82f4dbf 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -28,7 +28,7 @@ def hash_val{x0:(u64)} = { x ^= x >> 33; x } # CRC32 -if (hasarch{'SSE4.2'}) require{'x86intrin.h'} +if_inline (hasarch{'SSE4.2'}) require{'x86intrin.h'} def hash_val{x:(u32) & hasarch{'SSE4.2'}} = { emit{u32, '_mm_crc32_u32', 0x973afb51, x} } diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index 45074114..f2031c93 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -59,7 +59,7 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}} # Constant replicate -if (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) { +if_inline (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) { fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}} @@ -148,7 +148,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:(u64)} = { # onreps{inputVe } } -if (hasarch{'AVX2'}) { +if_inline (hasarch{'AVX2'}) { def rep_iter_from_sh{sh}{x, gen} = { def l = tuplen{sh} def h = l>>1 diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index dcf50ec4..225f91f3 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -1,7 +1,7 @@ include './base' -if (hasarch{'X86_64'}) { - if (hasarch{'PCLMUL'}) include './clmul' - if (hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}) include './avx512' +if_inline (hasarch{'X86_64'}) { + if_inline (hasarch{'PCLMUL'}) include './clmul' + if_inline (hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}) include './avx512' } include './mask' include './f64' diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index bb7c22df..137b536f 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -60,7 +60,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { 0 } -if (hasarch{'X86_64'} | hasarch{'AARCH64'}) { +if_inline (hasarch{'X86_64'} | hasarch{'AARCH64'}) { export{'simd_search_u8', searchOne{u64, u8}} export{'simd_search_u16', searchOne{u64, u16}} export{'simd_search_u32', searchOne{u64, u32}} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index a290baa5..514689a3 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -1,4 +1,4 @@ -if (hasarch{'AVX2'}) { +if_inline (hasarch{'AVX2'}) { include './base' include './cbqnDefs' diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index e66b97fa..b7181691 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -1,8 +1,8 @@ include './base' -if (hasarch{'X86_64'}) { - if (hasarch{'PCLMUL'}) include './clmul' - if (hasarch{'BMI2'}) include './bmi2' - if (hasarch{'AVX512F'}) include './avx512' +if_inline (hasarch{'X86_64'}) { + if_inline (hasarch{'PCLMUL'}) include './clmul' + if_inline (hasarch{'BMI2'}) include './bmi2' + if_inline (hasarch{'AVX512F'}) include './avx512' } include './mask' include 'util/tup' From 4d6612cb16ceeb1a87caba83b2bafb4550526512 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 10 May 2024 09:49:58 -0400 Subject: [PATCH 3/4] Replace tuplen{} and tupsel{} with length{} and select{} --- src/singeli/README.md | 10 +++++----- src/singeli/src/base.singeli | 14 +++++++------- src/singeli/src/bins.singeli | 14 +++++++------- src/singeli/src/bitops.singeli | 4 ++-- src/singeli/src/count.singeli | 2 +- src/singeli/src/dyarith.singeli | 8 ++++---- src/singeli/src/hashtab.singeli | 4 ++-- src/singeli/src/monarith.singeli | 2 +- src/singeli/src/neon.singeli | 2 +- src/singeli/src/replicate.singeli | 14 +++++++------- src/singeli/src/scan.singeli | 6 +++--- src/singeli/src/scan_common.singeli | 4 ++-- src/singeli/src/search.singeli | 6 +++--- src/singeli/src/select.singeli | 4 ++-- src/singeli/src/slash.singeli | 6 +++--- src/singeli/src/squeeze.singeli | 16 ++++++++-------- src/singeli/src/transpose.singeli | 4 ++-- 17 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index ad3cdfd9..b6658edd 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -148,10 +148,10 @@ Some may also support one scalar argument or arguments with different widths. - `zip{a:T,b:T} : tup{T, T}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` - `mzip{a:T,b:T} : tup{el_dbl{T}, el_dbl{T}}` - reinterpreted `zip{a, b}` -- `zipLo{a:T,b:T} : T` - `tupsel{0, zip{a, b}}` -- `zipHi{a:T,b:T} : T` - `tupsel{1, zip{a, b}}` -- `mzipLo{a:T,b:T} : T` - `tupsel{0, mzip{a, b}}` -- `mzipHi{a:T,b:T} : T` - `tupsel{1, mzip{a, b}}` +- `zipLo{a:T,b:T} : T` - `select{zip{a, b}, 0}` +- `zipHi{a:T,b:T} : T` - `select{zip{a, b}, 1}` +- `mzipLo{a:T,b:T} : T` - `select{mzip{a, b}, 0}` +- `mzipHi{a:T,b:T} : T` - `select{mzip{a, b}, 1}` ## Mask stuff @@ -353,4 +353,4 @@ bmi2.singeli clmul.singeli clmul ---> \ No newline at end of file +--> diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index c62bcd2c..af151f78 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -42,7 +42,7 @@ def storeu{p:T, v:(eltype{T}) & elwidth{T}==8} = store{p, v} def reinterpret{T, x:X & T==X} = x def exportN{f, ...ns} = each{export{.,f}, ns} -def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} } +def exportT{name, fs} = { v:*type{select{fs,0}} = fs; export{name, v} } # hints @@ -151,7 +151,7 @@ def collect{vars,begin,end,iter & knum{begin} & knum{end}} = { } # convert tuple to number in little-endian base b -def base{b,l} = if (0==tuplen{l}) 0 else tupsel{0,l}+b*base{b,slice{l,1}} +def base{b,l} = if (0==length{l}) 0 else select{l,0}+b*base{b,slice{l,1}} @@ -176,8 +176,8 @@ def { def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX -def homMask{...vs & tuplen{vs}>1} = { - def n = tuplen{vs} +def homMask{...vs & length{vs}>1} = { + def n = length{vs} def T = oneType{vs} def RT = ty_u{max{8,vcount{T}*n}} def a = promote{RT, homMask{...slice{vs,0,n/2}}} @@ -319,13 +319,13 @@ def makeOptBranch{enable, Ts, F} = if (enable) makeBranch{Ts, F} else 'not defin def tree_fold{F, x} = { - def h = tuplen{x}>>1 - F{tree_fold{F, slice{x,0,h}}, tree_fold{F, slice{x,h,tuplen{x}}}} + def h = length{x}>>1 + F{tree_fold{F, slice{x,0,h}}, tree_fold{F, slice{x,h,length{x}}}} } def tree_fold{F, {x}} = x def eachx{F, ...args} = { - def l = tree_fold{max, each{{x} => if(ktup{x}) tuplen{x} else 0, args}} + def l = tree_fold{max, each{{x} => if(ktup{x}) length{x} else 0, args}} each{F, ...each{{x} => if (istup{x}) x else l**x, args}} } diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index c5bf9aba..08459bce 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -74,7 +74,7 @@ def get_rtype{len} = { } def rtype_arr{gen} = { def t = each{gen, rtypes} - a:*(type{tupsel{0,t}}) = t + a:*(type{select{t,0}}) = t } # Write the last index of v at t+v, for each unique v in w @@ -230,7 +230,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def un = uninterleave def tr_half{a, b} = each{shufHalves{a,b,.}, tup{16b20, 16b31}} def un{{a,b}} = tr_half{un{a},un{b}} - if (not lanes) tupsel{1,wv} = load{wg, 1} + if (not lanes) select{wv,1} = load{wg, 1} wv = un{wv} if (ex>=2 and wn >= 2*svl) { assert{lanes} # Different transpose pattern needed @@ -238,7 +238,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { tup{wv, wv2} = each{un, tr_half{wv, un{load{wg, 1}}}} } } - def ms{v}{h} = getsel{re_el{I, if (lanes) half{v,h} else tupsel{h,v}}} + def ms{v}{h} = getsel{re_el{I, if (lanes) half{v,h} else select{v,h}}} def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' # Offset at end @@ -257,7 +257,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}} def ltx = cmpx{lt}; def eqx = cmpx{==} @unroll (j to klog) { - m := s | tupsel{klog-1-j,bits} + m := s | select{bits,klog-1-j} s = homBlend{m, s, ltx{selw, m}} } r := if (isub==1) s else s>>(lb{isub}+wd-wi) @@ -345,8 +345,8 @@ def bin_search_branchless{up, w, wn, x, n, res, rtype} = { fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { def param = tup{up, *T~~w, wn, *T~~x, xn, rp} def lookup{k} = { - if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param} - else if (k+1 < tuplen{rtypes}) lookup{k+1} + if (rty == k) bins_lookup{select{rtypes,k}, T, ...param} + else if (k+1 < length{rtypes}) lookup{k+1} } # For >=8 i8 values, vector bit-table is as good as binary search def wn_vec = if (T==i8) 8 else 2*256/width{T} @@ -377,7 +377,7 @@ fn saturate{F,T,...up}(dst:*void, src:*void, n:u64) : void = { def a = minvalue{T}; af := cast_i{F,a} def b = maxvalue{T}; bf := cast_i{F,b} @for (d in *T~~dst, xf in *F~~src over n) { - x := if (F==f64) (if (tupsel{0,up}) floor else ceil){xf} else xf + x := if (F==f64) (if (select{up,0}) floor else ceil){xf} else xf d = cast_i{T, x} if (xbf) d = b diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index e7818571..5dcf9c62 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -94,10 +94,10 @@ def loaduBitTrunc{x:(*u64), i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}} def loadBatchBit{T, x:(*u64), is & ktup{is}} = { - # def len = tuplen{is} + # def len = length{is} # def count = vcount{T} # assert{count*len <= 64} - # bits:= b_getBatchLo{count*len, x, tupsel{0,is}} + # bits:= b_getBatchLo{count*len, x, select{is,0}} # @collect(i to len) spreadBits{T, truncBits{count, bits>>(i*count)}} each{loadBatchBit{T, x, .}, is} } diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 3283be4d..1a8ed9ad 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -46,7 +46,7 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { total := trunc{usz, r0} # To compute last count def count_each{js, num} = { j := @collect (k to num) trunc{T, js+k} - c := copy{tuplen{j}, [vec]uT ** 0} + c := copy{length{j}, [vec]uT ** 0} e := each{{j}=>V**j, j} @for (xv over b) each{{c,e} => c -= xv == e, c, e} def add_sum{c, j} = { diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 8c628f44..e61074b7 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -107,7 +107,7 @@ def runner{u, R, F} = { } # homAny, topAny already give masked vals; anyne doesn't, and ~andAllZero assumes no masking -def runChecks_any{F, vals} = { F{tree_fold{|, each{tupsel{1,.}, vals}}} } +def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} } def runChecks{type=='homAny', vals, M} = runChecks_any{homAny, vals} def runChecks{type=='topAny', vals, M} = runChecks_any{topAny, vals} def runChecks{type=='none', vals, M} = 0 @@ -124,8 +124,8 @@ def runChecks{type=='anyne', vals, M} = { def arithProcess{F, run, overflow, M, is, cw, cx, TY} = { def {values, checks} = flip{each{{w1, x1} => run{F, M, w1, x1}, cw, cx}} - def ctype = oneVal{each{tupsel{0,.}, checks}} - if (rare{runChecks{ctype, checks, M}}) overflow{tupsel{0,is}*vcount{TY}} + def ctype = oneVal{each{select{.,0}, checks}} + if (rare{runChecks{ctype, checks, M}}) overflow{select{is,0}*vcount{TY}} each{{c} => TY~~c, values} } @@ -187,7 +187,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl @muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) { - def cws = tuplen{is}**cw + def cws = length{is}**cw sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}} } diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index c82f4dbf..3a2556fd 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -39,7 +39,7 @@ def hash_val{x:(u32) & hasarch{'SSE4.2'}} = { # Allocates the maximum at the start, resizes downwards within the existing allocation def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { def ws = each{width,Ts} - def wt = tupsel{0,ws} + def wt = select{ws,0} each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment # Variables updated on resize sz := usz~~1 << logsz @@ -61,7 +61,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { sh -= m; sz <<= m set_thresh{} cc = 0 # Collision counter - k:tupsel{0,Ts} = 0; --k # Index where to move an element to + k:select{Ts,0} = 0; --k # Index where to move an element to each{{p,v} => { p -= dif; memset{p, v, dif} }, ptrs, v0s} def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s s := sz+ext diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index ecdee8a3..17f7772e 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { def bulk = arch_defvw/width{T} def VT = [bulk]T @muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { - if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk} + if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk} sr{each{abs, cx}} } len diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 6d8f5704..dc773fc8 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -140,7 +140,7 @@ def homMask{a:T,b:T,c:T,d:T & T==[16]u8} = { t3:= addp{t1, t2} extract{[2]u64~~addp{t3,t3},0} } -def homMask{...as & tuplen{as}>1 & elwidth{type{tupsel{0,as}}}>=32} = homMask{...each{{i}=>narrowPair{tupsel{i*2,as},tupsel{i*2+1,as}}, iota{tuplen{as}/2}}} +def homMask{...as & length{as}>1 & elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = { def n = vcount{T} truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<{r:=v}, l**V**0} def tlen{e} = cdiv{l, e} # Length for e bytes, rounded up - def set{i} = { tupsel{i,sh} = each{load{shp,.}, i} } + def set{i} = { select{sh,i} = each{load{shp,.}, i} } def ext{e} = { def m = tlen{2*e}; def n = tlen{e} # m>1 def fs{v, s} = gen{sel{[16]i8, v, s}} a := shuf{[4]u64, x, 4b1010}; each{fs{a,.}, slice{sh,0,h}} - if (l%2) fs{x, tupsel{h, sh}} + if (l%2) fs{x, select{sh, h}} b := shuf{[4]u64, x, 4b3232}; each{fs{b,.}, slice{sh,-h}} } @@ -201,7 +201,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { re := r + n*wvb - h*step while (r <= re) { a := shufbase{i} - @unroll (j to h) store{*V~~r, j, shufrun{a, tupsel{j,sh}}} + @unroll (j to h) store{*V~~r, j, shufrun{a, select{sh,j}}} i += hs << ellw r += hs*wvb } @@ -211,7 +211,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { def end = makelabel{} @unroll (j to h) { - s = shufrun{a, tupsel{j,sh}} + s = shufrun{a, select{sh,j}} if (r > re) goto{end} store{*V~~r, 0, s} r+= step @@ -230,11 +230,11 @@ fn rcsh_sub{wv, V}(ellw:u64, x:*i8, r:*i8, n:u64, sh:*V) : void = { } fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { - if (wv > tupsel{-1,rcsh_vals}) { + if (wv > select{rcsh_vals,-1}) { return{rep_const_shuffle_partial4(wv, ellw, x, r, n)} } n <<= ellw - ri := wv - tupsel{0,rcsh_vals} + ri := wv - select{rcsh_vals,0} sh := *sdtype~~rcsh_data + load{rcsh_offs,ri} def try{k} = { if (wv==k) rcsh_sub{k, sdtype}(ellw, x, r, n, sh) } each{try, rcsh_vals} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 225f91f3..78f7294c 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -275,12 +275,12 @@ def simd_plus_scan_part{X, R}{x:(*X), c:(R), r:(*R), len:(u64), i:(u64)} = { def s1{v0,v1} = tup{v0,v1+toLast{v0}} def cr = eachx{+, widenFull{R, s1{...s0}}, cv} - cv = toLast{tupsel{-1, cr}} + cv = toLast{select{cr, -1}} assert{type{cv} == oneType{cr}} - assert{vcount{type{cv}} * tuplen{cr} == bulk} + assert{vcount{type{cv}} * length{cr} == bulk} - each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}} + each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{length{cr}}} i = i2 } diff --git a/src/singeli/src/scan_common.singeli b/src/singeli/src/scan_common.singeli index 238b1ff3..d4cf5ad1 100644 --- a/src/singeli/src/scan_common.singeli +++ b/src/singeli/src/scan_common.singeli @@ -1,11 +1,11 @@ # Used by scan.singeli and bins.singeli def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}} -def sel8{v:V, t & w256{V} & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} +def sel8{v:V, t & w256{V} & istup{t} & length{t}==16} = sel8{v, merge{t,t}} def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} -local def rev{t} = { def l=tuplen{t}; def j=l-1; tupsel{j-range{l}, j-t} } +local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} } local def rev{up,t} = if (up) t else rev{t} def sel8{v, t, up} = sel8{v, rev{up,t}} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 137b536f..e4e7c1d3 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -5,8 +5,8 @@ include './hashtab' def findFirst{C, M, F, ...v1} = { def exit = makelabel{} - def args = undef{M{...each{tupsel{0, .}, v1}}} - def am = tuplen{tupsel{0,v1}} + def args = undef{M{...each{select{., 0}, v1}}} + def am = length{select{v1,0}} each{{last, ...v2} => { if (last or C{...v2}) { each{=, args, M{...v2}} @@ -291,7 +291,7 @@ def acc{unr, init:T} = { def op{S=='get'} = a0v def op{S=='tr', F} = { a0v = tree_fold{F, a1} } def op{S=='upd', is, F} = { - if (tuplen{is}==1) a0 = F{a0} + if (length{is}==1) a0 = F{a0} else a1 = F{a1} } } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 514689a3..55fb35ab 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -70,8 +70,8 @@ def makeselx{VI, VD, nsel, xd, logv, cshuf} = { def bb{c}{f, v} = (if (f) bblendn{c0} = { - tupsel{0,b}{each{bs{slice{b,1}, c, .}, x}} + def bs{b, c, x & length{b}>0} = { + select{b,0}{each{bs{slice{b,1}, c, .}, x}} } def i = iota{logv} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index b7181691..b22c3b72 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -149,10 +149,10 @@ fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, @for_special_buffered{r,8} (w in *u8~~w over i to sum) { ind := load{itab, w} pc := popc_alt{w, ind, 8} - v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}} + v := mzipLo{j + I~~make{[2]u64, ind, 0}, select{top,0}} def st{k, v:V} = store{*V~~r, k, v} if (T==i16) st{0, v} - else each{st, iota{2}, mzip{v, tupsel{1,top}}} + else each{st, iota{2}, mzip{v, select{top,1}}} r += pc j += I**8 inctop{i, top} @@ -375,7 +375,7 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { } ro = ro2%64 } - def extract{t, i & istup{t}} = tupsel{i,t} + def extract{t, i & istup{t}} = select{t,i} def v = pext_width{} if (v > 1) { def V = [v]u64 diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index b7b531c2..a794c995 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -92,21 +92,21 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { @muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) { def int = { def {int, wdn} = { - if (hasarch{'AARCH64'} and tuplen{is}==2) { + if (hasarch{'AARCH64'} and length{is}==2) { def intp = narrowPair{...each{cvt{i64,.}, v0}} def wdn = each{cvt{f64,.}, widen{intp}} tup{intp, wdn} } else { def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0} def wdn = each{{v} => cvtWiden{XV, v}, ints} - if (vcount{type{tupsel{0,ints}}} == bulk) { + if (vcount{type{select{ints,0}}} == bulk) { def intp = { - if (tuplen{ints}==1) tupsel{0, ints} + if (length{ints}==1) select{ints, 0} else pair{ints} } tup{intp, wdn} - } else if (tuplen{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) { - tup{tupsel{0, ints}, wdn} + } else if (length{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) { + tup{select{ints, 0}, wdn} } else assert{0} } } @@ -115,16 +115,16 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { def as = each{conv, v0} def bs = each{conv, wdn} def cond = { - if (tuplen{is}==1) anynePositive{...as, ...bs, M} + if (length{is}==1) anynePositive{...as, ...bs, M} else ~homAll{tree_fold{&, each{==, as, bs}}} } if (cond) { # is any not an integer - if (B) case_B{tupsel{0, is}} # if B, need to give an even more special result + if (B) case_B{select{is, 0}} # if B, need to give an even more special result else return{0xffff_ffff} # else, not integer => float } int } - def acc = { if (tuplen{is}==2) r2; else r1 } + def acc = { if (length{is}==2) r2; else r1 } acc|= M{getAcc{type{acc}, int}} } diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 4fe11838..8ebab38e 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -7,9 +7,9 @@ include './bitops' # Group l (power of 2) elements into paired groups of length o # e.g. pairs{2, iota{8}} = {{0,1,4,5}, {2,3,6,7}} def pairs{o, x} = { - def i = iota{tuplen{x}/2} + def i = iota{length{x}/2} def g = 2*i - i%o - tupsel{tup{g, g+o}, x} + select{x, tup{g, g+o}} } def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} def permute_pass{o, x} = { From b2e3a5ff74d3025614ca0b12073b1a2b8ec90313 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 10 May 2024 10:26:28 -0400 Subject: [PATCH 4/4] Move from & to if/and for Singeli conditions --- src/singeli/src/avx.singeli | 56 ++++----- src/singeli/src/avx2.singeli | 120 +++++++++---------- src/singeli/src/avx512.singeli | 16 +-- src/singeli/src/base.singeli | 172 ++++++++++++++-------------- src/singeli/src/bins.singeli | 14 +-- src/singeli/src/bitops.singeli | 32 +++--- src/singeli/src/cbqnDefs.singeli | 30 ++--- src/singeli/src/clmul.singeli | 2 +- src/singeli/src/cmp.singeli | 24 ++-- src/singeli/src/count.singeli | 2 +- src/singeli/src/dyarith.singeli | 38 +++--- src/singeli/src/equal.singeli | 6 +- src/singeli/src/f64.singeli | 8 +- src/singeli/src/fold.singeli | 4 +- src/singeli/src/hashtab.singeli | 2 +- src/singeli/src/mask.singeli | 34 +++--- src/singeli/src/neon.singeli | 164 +++++++++++++------------- src/singeli/src/replicate.singeli | 10 +- src/singeli/src/scan.singeli | 22 ++-- src/singeli/src/scan_common.singeli | 20 ++-- src/singeli/src/search.singeli | 12 +- src/singeli/src/select.singeli | 6 +- src/singeli/src/slash.singeli | 60 +++++----- src/singeli/src/squeeze.singeli | 18 +-- src/singeli/src/sse.singeli | 38 +++--- src/singeli/src/sse2.singeli | 104 ++++++++--------- src/singeli/src/transpose.singeli | 12 +- src/singeli/src/vecfold.singeli | 4 +- 28 files changed, 515 insertions(+), 515 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index b4b6894f..d748be9d 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -1,54 +1,54 @@ # compact casting for the annoying intrinsic type system -def v2i{x:T & w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x -def v2f{x:T & w256{T}} = [8]f32 ~~ x -def v2d{x:T & w256{T}} = [4]f64 ~~ x +def v2i{x:T if w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x +def v2f{x:T if w256{T}} = [8]f32 ~~ x +def v2d{x:T if w256{T}} = [4]f64 ~~ x -def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} +def undefPromote{T, x:X if w128{X} and w256{T} and eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store -def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} -def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr} +def loadLow{ptr:P, w if w256{eltype{P}} and w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} +def loadLow{ptr:P, w if w256{eltype{P}} and w==256} = load{ptr} -def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}} -def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x} +def storeLow{ptr:P, w, x:T if w256{T} and w<=128} = storeLow{ptr, w, half{x, 0}} +def storeLow{ptr:P, w, x:T if w256{T} and w==256} = store{*T~~ptr, 0, x} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n} -def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3} -def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3} +def unord{a:T,b:T if T==[8]f32} = f32cmpAVX{a,b,3} +def unord{a:T,b:T if T==[4]f64} = f64cmpAVX{a,b,3} # f32 arith def rsqrtE{a:([8]f32)} = emit{[8]f32, '_mm256_rsqrt_ps', a} def rcpE{a:([8]f32)} = emit{[8]f32, '_mm256_rcp_ps', a} # conversion -def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} -def half{x:T, i==0 & w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} -def pair{a:T,b:T & width{T}==128} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} +def half{x:T, i if w256{T} and knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} +def half{x:T, i==0 if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} +def pair{a:T,b:T if width{T}==128} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} -def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x} -def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x} -def widen{T==[4]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} +def widen{T==[4]f64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x} +def widen{T==[4]f64, x:X if X==[4]f32} = emit{T, '_mm256_cvtps_pd', x} +def widen{T==[4]f64, x:X if w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T, x:X if w256{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}} # structural operations -def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} -def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m} +def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}} +def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} +def homBlend{f:T, t:T, m:M if w256{T}} = topBlend{f, t, m} # mask stuff -def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} +def andAllZero{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} -def topMask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} -def topMask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} -def homMask{x:T & w256{T}} = topMask{x} +def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} +def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} +def homMask{x:T if w256{T}} = topMask{x} -def homAny{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 -def homAll{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff +def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 +def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff -def topAny{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} != 0 -def topAll{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} == (1<=32} = topMask{x} != 0 +def topAll{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} == (1<=32} = topMaskStore{p,m,v} -def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v} -def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} +def topMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} # mask stuff -def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} -def topMask{x:T & w256{T, 16}} = { +def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} +def topMask{x:T if w256{T, 16}} = { msk:u32 = topMask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}} (msk&255) | (msk>>8) } -def homAny{x:T & w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} -def homAll{x:T & w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff -def topAny{x:T & w256i{T}} = topMask{x} != 0 -def topAll{x:T & w256i{T}} = topMask{x} == (1<=32} = topAny{x} +def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}} +def topAny{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}} +def homAny{x:T if w256i{T} and elwidth{T}>=32} = topAny{x} -def topAny{x:T & w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0} -def topAll{x:T & w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0} +def topAny{x:T if w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0} +def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0} # conversion -def widen{T==[16]u16, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x} -def widen{T==[ 8]u32, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x} -def widen{T==[ 8]u32, x:X & X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X & X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x} -def widen{T==[ 4]u64, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x} -def widen{T==[ 4]u64, x:X & X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X & X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x} -def widen{T==[ 4]u64, x:X & X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x} +def widen{T==[16]u16, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x} +def widen{T==[ 8]u32, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x} +def widen{T==[ 8]u32, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x} +def widen{T==[ 4]u64, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x} +def widen{T==[ 4]u64, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x} +def widen{T==[ 4]u64, x:X if X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x} -def narrow{T, x:X & w256i{X,32} & width{T}==8} = { +def narrow{T, x:X if w256i{X,32} and width{T}==8} = { a:= packQ{x, x} b:= packQ{a, a} re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}} } -def narrow{T, x:X & w256i{X,32} & width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} -def narrow{T, x:X & w256i{X,16} & width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} -def narrow{T, x:X & w256f{X,64} & T>1<<2) | (iota{32}&1)}}} -def narrow{T, x:X & w256u{X,64} & T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}} +def narrow{T, x:X if w256u{X,64} and T==u32} = re_el{T, sel{[8]i32, x, make{[8]i32, 2*iota{8}}}} +def narrow{T, x:X if w256u{X,64} and T==u16} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}} +def narrow{T, x:X if w256u{X,64} and T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}} -def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} -def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} +def cvt2{T, x:X if T==i32 and X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} +def cvt2{T, x:X if T==f64 and X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} diff --git a/src/singeli/src/avx512.singeli b/src/singeli/src/avx512.singeli index 9f7936fb..737ff41c 100644 --- a/src/singeli/src/avx512.singeli +++ b/src/singeli/src/avx512.singeli @@ -4,30 +4,30 @@ local { if (isfloat{T}) (if (width{T}==32) 'ps' else 'pd') else merge{'epi', fmtnat{width{T}}} } - def suf{V & isvec{V}} = suf{eltype{V}} + def suf{V if isvec{V}} = suf{eltype{V}} def pref{w} = merge{'_mm', if (w==128) '' else fmtnat{w}, '_'} - def pref{V & isvec{V}} = pref{width{V}} + def pref{V if isvec{V}} = pref{width{V}} } local def re_mask{M, sub} = { def l = vcount{M}; def w = max{32,l} sub{fmtnat{l}, fmtnat{w}, ty_u{w}} } -def reinterpret{M, a:T & ismask{M} & width{T}==width{M}} = { +def reinterpret{M, a:T if ismask{M} and width{T}==width{M}} = { re_mask{M, {l,w,W} => emit{M, merge{'_cvtu',w,'_mask',l}, promote{W, a}}} } -def reinterpret{T, a:M & ismask{M} & width{T}==width{M}} = { +def reinterpret{T, a:M if ismask{M} and width{T}==width{M}} = { re_mask{M, {l,w,W} => cast_i{T, emit{W, merge{'_cvtmask',l,'_u',w}, a}}} } -def maskStore{p:*V, m:M, v:V & ismask{M} & isvec{V} & vcount{M}==vcount{V}} = { +def maskStore{p:*V, m:M, v:V if ismask{M} and isvec{V} and vcount{M}==vcount{V}} = { emit{void, merge{pref{V}, 'mask_storeu_', suf{V}}, p, m, v} } def topMaskReg{x:V} = emit{[vcount{V}]u1, merge{pref{V},'mov',suf{V},'_mask'}, x} -def topMask{x:T & isvec{T} & 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x} -def homMask{x:T & isvec{T} & 512==width{T}} = topMask{x} +def topMask{x:T if isvec{T} and 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x} +def homMask{x:T if isvec{T} and 512==width{T}} = topMask{x} -def maskToHom{T, x:M & ismask{M} & isvec{T} & vcount{M}==vcount{T}} = { +def maskToHom{T, x:M if ismask{M} and isvec{T} and vcount{M}==vcount{T}} = { emit{T, merge{pref{T},'movm_',suf{T}}, x} } diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index af151f78..b8fe0710 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -18,38 +18,38 @@ def istup = ktup def isunsigned{T} = isint{T} & ~issigned{T} -def isvec {T} = 0; def isvec {T & istype{T}} = same{typekind{T},'vector'} -def isprim{T} = 0; def isprim{T & istype{T}} = same{typekind{T},'primitive'} -def isptr {T} = 0; def isptr {T & istype{T}} = same{typekind{T},'pointer'} +def isvec {T} = 0; def isvec {T if istype{T}} = same{typekind{T},'vector'} +def isprim{T} = 0; def isprim{T if istype{T}} = same{typekind{T},'primitive'} +def isptr {T} = 0; def isptr {T if istype{T}} = same{typekind{T},'pointer'} def elwidth{T} = width{eltype{T}} oper &~ andnot infix none 35 -def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b +def andnot{a, b if anyNum{a} and anyNum{b}} = a & ~b -def load{p:P, n & isvec{eltype{P}}} = assert{0} -def store{p:P, n, v & isvec{eltype{P}}} = assert{0} -def load{p:P & isptr{P}} = load{p, 0} -# def store{p:P, v & isptr{P}} = store{p, 0, v} -def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} -def storeu{p:T, v:(eltype{T}) & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} -def loadu{p:T & issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p} -def storeu{p:T, v:(eltype{T}) & issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}} -def loadu{p:T & elwidth{T}==8} = load{p} -def storeu{p:T, v:(eltype{T}) & elwidth{T}==8} = store{p, v} +def load{p:P, n if isvec{eltype{P}}} = assert{0} +def store{p:P, n, v if isvec{eltype{P}}} = assert{0} +def load{p:P if isptr{P}} = load{p, 0} +# def store{p:P, v if isptr{P}} = store{p, 0, v} +def loadu{p:T if isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} +def storeu{p:T, v:(eltype{T}) if isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} +def loadu{p:T if issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p} +def storeu{p:T, v:(eltype{T}) if issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}} +def loadu{p:T if elwidth{T}==8} = load{p} +def storeu{p:T, v:(eltype{T}) if elwidth{T}==8} = store{p, v} -def reinterpret{T, x:X & T==X} = x +def reinterpret{T, x:X if T==X} = x def exportN{f, ...ns} = each{export{.,f}, ns} def exportT{name, fs} = { v:*type{select{fs,0}} = fs; export{name, v} } # hints -def rare{x & knum{x}} = x +def rare{x if knum{x}} = x def rare{x:(u1)} = emit{u1, '__builtin_expect', x, 0} -def assert{x & x==0} = assert{'failed assertion'} -def assert{x & x==1} = 1 +def assert{x if x==0} = assert{'failed assertion'} +def assert{x if x==1} = 1 def unreachable{} = emit{void, 'si_unreachable'} def assert{x:(u1)} = { if (not x) emit{void, 'si_unreachable'} } @@ -65,24 +65,24 @@ def anyNum{x} = isconst{x} | knum{x} def anyNum{x:T} = isprim{T} def anyInt{x} = 0 -def anyInt{x & knum{x}} = (x>>0) == x -def anyInt{x & isreg{x}|isconst{x}} = isint{x} +def anyInt{x if knum{x}} = (x>>0) == x +def anyInt{x if isreg{x}|isconst{x}} = isint{x} # vector width/type checks -def w64 {T} = 0; def w64 {T & isvec{T}} = width{T}==64 -def w128{T} = 0; def w128{T & isvec{T}} = width{T}==128 -def w256{T} = 0; def w256{T & isvec{T}} = width{T}==256 -def w64 {T,w} = 0; def w64 {T,w & w64{T}} = elwidth{T}==w -def w128{T,w} = 0; def w128{T,w & w128{T}} = elwidth{T}==w -def w256{T,w} = 0; def w256{T,w & w256{T}} = elwidth{T}==w +def w64 {T} = 0; def w64 {T if isvec{T}} = width{T}==64 +def w128{T} = 0; def w128{T if isvec{T}} = width{T}==128 +def w256{T} = 0; def w256{T if isvec{T}} = width{T}==256 +def w64 {T,w} = 0; def w64 {T,w if w64{T}} = elwidth{T}==w +def w128{T,w} = 0; def w128{T,w if w128{T}} = elwidth{T}==w +def w256{T,w} = 0; def w256{T,w if w256{T}} = elwidth{T}==w # width+type checks def genchk{B, F} = { def r{T} = 0 - def r{T & B{T}} = F{eltype{T}} + def r{T if B{T}} = F{eltype{T}} def r{T,w} = 0 - def r{T,w & B{T}} = F{eltype{T}} & (elwidth{T}==w) - def r{T & ~isvec{T}} = 0 + def r{T,w if B{T}} = F{eltype{T}} & (elwidth{T}==w) + def r{T if ~isvec{T}} = 0 r } def w256i = genchk{w256, isint}; def w128i = genchk{w128, isint}; def w64i = genchk{w64, isint} @@ -92,14 +92,14 @@ def w256f = genchk{w256, isfloat}; def w128f = genchk{w128, isfloat}; de -def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x} -def trunc{T, x & knum{x}} = cast{T, x} +def trunc{T, x:U if isint{T} and isint{U} and T<=U} = emit{T, '', x} +def trunc{T, x if knum{x}} = cast{T, x} -def tern{c, T, F & anyInt{c}} = { +def tern{c, T, F if anyInt{c}} = { if(c) T else F } -def tern{c, t:T, f:T & anyInt{c}} = { +def tern{c, t:T, f:T if anyInt{c}} = { res:T = f if (c) res = t res @@ -111,42 +111,42 @@ def re_el{E, V} = [width{V}/width{E}]E def re_el{E, x:V} = re_el{E,V} ~~ x local def qualChange{q} = { - def f{w & knum{w}} = primtype{q, w} - def f{T & isprim{T}} = primtype{q, width{T}} - def f{T & isvec{T}} = re_el{f{eltype{T}}, T} + def f{w if knum{w}} = primtype{q, w} + def f{T if isprim{T}} = primtype{q, width{T}} + def f{T if isvec{T}} = re_el{f{eltype{T}}, T} def f{x:T} = f{T}~~x } def ty_u = qualChange{'u'} def ty_s = qualChange{'i'} def ty_f = qualChange{'f'} -def w_n{T, w & isprim{T}} = primtype{quality{T}, w} -def w_d{T & isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width -def w_h{T & isprim{T}} = to_w{T, width{T}/2} +def w_n{T, w if isprim{T}} = primtype{quality{T}, w} +def w_d{T if isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width +def w_h{T if isprim{T}} = to_w{T, width{T}/2} -def n_d{T & isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count -def n_h{T & isvec{T}} = [vcount{T}/2](eltype{T}) +def n_d{T if isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count +def n_h{T if isvec{T}} = [vcount{T}/2](eltype{T}) -def el_d{T & isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count -def el_h{T & isvec{T}} = [vcount{T}](w_h{eltype{T}}) +def el_d{T if isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count +def el_h{T if isvec{T}} = [vcount{T}](w_h{eltype{T}}) -def el_m{T & isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width -def el_s{T & isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x +def el_m{T if isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width +def el_s{T if isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x # type stats -def minvalue{T & isunsigned{T}} = 0 -def maxvalue{T & isunsigned{T}} = (1<v, range{n}} -def collect{vars,begin,end,iter & knum{begin} & knum{end}} = { +def iota{n if knum{n}} = range{n} +def broadcast{T, v if isprim{T}} = v +def broadcast{n, v if knum{n}} = each{{_}=>v, range{n}} +def collect{vars,begin,end,iter if knum{begin} and knum{end}} = { each{iter{., vars}, range{end-begin}+begin} } @@ -162,7 +162,7 @@ def fast_BMI2{} = if (SLOW_PDEP) 0 else hasarch{'BMI2'} # test if vector has a specific width & element type def lvec{T, n, w} = 0 -def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 +def lvec{T, n, w if isvec{T} and vcount{T}==n and elwidth{T}==w} = 1 # base cases def { @@ -176,7 +176,7 @@ def { def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX -def homMask{...vs & length{vs}>1} = { +def homMask{...vs if length{vs}>1} = { def n = length{vs} def T = oneType{vs} def RT = ty_u{max{8,vcount{T}*n}} @@ -204,15 +204,15 @@ def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} def packQ{{a, b}} = packQ{a, b} def pair{{a, b}} = pair{a, b} -def widen{T, x:X & T==X} = x -def narrow{T, x:X & T==eltype{X}} = x -def undefPromote{T, x:X & T==X} = x -def cvt{T, x:X & T==eltype{X}} = x +def widen{T, x:X if T==X} = x +def narrow{T, x:X if T==eltype{X}} = x +def undefPromote{T, x:X if T==X} = x +def cvt{T, x:X if T==eltype{X}} = x -def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} -def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} -def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} -def absu{a:T & isvec{T}} = ty_u{abs{a}} +def broadcast{T, v if isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} +def make{T, ...xs if isvec{T}} = vec_make{T, ...xs} +def iota{T if isvec{T}} = make{T, ...iota{vcount{T}}} +def absu{a:T if isvec{T}} = ty_u{abs{a}} def floor = __floor def ceil = __ceil @@ -226,29 +226,29 @@ def sqrt = __sqrt # more arith -def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +def min{a, b if anyNum{a} and anyNum{b}} = tern{ab, a, b} def cdiv{a,b} = (a+b-1)/b # ceiling divide -def cdiv{a,b & knum{a} & knum{b}} = ceil{a/b} -def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x} -def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x} -def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x} -def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x} -def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x} -def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x} +def cdiv{a,b if knum{a} and knum{b}} = ceil{a/b} +def popc{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_popcountll', x} +def popc{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_popcount', x} +def ctz{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_ctzll', x} +def ctz{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_ctz', x} +def clz{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_clzll', x} +def clz{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_clz', x} # count-leading-zeros complement, less type-dependent -def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} -def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} +def clzc{x:T if isint{T} and width{T}==64} = 64-clz{x} +def clzc{x:T if isint{T} and width{T}<=32} = 32-clz{x} def ceil_log2{n} = clzc{n-1} -def truncBits{n, v & n<=8} = cast_i{u8, v} -def truncBits{n, v & n==16} = cast_i{u16, v} -def truncBits{n, v & n==32} = cast_i{u32, v} -def truncBits{n, v & n==64} = cast_i{u64, v} +def truncBits{n, v if n<=8} = cast_i{u8, v} +def truncBits{n, v if n==16} = cast_i{u16, v} +def truncBits{n, v if n==32} = cast_i{u32, v} +def truncBits{n, v if n==64} = cast_i{u64, v} # base-2 log of a constant power of two -def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 +def lb{n if knum{n} and (n>>1<<1) == n and n>0} = lb{n>>1}+1 def lb{n==1} = 0 def zlow{n,x} = (x >> n) << n # zero out n least significant bits @@ -256,11 +256,11 @@ def tail{n,x} = x & ((1< if (istup{x}) x else l**x, args}} } -def undef{T, n & istype{T}} = @collect(n) undef{T} -def undef{Ts & istup{Ts}} = each{undef, Ts} +def undef{T, n if istype{T}} = @collect(n) undef{T} +def undef{Ts if istup{Ts}} = each{undef, Ts} def undef{x:T} = undef{T} -def undef{T & istype{T}} = { reg:=undefined{T} } +def undef{T if istype{T}} = { reg:=undefined{T} } diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 08459bce..128ab8bb 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -41,20 +41,20 @@ fn max_scan{T, up}(x:*T, len:u64) : void = { def getsel{...x} = assert{'shuffling not supported', show{...x}} if_inline (hasarch{'AVX2'}) { - def getsel{h:H & lvec{H, 16, 8}} = { + def getsel{h:H if lvec{H, 16, 8}} = { sel{H, pair{h,h}, .} } - def getsel{v:V & lvec{V, 32, 8}} = { + def getsel{v:V if lvec{V, 32, 8}} = { def H = n_h{V} vtop := V**(vcount{V}/2) hs := each{shuf{[4]u64, v, .}, tup{4b3232, 4b1010}} {i} => homBlend{...each{sel{H,.,i}, hs}, V~~i 1}; assert{wn < maxwn} diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 5dcf9c62..6d56a11f 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -4,13 +4,13 @@ def b_get{x:(*u64), n:(ux)} = { ((load{x,n>>6}>>(n&63)) & 1) != 0 } -def b_getBatchLo{sz, x:(*u64), n:(ux) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) -def b_getBatchLo{sz, x:(*u64), n:(ux) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) -def b_getBatchLo{sz, x:(*u64), n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) +def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) +def b_getBatchLo{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n} -def b_getBatch{sz, x:(*u64), n:(ux) & sz==2} = b_getBatchLo{sz, x, n} & 3 -def b_getBatch{sz, x:(*u64), n:(ux) & sz==4} = b_getBatchLo{sz, x, n} & 15 -def b_getBatch{sz, x:(*u64), n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatch{sz, x:(*u64), n:(ux) if sz==2} = b_getBatchLo{sz, x, n} & 3 +def b_getBatch{sz, x:(*u64), n:(ux) if sz==4} = b_getBatchLo{sz, x, n} & 15 +def b_getBatch{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n} def b_set{x:(*u64), n:(ux), v:(u1)} = { @@ -30,7 +30,7 @@ def b_setBatch{sz, x:(*u64), n:(ux), v} = { store{x, n/am, w} } -def b_setBatch{sz, x:(*u64), n:(ux), v & sz==4} = { +def b_setBatch{sz, x:(*u64), n:(ux), v if sz==4} = { x8:= *u8 ~~ x #w:u64 = cast_i{u64, load{x8,n/2}} @@ -49,10 +49,10 @@ def b_setBatch{sz, x:(*u64), n:(ux), v & sz==4} = { store{x8, n/2, cast_i{u8,w}} } -def b_setBatch{sz, x:(*u64), n:(ux), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}} -def b_setBatch{sz, x:(*u64), n:(ux), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}} -def b_setBatch{sz, x:(*u64), n:(ux), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}} -def b_setBatch{sz, x:(*u64), n:(ux), v & sz==64} = store{ x, n, cast_i{u64,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v if sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}} +def b_setBatch{sz, x:(*u64), n:(ux), v if sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v if sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}} +def b_setBatch{sz, x:(*u64), n:(ux), v if sz==64} = store{ x, n, cast_i{u64,v}} def spreadBits{T==[32]u8, a:(u32)} = { def idxs = iota{32} @@ -63,17 +63,17 @@ def spreadBits{T==[32]u8, a:(u32)} = { e == (d&e) } -def spreadBits{T==[16]u8, a:(u16) & hasarch{'AARCH64'}} = { +def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = { b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}} andnz{b, make{[16]u8, 1<<(iota{16}&7)}} } -def spreadBits{T==[16]u8, a:(u16) & hasarch{'X86_64'}} = { +def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = { b:= [16]u8~~[8]u16**a exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} (exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0 } -def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { +def spreadBits{T, a if vcount{T} <= elwidth{T} and quality{eltype{T}}=='u'} = { b:= make{T, 1< op0 - {(__le) & issigned{T}} => __gt - {(__ge) & issigned{T}} => __lt - {(__lt) & isunsigned{T}} => __ge - {(__gt) & isunsigned{T}} => __le - {(__ne) & isint{T}} => __eq + {_ if not hasarch{'X86_64'} or hasarch{'AVX512F'}} => op0 + {(__le) if issigned{T}} => __gt + {(__ge) if issigned{T}} => __lt + {(__lt) if isunsigned{T}} => __ge + {(__gt) if isunsigned{T}} => __le + {(__ne) if isint{T}} => __eq {_} => op0 } def mask = if (same{op0, op}) homMask else ({...x} => ~homMask{...x}) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 1a8ed9ad..624de946 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -4,7 +4,7 @@ include './vecfold' if_inline (hasarch{'SSE2'}) { fn sum_vec{T}(v:T) = vfold{+, fold{+, unpackQ{v, T**0}}} - def fold_addw{v:T & eltype{T}==i8} = sum_vec{T}(v) + def fold_addw{v:T if eltype{T}==i8} = sum_vec{T}(v) } def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index e61074b7..768b81c4 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -6,15 +6,15 @@ include './mask' include 'util/tup' -def rootty{T & isprim{T}} = T -def rootty{T & isvec{T}} = eltype{T} +def rootty{T if isprim{T}} = T +def rootty{T if isvec{T}} = eltype{T} def is_s{X} = issigned{rootty{X}} def is_u{X} = isunsigned{rootty{X}} def ty_sc{O, R} = R # keep floats as-is -def ty_sc{O, R & is_s{O} & is_u{R}} = ty_s{R} -def ty_sc{O, R & is_u{O} & is_s{R}} = ty_u{R} +def ty_sc{O, R if is_s{O} and is_u{R}} = ty_s{R} +def ty_sc{O, R if is_u{O} and is_s{R}} = ty_u{R} def bqn_or{a, b} = (a+b)-(a*b) @@ -22,18 +22,18 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - def arithChk1{F==__add, M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} def arithChk1{F==__sub, M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} -def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r} -def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r} +def arithChk1{F==__add, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r} +def arithChk1{F==__sub, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r} -def arithChk2{F, M, w:T, x:T & is_s{T} & (same{F,__add} | same{F,__sub})} = { +def arithChk2{F, M, w:T, x:T if is_s{T} and (same{F,__add} or same{F,__sub})} = { r:= F{w,x} tup{r, arithChk1{F, M, w, x, r}} } # × -def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i8==eltype{T} & hasarch{'X86_64'}} = { +def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and hasarch{'X86_64'}} = { def wp = unpackQ{w, T ~~ (T**0 > w)} def xp = unpackQ{x, T ~~ (T**0 > x)} def rp = each{__mul, wp, xp} @@ -46,12 +46,12 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i8==eltype{T} & hasarc tup{packQ{rp}, tup{'~andAllZero', RU~~tree_fold{|, bad}, RU**0xff80}} } } -def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i16==eltype{T} & hasarch{'X86_64'}} = { +def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i16==eltype{T} and hasarch{'X86_64'}} = { rl:= __mul{w,x} rh:= mulh{w,x} tup{rl, tup{'anyne', rh, rl>>15}} } -def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = { +def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} and hasarch{'X86_64'}} = { max:= re_el{f32, (ty_u{T})**0x4efffffe} def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} f32mul:= cf32{w} * cf32{x} @@ -68,7 +68,7 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasar # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M } -def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = { +def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and hasarch{'AARCH64'}} = { def r12 = mulw{w, x} rl:= packLo{r12} rh:= packHi{r12} @@ -82,24 +82,24 @@ def runner{u, R, F} = { def run{F, M, w, x} = { show{'todo', c, R, F, w, x}; emit{void,'__builtin_abort'}; w } - def run{F, M, w:T, x:T & c & R!=u32} = { + def run{F, M, w:T, x:T if c and R!=u32} = { arithChk2{F, M, w, x} } - def run{F, M, w, x & u} = tup{F{w, x}, tup{'none'}} # trivial base implementation + def run{F, M, w, x if u} = tup{F{w, x}, tup{'none'}} # trivial base implementation def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1)) - def run{F==__sub, M, w:VU, x:VU & c & is_u{VU}} = { # 'b'-'a' + def run{F==__sub, M, w:VU, x:VU if c and is_u{VU}} = { # 'b'-'a' def VS = ty_s{VU} run{F, M, VS~~toggleTop{w}, VS~~toggleTop{x}} } - def run{F, M, w:VU, x:VS & c & is_u{VU} & is_s{VS}} = { # 'a'+3, 'a'-3 + def run{F, M, w:VU, x:VS if c and is_u{VU} and is_s{VS}} = { # 'a'+3, 'a'-3 def {res, ok} = run{F, M, VS~~toggleTop{w}, x} tup{toggleTop{VU~~res}, ok} } - def run{F==__add, M, w:VS, x:VU & c & is_s{VS} & is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3 + def run{F==__add, M, w:VS, x:VU if c and is_s{VS} and is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3 - def run{F, M, w:VW, x:VX & c & R==u32 & (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1 + def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1 r:= F{ty_u{w}, ty_u{x}} tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} } @@ -111,7 +111,7 @@ def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} } def runChecks{type=='homAny', vals, M} = runChecks_any{homAny, vals} def runChecks{type=='topAny', vals, M} = runChecks_any{topAny, vals} def runChecks{type=='none', vals, M} = 0 -def runChecks{type=='~andAllZero', vals, M & ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}} +def runChecks{type=='~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}} def runChecks{type=='anyne', vals, M} = { def i{vals} = { def {_,xs,ys} = flip{vals} @@ -182,7 +182,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 def run = runner{(R==f64) | (mode>=2), R, F} def getW{v} = trunc{W, v} - def getW{v & W==f64} = interp_f64{v} + def getW{v if W==f64} = interp_f64{v} cw:= ty_sc{W, TY}**getW{w} def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index ade621f3..b80eb02d 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -39,9 +39,9 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { } else { # bitarr ≡ i8/i16/i32arr def T = [bulk]X def sh{c} = c << (width{X}-1) - def sh{c & X==u8} = T ~~ (re_el{u16,c}<<7) - def mask{x:X & hasarch{'X86_64'}} = topMask{x} - def mask{x:X & hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}} + def sh{c if X==u8} = T ~~ (re_el{u16,c}<<7) + def mask{x:X if hasarch{'X86_64'}} = topMask{x} + def mask{x:X if hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}} # TODO compare with doing the comparison in vector registers badBits:= T ** ~(X~~1) diff --git a/src/singeli/src/f64.singeli b/src/singeli/src/f64.singeli index 00dda936..62107337 100644 --- a/src/singeli/src/f64.singeli +++ b/src/singeli/src/f64.singeli @@ -6,10 +6,10 @@ def NaN = 0.0/0.0 def isNaN{x:(f64)} = x!=x def qNaN{x:(u64)} = (x<<1) == (cast{u64, 0x8ff8} << 49) -def ftrunc{T, x:(f64) & i8==T} = emit{i8, '', x} -def ftrunc{T, x:(f64) & i16==T} = emit{i16, '', x} -def ftrunc{T, x:(f64) & i32==T} = emit{i32, '', x} # maybe explicitly use _mm_cvtsd_si32? -def ftrunc{T, x:(f64) & i64==T} = emit{i64, '', x} +def ftrunc{T== i8, x:(f64)} = emit{T, '', x} +def ftrunc{T==i16, x:(f64)} = emit{T, '', x} +def ftrunc{T==i32, x:(f64)} = emit{T, '', x} # maybe explicitly use _mm_cvtsd_si32? +def ftrunc{T==i64, x:(f64)} = emit{T, '', x} def fext{x} = emit{f64, '', x} def interp_f64{x:(u64)} = emit{f64, 'interp_f64', x} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index bb132d4e..1f262be5 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -3,8 +3,8 @@ include './mask' def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}} def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}} -def mix{op, v:([4]f64) & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} } -def mix{op, v:([2]f64) & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032} +def mix{op, v:([4]f64) if hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} } +def mix{op, v:([2]f64) if hasarch{'X86_64'}} = opsh32{op}{v, 4b1032} def reduce_pairwise{op, plog, x:*T, len, init:T} = { # Pairwise combination to shorten dependency chains diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 3a2556fd..502fc1a2 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -29,7 +29,7 @@ def hash_val{x0:(u64)} = { } # CRC32 if_inline (hasarch{'SSE4.2'}) require{'x86intrin.h'} -def hash_val{x:(u32) & hasarch{'SSE4.2'}} = { +def hash_val{x:(u32) if hasarch{'SSE4.2'}} = { emit{u32, '_mm_crc32_u32', 0x973afb51, x} } diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 598bb439..11433d84 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -3,25 +3,25 @@ local def maskInit1{w} = { merge{(w/8-1)**255, (1<>3)^31 + 64*(n&7))} -mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} +mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n if width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))} +mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n if width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} mask256:*i64 = merge{4 ** -1, 4 ** 0} local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} # get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T} -def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256} -def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128} -def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64} +def maskOf{T,n if width{T}==256} = maskOfImpl{T, n, 256} +def maskOf{T,n if width{T}==128} = maskOfImpl{T, n, 128} +def maskOf{T,n if width{T}== 64} = maskOfImpl{T, n, 64} -def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y} -def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}} -def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y -def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 +def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~homAll{x==y} +def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = homAny{M{x!=y}} +def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y +def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} -def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M} -def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = { +def anynePositive{x:T, y:T, M if M{0}==0} = anyne{x, y, M} +def anynePositive{x:T, y:T, M if M{0}==1 and isvec{T}} = { def {n,m} = homMaskX{x==y} def E = tern{type{m}==u64, u64, u32} (promote{E,~m} << (width{E}-M{'count'}*n)) != 0 @@ -30,8 +30,8 @@ def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = { def maskNone{x} = x def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x} def maskAfter{n} = { - def mask{x:X & isvec{X}} = x & (X~~maskOf{X,n}) - def mask{x:X & anyInt{x}} = x & ((1< loadBatch {ptr, n, T }, ns} -def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} +def loadBatch {ptr:P, ns, T if istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns} +def storeBatch{ptr:P, ns, xs, M if istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} @@ -74,7 +74,7 @@ def hCast{T,p:*T} = p def hCast{T,p:(*void)} = *T~~p def mlExec{i, iter, vars0, bulk, M} = { - def vproc{p:P & isptr{P}} = p + def vproc{p:P if isptr{P}} = p def vproc{('m')} = tptr{{_}=>M, '!'} def vproc{{T,p:P}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index dc773fc8..70933be5 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -1,157 +1,157 @@ def nvec{T} = 0 -def nvec{T & isvec{T}} = (width{T}==64) | (width{T}==128) +def nvec{T if isvec{T}} = (width{T}==64) | (width{T}==128) def nvec{T,w} = 0 -def nvec{T,w & nvec{T}} = elwidth{T}==w +def nvec{T,w if nvec{T}} = elwidth{T}==w def nveci = genchk{nvec, isint} def nvecs = genchk{nvec, issigned} def nvecu = genchk{nvec, isunsigned} def nvecf = genchk{nvec, isfloat} -def reinterpret{T, v & same{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v } +def reinterpret{T, v if same{'pointer',typekind{T}} and ktup{v}} = { tmp:T=v } def nty{T} = { def q = quality{T} merge{if (q=='i') 's' else q, fmtnat{width{T}}} } -def nty{T & isvec{T}} = nty{eltype{T}} -def ntyp{S, ...S2, T & w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} -def ntyp{S, ...S2, T & w64{T}} = merge{S, ...S2, '_', nty{T}} +def nty{T if isvec{T}} = nty{eltype{T}} +def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} +def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}} def ntyp0{S, T} = merge{S, '_', nty{T}} -def addwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b} -def subwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b} -def mulwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b} -def mulwHi{a:T,b:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b} -def mulw {a:T,b:T & w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}} +def addwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b} +def subwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b} +def mulwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b} +def mulwHi{a:T,b:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b} +def mulw {a:T,b:T if w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}} -def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed -def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) -def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed +def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) +def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a< elwidth{X}*2} = widen{T, widen{el_s{T}, x}} -def widen{T, x:X & w64{X} & isfloat{eltype{T}}!=isfloat{eltype{X}} & elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}} -def widen{T, x:X & w128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} +def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x} +def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}} +def widen{T, x:X if w64{X} and isfloat{eltype{T}}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}} +def widen{T, x:X if w128{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}} -def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}} -def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x} -def narrow{T, x:X & w128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T}=vcount{T}} = { +def homMask{x:T if nvecu{T} and elwidth{T}>=vcount{T}} = { truncBits{vcount{T}, fold_add{x & make{T, 1<1 & elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} -def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = { +def homMask{...as if length{as}>1 and elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} +def homMask{a:T,b:T if vcount{T}*2<=elwidth{T}} = { def n = vcount{T} truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<>63) # repeat sign bit } } -fn clmul_scan_ne_any{& hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { +fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { def V = [2]u64 m := V**mark def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total @@ -101,10 +101,10 @@ fn clmul_scan_ne_any{& hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} } } -fn scan_neq{& hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { +fn scan_neq{if hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { clmul_scan_ne_any{}(*void~~x, *void~~r, init, nw, -(u64~~1)) } -fn scan_neq{& hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { +fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { def V = [8]u64 def sse{a} = make{[2]u64, a, 0} carry := sse{init} @@ -136,7 +136,7 @@ fn bcs{T}(x:*u64, r:*T, l:u64) : void = { c:T = 0 @for (r over i to l) { c+= cast_i{T, bitp_get{x,i}}; r = c } } -fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { +fn bcs{T if hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { def U = ty_u{T} def w = width{T} def vl= 256 / w @@ -202,7 +202,7 @@ def addChk{a:T, b:T} = { def bad = emit{u1, '__builtin_add_overflow', a, b, mem} tup{bad, load{mem}} } -def addChk{a:T, b:T & T==f64} = tup{0, a+b} +def addChk{a:T, b:T if T==f64} = tup{0, a+b} def widenFull{E, xs} = { merge{...each{{x:X} => { @@ -219,9 +219,9 @@ def widenFull{E, xs} = { }, xs}} } -def floor{x & knum{x}} = x - (x%1) -def maxabsval{T & issigned{T}} = -minvalue{T} -def maxsafeint{T & issigned{T}} = maxvalue{T} +def floor{x if knum{x}} = x - (x%1) +def maxabsval{T if issigned{T}} = -minvalue{T} +def maxsafeint{T if issigned{T}} = maxvalue{T} def maxsafeint{T==f64} = 1<<53 fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { diff --git a/src/singeli/src/scan_common.singeli b/src/singeli/src/scan_common.singeli index d4cf5ad1..0d861f2e 100644 --- a/src/singeli/src/scan_common.singeli +++ b/src/singeli/src/scan_common.singeli @@ -1,9 +1,9 @@ # Used by scan.singeli and bins.singeli def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}} -def sel8{v:V, t & w256{V} & istup{t} & length{t}==16} = sel8{v, merge{t,t}} +def sel8{v:V, t if w256{V} and istup{t} and length{t}==16} = sel8{v, merge{t,t}} -def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} +def shuf{T, v, n if istup{n}} = shuf{T, v, base{4,n}} local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} } local def rev{up,t} = if (up) t else rev{t} @@ -19,14 +19,14 @@ def spread{a:VT, ...up} = { } # Set all elements with the last element of the input -def toLast{n:VT, up & hasarch{'X86_64'} & w128{VT}} = { +def toLast{n:VT, up if hasarch{'X86_64'} and w128{VT}} = { def l{v, w} = l{zip{up,v}, 2*w} - def l{v, w & hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)} + def l{v, w if hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)} def l{v, w==32} = shuf{[4]i32, v, 4**(up*3)} def l{v, w==64} = shuf{[4]i32, v, (2*up) + tup{0,1,0,1}} l{n, elwidth{VT}} } -def toLast{n:VT, up & hasarch{'AVX2'} & w256{VT}} = { +def toLast{n:VT, up if hasarch{'AVX2'} and w256{VT}} = { if (elwidth{VT}<=32) sel{[8]i32, spread{n,up}, [8]i32**(up*7)} else shuf{[4]u64, n, 4**(up*3)} } @@ -51,17 +51,17 @@ def make_scan_idem{T, op, up} = { def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}} (if (up) shl else shr){[16]u8, v, k/8} | id } - def shb{v, k & hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} - def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}} - def shb{v, k & k==128 & hasarch{'AVX2'}} = { + def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} + def shb{v, k if k>=32} = shuf{[4]u32, v, shift{k/32,4}} + def shb{v, k if k==128 and hasarch{'AVX2'}} = { # After lanewise scan, broadcast end of lane 0 to entire lane 1 sel{[8]i32, spread{v,up}, make{[8]i32, rev{up,3*(30} @@ -600,7 +600,7 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it def try_vec_memb{..._} = {} def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done - & hasarch{'SSE4.2'} & T==u32} = { + if hasarch{'SSE4.2'} and T==u32} = { # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) # i-h>>sh = i+((1<>sh = (((i+1)<>sh # We maintain io = (i+1)<0} = { + def bs{b, c, x if length{b}>0} = { select{b,0}{each{bs{slice{b,1}, c, .}, x}} } diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index b22c3b72..2f6cdc9b 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -7,8 +7,8 @@ if_inline (hasarch{'X86_64'}) { include './mask' include 'util/tup' -def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones -def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} +def popcRand{x:T if isint{T} and width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones +def popcRand{x:T if isint{T} and width{T}<=32} = emit{u8, 'rand_popc64', x} # Table from l bits to w-bit indices, shifted left by s, and G applied afterwards def maketab{l,w,s,G} = { @@ -64,7 +64,7 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { } else { if (has_simd) { def bufw = bufn * tw - def vc = tern{hasarch{'X86_64'} & (bufw==128), 128, arch_defvw} / tw; + def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw; def R = [vc]T @unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack @@ -118,7 +118,7 @@ def topper{T, U, k, x} = { itab_4_16:*u64 = maketab{4,16} # 16 elts, 128B def thresh{c==0, T==i8 } = 32 def thresh{c==0, T==i16} = 16 -fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +fn slash{c==0, T if T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def n = 64/tw def tab = if (tw==8) itab else itab_4_16 @@ -140,9 +140,9 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { # i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'} -def thresh{c==0, T==i16 & simd128{}} = 32 -def thresh{c==0, T==i32 & simd128{}} = 16 -fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==0, T==i16 if simd128{}} = 32 +def thresh{c==0, T==i32 if simd128{}} = 16 +fn slash{c==0, T if simd128{} and i16<=T and T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def I = [16]i8 j := I**(if (T==i16) 0 else cast_i{i8,x}) def {top, inctop} = topper{T, I, 8, x} @@ -161,9 +161,9 @@ fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, # i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'} -def thresh{c==1, T==i8 & shufb128{}} = 64 -def thresh{c==1, T==i16 & shufb128{}} = 32 -fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==1, T==i8 if shufb128{}} = 64 +def thresh{c==1, T==i16 if shufb128{}} = 32 +fn slash{c==1, T if T<=i16 and shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def V = [16]i8 @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { ind := load{itab, w} @@ -179,8 +179,8 @@ fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u6 # i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B -def thresh{c==1, T==i32 & shufb128{}} = 8 -fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==1, T==i32 if shufb128{}} = 8 +fn slash{c==1, T==i32 if shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def V = [16]i8 expander := make{V, iota{16}>>2} trail := make{V, tail{2,iota{16}}} @@ -202,9 +202,9 @@ fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : # i32 & i64 w/x & x+/w; 256 bits/step, 8 elts/iter; [8]i32 shuffle i64tab:*u64 = maketab{4,16,1,{x}=>(1+x)*0x100 + x} # 16 elts, 128B -def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 -def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 -fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T==i32 if hasarch{'AVX2'}} = 32 +def thresh{c, T==i64 if hasarch{'AVX2'}} = 8 +fn slash{c, T if hasarch{'AVX2'} and T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{tup{., ... 3**128}, iota{8}>>lb{tw/32}}}} @@ -235,11 +235,11 @@ fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: } # everything; 512 bits/iter; AVX-512 compress -def thresh{c, T==i8 & hasarch{'AVX512VBMI2'}} = 256 -def thresh{c, T==i16 & hasarch{'AVX512VBMI2'}} = 128 -def thresh{c, T==i32 & hasarch{'AVX512F'}} = 64 -def thresh{c, T==i64 & hasarch{'AVX512F'}} = 16 -fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T==i8 if hasarch{'AVX512VBMI2'}} = 256 +def thresh{c, T==i16 if hasarch{'AVX512VBMI2'}} = 128 +def thresh{c, T==i32 if hasarch{'AVX512F'}} = 64 +def thresh{c, T==i64 if hasarch{'AVX512F'}} = 16 +fn slash{c, T if hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def f = fmtnat def wt = width{T} def vl = 512/wt @@ -283,7 +283,7 @@ def pext_popc{x:T, m:T} = { # - z tells how many bits in the group are NOT used # - x contains the bits, with z zeros above def build{k==1} = tup{x&m, ~m} - def build{k & k > 1} = { + def build{k if k > 1} = { def h = k>>1 # Increase size from h to k {x,z} := build{h} def low_s = lowbits{w,k} # Low bit in each new group @@ -326,12 +326,12 @@ def pext_popc{x:T, m:T} = { pe := fold{|, x&s0, each{gr, g*slice{iota{k/g},1}}} tup{pe, o>>(k-g)} } - def build{k==32 & hasarch{'AVX2'} & isvec{T}} = { + def build{k==32 if hasarch{'AVX2'} and isvec{T}} = { def S = re_el{ty_u{k}, T} def c{T,vs} = each{{v}=>T~~v, vs} c{T, multi_shift{...c{S, build{8}}, 8, k, {s}=>S**s}} } - def build{k & ~isvec{T} & k > 8} = { + def build{k if not isvec{T} and k > 8} = { multi_shift{...build{8}, 8, k, {s}=>s} } # Final result @@ -339,9 +339,9 @@ def pext_popc{x:T, m:T} = { tup{pe, scal{w} - z} } -def pext_width {& hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 -def thresh_bool{& hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32 -def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { +def pext_width {if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 +def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32 +def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = { def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} m := m0 x := x0 & m @@ -359,9 +359,9 @@ def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { tup{x, @collect (j to 2) popc{extract{m0,j}}} } -def pext_width {& fast_BMI2{}} = 1 -def thresh_bool{& fast_BMI2{}} = 512 -def pext_popc{x:T, m:T & fast_BMI2{} & T==u64} = tup{pext{x, m}, popc{m}} +def pext_width {if fast_BMI2{}} = 1 +def thresh_bool{if fast_BMI2{}} = 512 +def pext_popc{x:T, m:T if fast_BMI2{} and T==u64} = tup{pext{x, m}, popc{m}} fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { cw:u64 = 0; # current word @@ -375,7 +375,7 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { } ro = ro2%64 } - def extract{t, i & istup{t}} = select{t,i} + def extract{t, i if istup{t}} = select{t,i} def v = pext_width{} if (v > 1) { def V = [v]u64 diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index a794c995..2d16a10f 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -7,24 +7,24 @@ include './vecfold' def preserve_negative_zero = 0 # SSE2 versions avoid any 64-bit integer comparsions -def anySNaN{M, x:T & eltype{T}==u64} = { +def anySNaN{M, x:T if eltype{T}==u64} = { homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}} } -def anySNaN{M, x:T & T==[2]u64 & hasarch{'X86_64'} & ~hasarch{'SSE4.2'}} = { +def anySNaN{M, x:T if T==[2]u64 and hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}} } -def anyNonChar{M, x:T & isvec{T} & eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} -def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = { +def anyNonChar{M, x:T if isvec{T} and eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} +def anyNonChar{M, x:T if isvec{T} and hasarch{'X86_64'}} = { def H = re_el{u32, T} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} topAny{M{T~~ne}} } -def cvtNarrow{T, x:X & width{T}==elwidth{X}} = cvt{T, x} -def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x} -def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x} -def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x} +def cvtNarrow{T, x:X if width{T}==elwidth{X}} = cvt{T, x} +def cvtNarrow{T, x:X if width{T}< elwidth{X}} = narrow{T, x} +def cvtWiden{T, x:X if elwidth{T}==elwidth{X}} = cvt{eltype{T}, x} +def cvtWiden{T, x:X if elwidth{T}> elwidth{X}} = widen{T, x} fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { assert{len>0} @@ -36,7 +36,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { # fold with either Max or Bitwise Or, truncating/zero-extending to TE def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}} - def foldTotal{TE, x:T & hasarch{'AARCH64'}} = { + def foldTotal{TE, x:T if hasarch{'AARCH64'}} = { if (elwidth{T}==64) { if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} else vfold{max, narrow{TE, x}} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index a190880c..cc1a6309 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -1,32 +1,32 @@ ### SSSE3 ### -def sel{L, x:T, i:I & hasarch{'SSSE3'} & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} -def vshl{a:T, b:T, n & hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)} +def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} +def vshl{a:T, b:T, n if hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)} ### SSE4.1 ### -def packs{a:T,b:T & hasarch{'SSE4.1'} & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} -def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} +def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} +def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} # conversion -def widen{T==[8]u16, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} -def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} -def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} -def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} -def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} -def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} -def widen{T==[2]f64, x:X & hasarch{'SSE4.1'} & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T==[8]u16, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} +def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} +def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} +def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} +def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} +def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} +def widen{T==[2]f64, x:X if hasarch{'SSE4.1'} and w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} -def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} +def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} +def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} # mask stuff -def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} -def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} +def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} # assumes all bits are the same in each mask item -def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m} -def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m} +def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m} +def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 8e82ad46..eff66f82 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -1,37 +1,37 @@ # compact casting for the annoying intrinsic type system -def v2i{x:T & w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x -def v2f{x:T & w128{T}} = [4]f32 ~~ x -def v2d{x:T & w128{T}} = [2]f64 ~~ x +def v2i{x:T if w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x +def v2f{x:T if w128{T}} = [4]f32 ~~ x +def v2d{x:T if w128{T}} = [2]f64 ~~ x # load & store -def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr} -def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr} -def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = load{ptr} +def loadLow{ptr:P, w if w128{eltype{P}} and w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr} +def loadLow{ptr:P, w if w128{eltype{P}} and w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr} +def loadLow{ptr:P, w if w128{eltype{P}} and w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr} +def loadLow{ptr:P, w if w128{eltype{P}} and w==128} = load{ptr} -def storeLow{ptr:P, w, x:T & w128{T} & w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T & w128{T} & w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x} +def storeLow{ptr:P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} +def storeLow{ptr:P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} +def storeLow{ptr:P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} +def storeLow{ptr:P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x} # float comparison -def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} -def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} +def unord{a:T,b:T if T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} +def unord{a:T,b:T if T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} # shift -def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} -def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} +def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} +def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} # integer arith -def mulh{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} -def mulh{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} -def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments -def __mul{a:T,b:T & [4]i32==T} = { +def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} +def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} +def mul32{a:T,b:T if [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments +def __mul{a:T,b:T if [4]i32==T} = { def mu{x, y} = [4]i32 ~~ mul32{[2]u64~~x, [2]u64~~y} def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs} lo:= mu{a, b} @@ -44,55 +44,55 @@ def rsqrtE{a:([4]f32)} = emit{[4]f32, '_mm_rsqrt_ps', a} def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a} # mask stuff -def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0} +def andAllZero{x:T, y:T if w128i{T}} = homAll{(x & y) == T**0} -def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} -def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} -def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} -def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} -def homMask{x:T & w128{T}} = topMask{x} -def homMaskX{a:T & elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}} -def homMask{a:T, b:T & w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}} +def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} +def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} +def topMask{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} +def topMask{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} +def homMask{x:T if w128{T}} = topMask{x} +def homMaskX{a:T if elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}} +def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}} -def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0 -def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff +def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0 +def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff -def topAny{x:T & w128i{T}} = topMask{x} != 0 -def topAll{x:T & w128i{T}} = topMask{x} == (1<elwidth{X}} = { +def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = { def s{v} = s{mzipLo{v, v}} - def s{v:V & V==T} = v + def s{v:V if V==T} = v s{x} >> (elwidth{T} - elwidth{X}) } -def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} -def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} +def widen{T==[2]f64, x:X if w128s{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T==[2]f64, x:X if X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} +def widen{T==[2]f64, x:X if X==[4]f32} = emit{T, '_mm_cvtps_pd', x} def narrow{T==i16, x:([4]i32)} = packs{x,x} def narrow{T==i8, x:([8]i16)} = packs{x,x} @@ -104,5 +104,5 @@ def narrow{T==u8, x:([2]u64)} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{f{x}}}} def narrow{T==u16, x:([2]u64)} = shuf16Lo{[8]u16~~shuf{[4]i32, x, 4b3320}, 4b3320} def narrow{T==u32, x:([2]u64)} = [4]u32~~shuf{[4]i32, x, 4b3320} -def narrow{T, x:X & w128f{X,64} & T shuf{[4]i64, v, 4b3120}, x} # Square kernel where width is a full vector -def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x} +def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x} def load2{a:T, b:T} = pair{load{a}, load{b}} -def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { +def store2{a:T, b:T, v:T2 if w128i{eltype{T}} and w256{T2}} = { each{{p:P, i} => store{p, 0, eltype{P}~~half{v,i}}, tup{a,b}, iota{2}} } -def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} -def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} -def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} -def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} +def load_k {VT, src, l, w if w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} +def store_k{VT, dst, x, l, h if w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} +def load_k {VT, src, l, w if w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} +def store_k{VT, dst, x, l, h if w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} # Transpose kernel of size kw,kh in size w,h array def kernel{src:P, dst:P, kw, kh, w, h} = { diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index f0aadea2..92a57f45 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -1,6 +1,6 @@ # Fold associative/commutative operation across a register -def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { +def vfold{F, x:T if w128{T} and hasarch{'X86_64'}} = { c:= x def EW = elwidth{T} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} @@ -9,4 +9,4 @@ def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} } else extract{c, 0} } -def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}} +def vfold{F, x:T if w256{T} and hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}