From 8542ba071dd49335443e6bbc44a15306dd4d73d6 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 7 Jun 2023 17:39:38 +0300 Subject: [PATCH] homMaskX, merging multi-arg homMask --- src/singeli/src/base.singeli | 21 +++++++++++++++++++++ src/singeli/src/bitops.singeli | 4 ---- src/singeli/src/cmp.singeli | 6 ++---- src/singeli/src/mask.singeli | 7 +++++-- src/singeli/src/neon.singeli | 33 ++++++++++++++++++++++++++++++--- src/singeli/src/scan.singeli | 2 +- src/singeli/src/search.singeli | 2 +- src/singeli/src/sse2.singeli | 1 + 8 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 82a5fd82..f136a511 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -24,6 +24,11 @@ def ctz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_ctz', x} def clz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_clzll', x} def clz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_clz', x} +def truncBits{n, v & n<=8} = cast_i{u8, v} +def truncBits{n, v & n==16} = cast_i{u16, v} +def truncBits{n, v & n==32} = cast_i{u32, v} +def truncBits{n, v & n==64} = cast_i{u64, v} + # base-2 log of a constant power of two def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 def lb{n==1} = 0 @@ -44,6 +49,13 @@ def unreachable{} = emit{void, 'si_unreachable'} def assert{x:u1} = { if (not x) emit{void, 'si_unreachable'} } # various checks +def oneType{vs} = { + def T = type{tupsel{0,vs}} + each{{x:C} => assert{C==T}, vs} + T +} +def oneType{vs & tuplen{vs}==0} = 1 + def isreg = kreg def isconst = kcon def istype = ktyp @@ -187,6 +199,15 @@ def cvt{T, x:X & T==eltype{X}} = x def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +def homMask{...vs & tuplen{vs}>1} = { + def n = tuplen{vs} + def T = oneType{vs} + def RT = ty_u{max{8,vcount{T}*n}} + tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}} +} +def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times +def ctzX{t} = { def {n,v}=t; ctz{v}/n } # ctz for a result of homMaskX + def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count) def TU = ty_u{TS} (TU~~(x-TS**start)) < TU**count diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index bc9d18d8..bc08b317 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -82,10 +82,6 @@ def loadBatchBit{T, x:*u64, n:(Size)} = { # vector with type T with each element spreadBits{T, b_getBatchLo{vcount{T}, x, n}} } -def truncBits{n, v & n<=8} = cast_i{u8, v} -def truncBits{n, v & n==16} = cast_i{u16, v} -def truncBits{n, v & n==32} = cast_i{u32, v} -def truncBits{n, v & n==64} = cast_i{u64, v} def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 4e829c7c..e497fd67 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -78,9 +78,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = { while (ri < am) { r:u64 = 0 if (hasarch{'AARCH64'}) { - def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}} - def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}} - r = promote{u64, homMask{step{0, unr}}} + r = promote{u64, homMask{...each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}} } else { @unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) } @@ -130,7 +128,7 @@ def table{aa, F, w, G_bit, G_vec} = { ...each{{E}=>{ def E2 = (if (aa and isunsigned{E} and ((match{F,__eq} | match{F,__ne}) | (E==u32))) {ty_s{E}} else E) def bulk = w/width{E2} - G_vec{[bulk]E2, max{1, 8/bulk}} + G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, 8/bulk}} }, tup{i8, i16, i32, f64, u8, u16, u32}} } } diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index d4c03b83..d8be8521 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -21,8 +21,11 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M} -def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~homMask{ x==y }} << (32-M{'count'} )) != 0 -def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~homMask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 +def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = { + def {n,m} = homMaskX{x==y} + def E = tern{type{m}==u64, u64, u32} + (promote{E,~m} << (width{E}-M{'count'}*n)) != 0 +} def maskNone{x} = x def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 8e4e7804..440a45e1 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -45,9 +45,13 @@ def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b} def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} +def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b} + +def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s0 & s8} = { def H=v_half{ty_half{T}}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed +def shrm{a:T, d:T, s & w128u{T}} = emit{T, ntyp{'vsri', '_n', T}, a, d, s} # (a>>s) | (d & newZeroesIn(a>>s)) -def __shl{a:T,b & nveci{T} & knum{b} & b>0 & b0 & b=vcount{T}} = { - cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1<1 & elwidth{type{tupsel{0,as}}}>=32} = homMask{...each{{i}=>narrowPair{tupsel{i*2,as},tupsel{i*2+1,as}}, iota{tuplen{as}/2}}} +def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = { + def n = vcount{T} + truncBits{n*2, fold_add{shrm{b,a,elwidth{T}-n} & make{T, (1< assert{T==type{cv}}, cr} + assert{type{cv} == oneType{cr}} assert{vcount{type{cv}} * tuplen{cr} == bulk} each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 8c11e7cc..2b063827 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -35,7 +35,7 @@ def search{E, x, n:u64, OP} = { def VT = [bulk]E def end = makeBranch{ tup{u64, ty_u{VT}}, - {i,c} => return{i*bulk + ctzi{homMask{c}}} + {i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}} } muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 13ee3289..d9d0ac1c 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -170,6 +170,7 @@ def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} def homMask{x:T & w128{T}} = topMask{x} +def homMaskX{a:T & elwidth{T}==16} = tup{2, homMask{to_el{u8,a}}} def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0 def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff