define ux in Singeli, replacing Size; return ux from popc/clz/ctz

This commit is contained in:
dzaima 2023-07-11 18:02:42 +03:00
parent f416d58579
commit 0d7de8b190
6 changed files with 47 additions and 49 deletions

View File

@ -2,7 +2,7 @@ include 'skin/c'
include 'arch/c'
include 'util/kind'
def Size = u64
def ux = u64
config usz = u32
def same = is
@ -38,12 +38,12 @@ def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} }
# more arith
def cdiv{a,b} = (a+b-1)/b # ceiling divide
def cdiv{a,b & knum{a} & knum{b}} = ((a+b-1)/b)>>0
def popc{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_popcountll', x}
def popc{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_popcount', x}
def ctz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_ctzll', x}
def ctz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_ctz', x}
def clz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_clzll', x}
def clz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_clz', x}
def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x}
def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x}
def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x}
def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x}
def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x}
def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x}
# count-leading-zeros complement, less type-dependent
def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x}
def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x}
@ -244,14 +244,14 @@ def unroll{vars,begin,end,iter & knum{begin} & knum{end}} = {
f{begin,end}
}
def for{vars,begin,end,iter} = {
i:u64 = begin
i:ux = begin
while (i < end) {
iter{i, vars}
++i
}
}
def forNZ{vars,begin,end,iter} = {
i:u64 = begin
i:ux = begin
assert{i!=end}
while (i < end) {
iter{i, vars}
@ -259,14 +259,14 @@ def forNZ{vars,begin,end,iter} = {
}
}
def for_backwards{vars,begin,end,iter} = {
i:u64 = end
i:ux = end
while (i > begin) {
--i
iter{i, vars}
}
}
def forUnroll{exp,unr}{vars,begin,end,iter} = {
i:u64 = begin
i:ux = begin
while ((i+unr) <= end) {
iter{each{{j}=>i+j, iota{unr}}, vars}
i+= unr

View File

@ -162,7 +162,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = {
# Convert to bit table
def no_bittab = makelabel{}
def nb = 256/vl
nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques
nu:u8 = 0; def addu{b} = { nu+=cast_i{u8,popc{b}}; b } # Number of uniques
vb := U~~make{[nb](ty_u{vl}),
@collect (t in *V~~t0 over nb) addu{homMask{t > V**0}}
}
@ -235,7 +235,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def lt = if (up) <; else >
# Number of steps
log := ceil_log2{wn+1}
gap := 1<<log - cast_i{u8, wn}
gap := 1<<log - wn
# Fill with minimum value at the beginning
def pre = (if (up) minvalue else maxvalue){T}
wg := *V~~(w-gap)
@ -263,7 +263,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef'
def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef'
# Offset at end
off := U~~V**i8~~(gap - 1)
off := U~~V**cast_i{i8, gap-1}
# Midpoint bits for each step
def lowbits = bb{copy{isub,isub}}
bits := each{{j} => U**(lowbits << j), iota{lstep}}
@ -348,7 +348,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = {
# initializing the table (constant, much higher for i16)
} else if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) {
lookup{0}
} else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/promote{u64,ceil_log2{wn}+2})) {
} else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/(ceil_log2{wn}+2))) {
lookup{0}
} else {
bin_search_branchless{...param, rty}

View File

@ -1,26 +1,26 @@
def ones{T, n} = (cast{T,1}<<cast{T,n}) - 1
def b_get{x:*u64, n:(Size)} = {
def b_get{x:*u64, n:(ux)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatchLo{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
def b_getBatchLo{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
def b_getBatchLo{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatchLo{sz, x:*u64, n:(ux) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
def b_getBatchLo{sz, x:*u64, n:(ux) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
def b_getBatchLo{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = b_getBatchLo{sz, x, n} & 3
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = b_getBatchLo{sz, x, n} & 15
def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatch{sz, x:*u64, n:(ux) & sz==2} = b_getBatchLo{sz, x, n} & 3
def b_getBatch{sz, x:*u64, n:(ux) & sz==4} = b_getBatchLo{sz, x, n} & 15
def b_getBatch{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_set{x:*u64, n:(Size), v:u1} = {
def b_set{x:*u64, n:(ux), v:u1} = {
m:u64 = cast{u64,1}<<(n&63)
p:u64 = load{x,n>>6}
if (v) store{x,n>>6,p | m}
else store{x,n>>6,p & ~m}
}
def b_setBatch{sz, x:*u64, n:(Size), v} = {
def b_setBatch{sz, x:*u64, n:(ux), v} = {
vc:u64 = promote{u64,v}
am:u64 = 64/sz
w:u64 = load{x,n/am}
@ -30,7 +30,7 @@ def b_setBatch{sz, x:*u64, n:(Size), v} = {
store{x, n/am, w}
}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==4} = {
def b_setBatch{sz, x:*u64, n:(ux), v & sz==4} = {
x8:= *u8 ~~ x
#w:u64 = cast_i{u64, load{x8,n/2}}
@ -49,10 +49,10 @@ def b_setBatch{sz, x:*u64, n:(Size), v & sz==4} = {
store{x8, n/2, cast_i{u8,w}}
}
def b_setBatch{sz, x:*u64, n:(Size), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==64} = store{ x, n, cast_i{u64,v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==64} = store{ x, n, cast_i{u64,v}}
def spreadBits{T==[32]u8, a:u32} = {
def idxs = iota{32}
@ -78,7 +78,7 @@ def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
b == (b & T ~~ re_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}
def loadBatchBit{T, x:*u64, n:(Size)} = { # vector with type T with each element being either all 0s or 1s
def loadBatchBit{T, x:*u64, n:(ux)} = { # vector with type T with each element being either all 0s or 1s
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
}

View File

@ -13,11 +13,11 @@ if (hasarch{'AVX2'}) {
include './bitops'
def fillbits{dst:*u64, len:(Size), v } = { emit{void, 'fillBits', dst, len, v }; return{}; }
def fillbits{dst:*u64, len:(Size), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
def fillbits{dst:*u64, len:(ux), v } = { emit{void, 'fillBits', dst, len, v }; return{}; }
def fillbits{dst:*u64, len:(ux), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
def cmp_err{x} = { emit{void, 'cmp_err'}; return{}; }
fn cmpIX(dst:*u64, len:Size, x:u64, v:u1) : void = {
fn cmpIX(dst:*u64, len:ux, x:u64, v:u1) : void = {
nan:u1 = q_f64{x}
if (~(nan | q_chr{x})) cmp_err{x}
fillbits{dst, len, v&~nan, x}
@ -69,11 +69,11 @@ def pathAS{dst, len, T, op, x & isunsigned{T}} = {
def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
xi:Size = 0
ri:Size = 0
def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(ux)} = {
xi:ux = 0
ri:ux = 0
def bulk = vcount{VT}*unr
am:Size = cdiv{len,bulk}
am:ux = cdiv{len,bulk}
assert{am>0}
while (ri < am) {
r:u64 = 0
@ -87,26 +87,26 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
ri+= 1
}
}
fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:ux) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= *VT~~xr; xs:= *eltype{VT}~~xr
any2bit{VT, unr, op, load{ws,.}, load{wv,.}, load{xs,.}, load{xv,.}, dst, len}
}
fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:Size) : void = {
fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = {
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
xv:= VT**pathAS{dst, len, eltype{VT}, op, x}
any2bit{VT, unr, op, load{ws,.}, load{wv,.}, {i}=>x, {i}=>xv, dst, len}
}
fn bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:Size) : void = {
fn bitAA{bitop}(dst:*u64, wr:*void, xr:*void, len:ux) : void = {
@forNZ (dst, w in *u64~~wr, x in *u64~~xr over cdiv{len,64}) dst = bitop{w,x}
}
fn not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
fn cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
fn not(dst:*u64, x:*u64, len:ux) : void = { am:=cdiv{len,64}; emit{void, 'bit_negatePtr', dst, x, am} }
fn cpy(dst:*u64, x:*u64, len:ux) : void = { am:=cdiv{len,64}; emit{void, 'memcpy', dst, x, am*8} }
fn bitAS{op}(dst:*u64, wr:*void, x:u64, len:Size) : void = { # show{'bitAS'}
fn bitAS{op}(dst:*u64, wr:*void, x:u64, len:ux) : void = { # show{'bitAS'}
xf:f64 = interp_f64{x}
r0:u1 = op{0,xf}
r1:u1 = op{1,xf}

View File

@ -13,8 +13,6 @@ if (hasarch{'AVX2'}) {
include './mask'
include './vecfold'
def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index
def findFirst{C, M, F, ...v1} = {
def exit = makelabel{}
def args = undef{M{...each{tupsel{0, .}, v1}}}
@ -213,7 +211,7 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = {
} else {
# Add values to the table and filter m
if (rbit) rv = m
im:= i0 + ctzi{m}
im:= i0 + ctz{m}
xi:= load{x, im}
settab1{xi, im}
if ((m&(m-1)) != 0) { # More bits than one
@ -225,7 +223,7 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = {
if (rbit) rv&= e | -m # Don't remove first bit
m&= e
while (m != 0) {
im:= i0 + ctzi{m}
im:= i0 + ctz{m}
new:= settab{uk, load{x, im}, im}
m1:= m-1; m&= m1 # Clear low bit
if (rbit) rv&= m1 | new # Clear if not new

View File

@ -35,7 +35,7 @@ def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
assert{len>0}
def bulk = vw / width{X}
def XV = [bulk]X
@ -79,7 +79,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
}
0
} else { # i16, i32, f64
def case_B = makeOptBranch{B, tup{Size}, {iCont} => {
def case_B = makeOptBranch{B, tup{ux}, {iCont} => {
def XU = [bulk]u64
@maskedLoop{bulk, iCont}(xv in tup{XV,xp}, M in 'm' over len) {
v:= XU ~~ xv