From a5c6e3271c556892c27010ff532b2e809612e063 Mon Sep 17 00:00:00 2001 From: dzaima Date: Mon, 28 Apr 2025 17:23:12 +0300 Subject: [PATCH] use tail{...} much more --- src/singeli/src/avx.singeli | 2 +- src/singeli/src/avx2.singeli | 2 +- src/singeli/src/base.singeli | 6 ++++-- src/singeli/src/bitalign.singeli | 6 +++--- src/singeli/src/bitops.singeli | 2 +- src/singeli/src/count.singeli | 4 ++-- src/singeli/src/fold.singeli | 14 +++++++------- src/singeli/src/hashtab.singeli | 2 +- src/singeli/src/mask.singeli | 4 ++-- src/singeli/src/replicate.singeli | 2 +- src/singeli/src/scan.singeli | 4 ++-- src/singeli/src/search.singeli | 2 +- src/singeli/src/select.singeli | 2 +- src/singeli/src/slash.singeli | 12 ++++++------ src/singeli/src/sse.singeli | 2 +- 15 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index f7841e89..217bd92f 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -26,4 +26,4 @@ def any_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} != 0 def all_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} == 0xff def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = top_to_int{x} != 0 -def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = top_to_int{x} == (1<=32} = top_to_int{x} == tail{k} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 11c5740d..6f83a0cf 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -25,7 +25,7 @@ def top_to_int{x:T if w256{T, 16}} = { def any_hom{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} def all_hom{x:T if w256i{T}} = hom_to_int{[32]u8 ~~ x} == 0xffff_ffff def any_top{x:T if w256i{T}} = top_to_int{x} != 0 -def all_top{x:T=[k]_ if w256i{T}} = top_to_int{x} == (1<v, range{n}} # type stats def minvalue{T if isunsigned{T}} = 0 -def maxvalue{T if isunsigned{T}} = (1<> n) << n # zero out n least significant bits def tail{n,x} = x & tail{n} # get the n least significant bits def bit {k,x} = x & (1<> idx) & 1) != 0 diff --git a/src/singeli/src/bitalign.singeli b/src/singeli/src/bitalign.singeli index f1c0ace9..9abff4b6 100644 --- a/src/singeli/src/bitalign.singeli +++ b/src/singeli/src/bitalign.singeli @@ -148,7 +148,7 @@ def bitalign{{2,8,s}, 8, G if hasarch{'AVX512VBMI'}} = G{s, {a:V=[k](u8)} => { def cyc = make{V, cycle{k, range{8}}} def b = sel{V, a, cyc + muls{make{V, replicate{8, range{k/8}}}}} def c = multishift{re_el{u64,b}, muls{cyc}} - c & V**cast_i{u8, tail{s}} + c & V**tail{u8,s} }} @@ -162,7 +162,7 @@ def bitalign{{2,8,s}, 8, G if hasarch{'AARCH64'}} = G{s, {a:V==[16]u8} => { def r0 = sel{[16]u8, a, shuf1} << shift1 def r1 = sel{[16]u8, a, shuf2} << shift2 - (r0 | r1) & V**cast_i{u8, tail{s}} + (r0 | r1) & V**tail{u8,s} }} def bitalign{8, {2,8,d}, G if hasarch{'AARCH64'}} = { @@ -200,7 +200,7 @@ def bitalign{8, {2,8,d}, G if hasarch{'AARCH64'}} = { def run{do_blend}{a:V==[16]u8} = { def shuf1 = shuf0 + V**1 def shift1 = shift0 + [16]i8**cast_i{i8,d} - def b = a & V**cast_i{u8, tail{d}} + def b = a & V**tail{u8,d} def r0 = sel{[16]u8, b, shuf0} << shift0 def r1 = sel{[16]u8, b, shuf1} << shift1 def r01 = r0 | r1 diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 748fb2a4..a8e71eeb 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -51,7 +51,7 @@ def store_bits{sz, x:(*u64), n:(ux), v} = match (sz) { am:u64 = 64/sz w:u64 = load{x,n/am} sh:u64 = (n&(am-1)) * sz - w&= ~(u64~~tail{sz} << sh) + w&= ~(tail{u64,sz} << sh) w|= (vc< V**bot} - if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1 + if (jv == n/vl) o &= tail{type{o}, n%vl} while (o > 0) { jv := jv*vl + cast_i{usz, ctz{o}} store{tab, jv, load{tab, jv} & bot} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 3b4bba5d..270ee768 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -105,7 +105,7 @@ def extract_column_pow2{T, x0, r0, nv, k} = { def f = tree_fold{unzip0{32}, .} def proc{hx} = { ri := D~~f{hx} - top := D**(1<<15); m := D**(1<<16 - 1) + top := D**(1<<15); m := D**tail{16} (ri & m) | (D~~(ri&top == top) &~ m) } r = V~~packs128{each{proc, split{k/2, xs}}} @@ -128,7 +128,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { e := p2 + promote{ux,el} # Absorb into element size for most computation l8 := cast_i{u8, l} li := cast_i{u8, l + 2 * ((l-1) + (l&2))} # Inverse mod vl - elo:= V**(u8~~1<>p2) dz := (il & l) | (h &~ il)>>(4-e) # low, high->middle dz |= (il &~ (l | h))<high @@ -222,7 +222,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = { shuf{[8]u32, ., cr} } # Run, writing every 1< { def ra = add_res{r} if ((plo &~ i) == 0) write{cross{uz_lane{ra}}} @@ -274,7 +274,7 @@ def fold_rows_bit_lt64{ # repeated shift-or-mask def step{x, {m, sh}} = { def a=x&m; a|a>>sh } def ss = 1< loop_T{u32, {x} => extract{op{x, x>>1}}}} @@ -351,7 +351,7 @@ def fold_rows_bit_lt64{ assert{l > 4} ld:= l-1; lld:= l*ld {mult0, _} := unaligned_spaced_mask_mod{ld} - mult0 &= u64~~1< { - merge{(w/8-1)**255, (1<>3)^31 + 64*(n&7))} @@ -37,7 +37,7 @@ def mask_first{n} = { def mask{'count'} = n def mask{{x}} = tup{mask{x}} def mask{x:X if vect{X}} = x & (X~~mask_of_first{X,n}) - def mask{x:X if any_int{x}} = x & ((1< { ca := if (hasarch{'SSE4.2'} or hasarch{'AARCH64'}) { def S = [l]i64; S~~c > S**0 } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 5674bdc2..4209c299 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -264,7 +264,7 @@ fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u def sse{a} = make{[2]u64, a, 0} carry := sse{init} # xor-scan on bytes - xmat := V**base{256, 1<<(8-iota{8}) - 1} + xmat := V**base{256, tail{8-iota{8}}} def xor8 = emit{V, '_mm512_gf2p8affine_epi64_epi8', ., xmat, 0} # Exclusive xor-scan on one word def exor64 = clmul{., sse{1<<64 - 2}, 0} @@ -600,7 +600,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, nl:usz, l:usz) : void = { i :usz = 0 # row bit index wn:usz = 0 # starting word of next row c:u64 = id # carry - def word{bit} = bit * ((1<<64) - 1) + def word{bit} = bit * tail{64} we:= nl/64; while (wn < we) { iw:= wn r := res_m1{load{src, iw}, c, u64~~1 << (i%64)} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 3db24588..d6a0d7c2 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -276,7 +276,7 @@ fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi u = fill_bittab(w0, nw, tab, u, -1) if (u == 0) { # All found! - @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + @for (r in *u64~~r0 over cdiv{nx,64}) r = tail{64} } else { bittab_lookup{x0, nx, r0, tab} } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 9057fb5b..d9ba939e 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -223,7 +223,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { @for_masked{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over wl) { cw:= wrapChk{cw0, xlf, M} got:= gather{VD**0, x, cw, M} - if (TDE!=TD) got&= VD**((1<>(width{I}-w) + 1) & (1<>(width{I}-w) + 1} def popc_alt{v, i, w} = if (has_popc) popc{v} else tab_popc{i, w} # slash{c, T} defines: @@ -289,7 +289,7 @@ def pext_popc{x:T, m:T} = { tup{ x - (x>>1 & z0), zm + z0 } } else if (hasarch{'AVX2'} and vect{T} and k >= 32) { # We have variable shifts at these sizes - lh := scal{low_s*(1<> S~~zl) | (x&lh), T~~(S~~z >> h) + zl} @@ -303,7 +303,7 @@ def pext_popc{x:T, m:T} = { # Shift high x group down by low z, then add halves of z odd:T = scal{low_s*(1<0 move tup{ (x&~move) | shift{1, z1, x&move}>>1, @@ -314,7 +314,7 @@ def pext_popc{x:T, m:T} = { # Compose k/g groups with k/g-1 regular shifts def multi_shift{x, z, g, k, sc} = { o := z * sc{lowbits{k,g}} # Offsets by prefix sum - def s = 1<>(sh-g) # Offset for group def gr{sh} = (x & sc{s<> (oo{sh} & s0) # Shifted group @@ -341,7 +341,7 @@ def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = { m := m0 x := x0 & m d := ~m << 1 # One bit of the position difference at x - c := V**(1<<64-1) + c := V**tail{64} @unroll (i to lb{scalwidth{V}}) { def sh = 1 << i def shift_at{v, s} = { v = (v&~s) | (v&s)>>sh } @@ -400,6 +400,6 @@ export{'si_thresh_compress_bool', u64~~thresh_bool{}} # Mask i is the smallest possible mask containing 1 every i bits: # there would be a 1 just past the top bit. # The number of trailing zeros is 64%i , and the popcount is 64/i . -def get_spaced_masks{i} = (1<<64 - 1<<(64%i)) / (1<