bitops.singeli renames

This commit is contained in:
dzaima 2025-02-02 18:08:31 +02:00
parent f409ee8729
commit 189ffa7d1f
9 changed files with 42 additions and 56 deletions

View File

@ -280,8 +280,8 @@ Tuples can be used in the iterated variable list for various things:
```
p:*T - regular pointer
tup{VT,p:*T} - loadBatch/storeBatch vector data
tup{'b',p:P} - b_getBatch (bits.singeli)
tup{'b',VT,p:P} - loadBatchBit (bits.singeli)
tup{'b',p:P} - load_bits (bits.singeli)
tup{'b',VT,p:P} - load_expand_bits (bits.singeli)
tup{'g',VT,p:*T} - gives a generator, used as g{} to loadBatch and g{newValue} to storeBatch
tup{'g',p:*T} - the above, but without load support
'm' - get the mask object - either maskNone or some maskAfter{n}
@ -337,18 +337,6 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un
<!--
bitops.singeli
b_get
b_getBatch
b_getBatchLo
b_set
b_setBatch
loadBatchBit
loadu
loaduBit
loaduBitRaw
loaduBitTrunc
ones
spreadBits
bmi2.singeli
pdep
pext

View File

@ -171,11 +171,11 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
# base cases
def {
absu,and_bit_none,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
absu,and_bit_none,andnz,load_bits,blend,blend_units,clmul,cvt,extract,fold_addw,half,
all_bit,any_bit,blend_bit,
all_hom,any_hom,blend_hom,hom_to_int,store_masked_hom,store_blended_hom,
all_top,any_top,blend_top,top_to_int,store_masked_top,store_blended_top,
loadBatchBit,load_low,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair,
load_expand_bits,load_low,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair,
pair,pdep,pext,rbit,sel,shufInd,store_low,reverse_units,
unord,unzip,vfold,vec_select,vec_shuffle,widen,widen_upper,multishift,
}
@ -277,10 +277,10 @@ def ceil_log2{1} = 0
def base{b,{}} = 0
def base{b,{h,...t}} = h + b*base{b,t}
def truncBits{n, v if n<=8} = cast_i{u8, v}
def truncBits{n, v if n==16} = cast_i{u16, v}
def truncBits{n, v if n==32} = cast_i{u32, v}
def truncBits{n, v if n==64} = cast_i{u64, v}
def trunc_bits{n, v if n<=8} = cast_i{u8, v}
def trunc_bits{n, v if n==16} = cast_i{u16, v}
def trunc_bits{n, v if n==32} = cast_i{u32, v}
def trunc_bits{n, v if n==64} = cast_i{u64, v}
# base-2 log of a constant power of two
def lb{n if knum{n} and (n>>1<<1) == n and n>0} = lb{n>>1}+1

View File

@ -1,30 +1,28 @@
def ones{T, n} = (cast{T,1}<<cast{T,n}) - 1
def b_get{x:(*u64), n:(ux)} = {
def load_bit{x:(*u64), n:(ux)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatchLo{sz, x:(*u64), n:(ux)} = match (sz) {
def load_bits_lo{sz, x:(*u64), n:(ux)} = match (sz) {
{2} => (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
{4} => (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
{sz if sz>=8} => load{*ty_u{sz}~~x, n}
}
def b_getBatch{sz, x:(*u64), n:(ux)} = match (sz) {
{2} => b_getBatchLo{sz, x, n} & 3
{4} => b_getBatchLo{sz, x, n} & 15
def load_bits{sz, x:(*u64), n:(ux)} = match (sz) {
{2} => load_bits_lo{sz, x, n} & 3
{4} => load_bits_lo{sz, x, n} & 15
{sz if sz>=8} => load{*ty_u{sz}~~x, n}
}
def b_set{x:(*u64), n:(ux), v:(u1)} = {
def store_bit{x:(*u64), n:(ux), v:(u1)} = {
m:u64 = cast{u64,1}<<(n&63)
p:u64 = load{x,n>>6}
if (v) store{x,n>>6,p | m}
else store{x,n>>6,p & ~m}
}
def b_setBatch{sz, x:(*u64), n:(ux), v} = match (sz) {
def store_bits{sz, x:(*u64), n:(ux), v} = match (sz) {
{4} => {
x8:= *u8 ~~ x
@ -53,13 +51,13 @@ def b_setBatch{sz, x:(*u64), n:(ux), v} = match (sz) {
am:u64 = 64/sz
w:u64 = load{x,n/am}
sh:u64 = (n&(am-1)) * sz
w&= ~(ones{u64,sz}<<sh)
w&= ~(u64~~tail{sz} << sh)
w|= (vc<<sh)
store{x, n/am, w}
}
}
def spreadBits{T==[32]u8, a:(u32)} = {
def expand_bits{T==[32]u8, a:(u32)} = {
def idxs = iota{32}
b:= [8]u32**a
c:= [32]u8~~b
@ -68,42 +66,42 @@ def spreadBits{T==[32]u8, a:(u32)} = {
e == (d&e)
}
def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
def expand_bits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
b:= sel{T, T~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{T, 1<<(iota{16}&7)}}
}
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
def expand_bits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
b:= T~~[8]u16**a
exp:= T~~shuf{[4]i32, vec_shuffle16_lo{mzip{b,b,0}, tup{0,0,1,1}}, 0,0,1,1}
(exp & make{T, 1<<(iota{16}&7)}) != T**0
}
def spreadBits{V=[k]T, a:A if k<=width{T} and quality{T}=='u'} = {
def expand_bits{V=[k]T, a:A if k<=width{T} and quality{T}=='u'} = {
b:= make{V, 1<<iota{k}}
b == (b & V ~~ re_el{A, V}**a) # not just V**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}
# vector with type V with each element being either all 0s or 1s
def loadBatchBit{V=[k]T, x:(*u64), n:(ux)} = {
spreadBits{V, b_getBatchLo{k, x, n}}
def load_expand_bits{V=[k]T, x:(*u64), n:(ux)} = {
expand_bits{V, load_bits_lo{k, x, n}}
}
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
def loaduBitRaw{x:(*u64), i} = {
def loadu_bits_raw{x:(*u64), i} = {
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
}
def loaduBit{x:(*u64), i, n} = {
def loadu_bits{x:(*u64), i, n} = {
assert{n<58 or ((n==58 or n==60) and (i%n == 0))}
loaduBitRaw{x, i}
loadu_bits_raw{x, i}
}
def loaduBitTrunc{x:(*u64), i, n if knum{n}} = truncBits{n, loaduBit{x, i, n}}
def loadu_bits_trunc{x:(*u64), i, n if knum{n}} = trunc_bits{n, loadu_bits{x, i, n}}
def loadBatchBit{T, x:(*u64), {...is}} = {
def load_expand_bits{T, x:(*u64), {...is}} = {
# def len = length{is}
# def [count]_ = T
# assert{count*len <= 64}
# bits:= b_getBatchLo{count*len, x, select{is,0}}
# @collect(i to len) spreadBits{T, truncBits{count, bits>>(i*count)}}
each{loadBatchBit{T, x, .}, is}
# bits:= load_bits_lo{count*len, x, select{is,0}}
# @collect(i to len) expand_bits{T, trunc_bits{count, bits>>(i*count)}}
each{load_expand_bits{T, x, .}, is}
}

View File

@ -76,7 +76,7 @@ def any2bit{VT=[k]T, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
def mask = if (same{op0, op}) hom_to_int else ({x} => ~hom_to_int{x})
@forNZ (ri to cdiv{len,bulk}) {
b_setBatch{bulk, dst, ri, mask{each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
store_bits{bulk, dst, ri, mask{each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
xi+= unr
}
}

View File

@ -38,7 +38,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
@maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
} else if (X==u1) {
# show{'X==u1', X, R}
copyFromBits{[bulk]R, loadBatchBit{., xp, .}, r, l}
copyFromBits{[bulk]R, load_expand_bits{., xp, .}, r, l}
} else if (R==u1) {
# show{'R==u1', X, R}
@ -49,7 +49,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
def bulk2 = bulk*unr
xi:ux = 0
@forNZ (i to cdiv{l,bulk2}) {
b_setBatch{bulk2, rp, i, hom_to_int{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}}
store_bits{bulk2, rp, i, hom_to_int{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}}
xi+= unr
}
} else if (width{X}<=width{R}) {
@ -73,7 +73,7 @@ fn copy_ubit{R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
def RV = [bulk]R
rp:= *R~~r
copyFromBits{RV, {T=[k]_, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*k, k}}, r, l}
copyFromBits{RV, {T=[k]_, i} => expand_bits{T, loadu_bits_trunc{*u64~~x0, xs+i*k, k}}, r, l}
# }
}

View File

@ -28,10 +28,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
f1:= TF**1.0
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
wu:= (if (hasarch{'AVX2'}) {
cw:= b_getBatchLo{bulk, *u64~~w, i}
cw:= load_bits_lo{bulk, *u64~~w, i}
blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
} else {
cw:= loadBatchBit{TU, *u64~~w, i}
cw:= load_expand_bits{TU, *u64~~w, i}
blend_hom{f0, f1, cw}
})
cx:= load{*TF ~~ x, i}

View File

@ -76,8 +76,8 @@ def mlExec{i, iter, vars0, bulk, M} = {
def vproc{'m'} = tptr{{_}=>M, '!'}
def vproc{{T,p:*E}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}}
def vproc{{'b', p:*E}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'}
def vproc{{'b',T,p:*E}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'}
def vproc{{'b', p:*E}} = tptr{{i} => load_bits{bulk, hCast{u64,p}, i}, '!'}
def vproc{{'b',T,p:*E}} = tptr{{i} => load_expand_bits{T, hCast{u64,p}, i}, '!'}
def vproc{{'g', p:*E}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'}
def vproc{{'g',T,p:*E}} = tptr{{i} => {
def dv{} = loadBatch{p, i, T}

View File

@ -139,7 +139,7 @@ local def all_type{G, vs} = all{match { {x:T}=>G{T}; {_}=>0 }, vs}
def hom_to_int{...vs if all_type{nvec,vs} and not all_type{nvecu,vs}} = hom_to_int{...each{ty_u, vs}}
def hom_to_int{x:V=[k]E if nvecu{V} and width{E}>=k} = {
truncBits{k, fold_add{x & make{V, 1<<iota{k}}}}
trunc_bits{k, fold_add{x & make{V, 1<<iota{k}}}}
}
def hom_to_int{x:V=([16]u8)} = {
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, tr_iota{3,0,1,2}}}
@ -164,14 +164,14 @@ def hom_to_int{a:V,b:V,c:V,d:V=([16]u8)} = {
}
def hom_to_int{...as={a0:V=[_]E, _, ..._} if w128u{V} and width{E}>=32} = hom_to_int{...each{{i}=>narrow_pair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
def hom_to_int{a:V,b:V=[k]E if nvecu{V} and k*2<=width{E}} = {
truncBits{k*2, fold_add{shrm{a,width{E}-k,b} & make{V, (1<<iota{k}) | (1<<(iota{k}+k))}}}
trunc_bits{k*2, fold_add{shrm{a,width{E}-k,b} & make{V, (1<<iota{k}) | (1<<(iota{k}+k))}}}
}
def and_bit_none{x:T, y:T if nveci{T}} = ~any_bit{x&y}
def hom_to_int_ext{a:T=[k]E if E!=u64} = {
def h = width{E}/2
tup{h, truncBits{k*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
tup{h, trunc_bits{k*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
}

View File

@ -283,7 +283,7 @@ export_tab{'si_select_tab', join{table{select_fn,
}
} else {
if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
xb:= shuf{[4]u64, expand_bits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
sr = hom_to_int{shuf{[16]i8, xb, cw}}