Initial 1-byte ⊣˝˘ with blends and modular permutation

This commit is contained in:
Marshall Lochbaum 2024-10-23 14:32:41 -04:00
parent 18f594e829
commit 9997c52e4c
4 changed files with 78 additions and 22 deletions

View File

@ -220,6 +220,7 @@ NOINLINE B leading_axis_arith(FC2 fc2, B w, B x, usz* wsh, usz* xsh, ur mr) { //
// fast special-case implementations // fast special-case implementations
extern void (*const si_select_cells_bit_lt64)(u64*,u64*,usz,usz,usz); // from fold.c (fold.singeli) extern void (*const si_select_cells_bit_lt64)(u64*,u64*,usz,usz,usz); // from fold.c (fold.singeli)
extern usz (*const si_select_cells_byte)(void*,void*,usz,usz);
static NOINLINE B select_cells(usz ind, B x, usz cam, usz k, bool leaf) { // ind {leaf? <∘⊑; ⊏}⎉¯k x; TODO probably can share some parts with takedrop_highrank and/or call ⊏? static NOINLINE B select_cells(usz ind, B x, usz cam, usz k, bool leaf) { // ind {leaf? <∘⊑; ⊏}⎉¯k x; TODO probably can share some parts with takedrop_highrank and/or call ⊏?
ur xr = RNK(x); ur xr = RNK(x);
assert(xr>1 && k<xr); assert(xr>1 && k<xr);
@ -258,18 +259,23 @@ static NOINLINE B select_cells(usz ind, B x, usz cam, usz k, bool leaf) { // ind
} else { } else {
void* rp = m_tyarrlbp(&ra, ewl, ria, el2t(xe)); void* rp = m_tyarrlbp(&ra, ewl, ria, el2t(xe));
void* xp = tyany_ptr(x); void* xp = tyany_ptr(x);
switch(xl) { if (xl == 0) {
case 0: #if SINGELI
#if SINGELI if (l < 64) si_select_cells_bit_lt64(xp, rp, cam, l, ind);
if (l < 64) si_select_cells_bit_lt64(xp, rp, cam, l, ind); else
else #endif
#endif for (usz i=0; i<cam; i++) bitp_set(rp, i, bitp_get(xp, i*l+ind));
for (usz i=0; i<cam; i++) bitp_set(rp, i, bitp_get(xp, i*l+ind)); } else {
break; usz i0 = 0;
case 3: PLAINLOOP for (usz i=0; i<cam; i++) ((u8* )rp)[i] = ((u8* )xp)[i*l+ind]; break; #if SINGELI
case 4: PLAINLOOP for (usz i=0; i<cam; i++) ((u16*)rp)[i] = ((u16*)xp)[i*l+ind]; break; if (xl==3) i0 = si_select_cells_byte((u8*)xp + (ind<<(xl-3)), rp, cam, l);
case 5: PLAINLOOP for (usz i=0; i<cam; i++) ((u32*)rp)[i] = ((u32*)xp)[i*l+ind]; break; #endif
case 6: PLAINLOOP for (usz i=0; i<cam; i++) ((f64*)rp)[i] = ((f64*)xp)[i*l+ind]; break; switch(xl) { default: UD;
case 3: PLAINLOOP for (usz i=i0; i<cam; i++) ((u8* )rp)[i] = ((u8* )xp)[i*l+ind]; break;
case 4: PLAINLOOP for (usz i=i0; i<cam; i++) ((u16*)rp)[i] = ((u16*)xp)[i*l+ind]; break;
case 5: PLAINLOOP for (usz i=i0; i<cam; i++) ((u32*)rp)[i] = ((u32*)xp)[i*l+ind]; break;
case 6: PLAINLOOP for (usz i=i0; i<cam; i++) ((f64*)rp)[i] = ((f64*)xp)[i*l+ind]; break;
}
} }
} }
} }

View File

@ -74,6 +74,54 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = {
export{'si_sum_f64', fold_assoc_0{f64,+}} export{'si_sum_f64', fold_assoc_0{f64,+}}
fn select_rows_byte(x0:*void, r0:*void, n:usz, l:usz) : usz = {
if (hasarch{'AVX2'} and n>=32 and l<20 and (l&1)==1) {
def V = [32]u8
def swap_by{x, m} = homBlend{x, shuf{[4]u64, x, 2,3,0,1}, m}
l8 := cast_i{u8, l}
li := cast_i{u8, l + 2 * ((l-1) + (l&2))} # Inverse mod 32
ie := make{[16]u16, 2*iota{16}}
def mu16{k} = {
k16 := [16]u16 ** k
(shuf{V~~(ie * k16), 0,0} + V~~(k16 << 8)) & V**31
}
sii := mu16{li}
ms := (V**16 & sii) == (V**16 &~ iota{V})
si := mu16{l8}
# Blend masks
def mg = { # Iteration i should select where mg == V**i
vs := V**0xff - scan_assoc_id0{+}{si < V**l8}
swap_by{shuf{[16]u8, vs, sii}, ms}
}
mgo := mg - V**(l8 & 3)
mgm := (mgo - V**1) & V**3
m4s := @collect (i to 3) mgm == V**i
# Main loop
xv := *V~~x0
nv := n / 32
@for (r in *V~~r0 over i to nv) {
r = load{xv,0}; ++xv
if ((l & 2) != 0) {
def {m0, _, m2} = m4s
re := homBlend{load{xv,0}, load{xv,1}, m2}
r = homBlend{re, r, m0}
xv += 2
}
mh := mgo
@for (l/4) {
{l0, ...ls} := each{load{xv,.}, iota{4}}
r4 := fold{homBlend, l0, ls, m4s}
r = homBlend{r, r4, mh < V**4}
mh -= V**4; xv += 4
}
r = shuf{[16]u8, swap_by{r, ms}, si}
}
return{32 * nv}
}
0
}
# Short-row boolean folds: main challenge is bit packing # Short-row boolean folds: main challenge is bit packing
def fold_rows_bit_lt64{ def fold_rows_bit_lt64{
op, run_loop2, run_loop4, pext_res, mult_in, op, run_loop2, run_loop4, pext_res, mult_in,
@ -330,3 +378,4 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = {
export{'si_xor_rows_bit', xor_rows_bit} export{'si_xor_rows_bit', xor_rows_bit}
export{'si_or_rows_bit', or_rows_bit} export{'si_or_rows_bit', or_rows_bit}
export{'si_select_cells_bit_lt64', select_rows_bit_lt64} export{'si_select_cells_bit_lt64', select_rows_bit_lt64}
export{'si_select_cells_byte', select_rows_byte}

View File

@ -74,16 +74,7 @@ export{'si_scan_min_i16', scan_idem_id{i16, min}}; export{'si_scan_max_i16', sca
export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', scan_idem_id{i32, max}} export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', scan_idem_id{i32, max}}
# Assumes identity is 0 # Assumes identity is 0
def scan_assoc{op} = { def scan_plus = scan_assoc_id0{+}
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
# Broadcast end of lane 0 to entire lane 1
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
sel{[8]i32, l, make{[8]i32, 3*(3<iota{8})}}
}
prefix_byshift{op, shl0}
}
def scan_plus = scan_assoc{+}
# Associative scan # Associative scan
def scan_assoc_0 = scan_scal def scan_assoc_0 = scan_scal

View File

@ -66,3 +66,13 @@ def make_scan_idem{(f64), op, up} = {
sc sc
} }
def make_scan_idem{T, op} = make_scan_idem{T, op, 1} def make_scan_idem{T, op} = make_scan_idem{T, op, 1}
def scan_assoc_id0{op} = {
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
# Broadcast end of lane 0 to entire lane 1
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
sel{[8]i32, l, make{[8]i32, 3*(3<iota{8})}}
}
prefix_byshift{op, shl0}
}