Non-CLMUL SIMD ≠`

This commit is contained in:
Marshall Lochbaum 2025-02-26 08:13:06 -05:00
parent d85d69e759
commit 422e63bc12
2 changed files with 41 additions and 20 deletions

View File

@ -90,11 +90,37 @@ export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}}
export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}} export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}}
export{'si_scan_pluswrap_u32', scan_assoc_0{u32, +}} export{'si_scan_pluswrap_u32', scan_assoc_0{u32, +}}
def rotate_right{x:[l]_} = shuf{x, (iota{l}-1)%l}
def broadcast_last{x:[l]_} = shuf{x, l**(l-1)}
def blend_first{x:V=[l]_, y:V} = blend{x, y, 0 < iota{l}}
def shift_first{c:V=[l]_, p:V} = {
if (l==2) zip{c, p, 0}
else blend_first{c, rotate_right{p}}
}
# xor scan # xor scan
fn scan_neq{}(p:u64, x:*u64, r:*u64, nw:u64) : void = { def vec_prefix_byshift{op, sh} = {
def pre{v:V, k} = if (k < elwidth{V}) pre{op{v, sh{v,k}}, 2*k} else v
{v:T} => pre{v, 1}
}
def scan_word_ne = prefix_byshift{^, <<}
def scan_words_ne = vec_prefix_byshift{^, <<}
fn scan_neq{}(c:u64, x:*u64, r:*u64, nw:u64) : void = {
@for (x, r over nw) { @for (x, r over nw) {
r = p ^ prefix_byshift{^, <<}{x} r = c ^ scan_word_ne{x}
p = -(r>>63) # repeat sign bit c = -(r>>63) # repeat sign bit
}
}
fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:u64) : void = {
def vl = arch_defvw / 64
def V = [vl]u64
c := V**c0
@for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) {
s:= scan_words_ne{x}
p:= scan_assoc_id0{^}{-(s>>63)} ^ c
r = s ^ shift_first{c, p}
c = broadcast_last{p}
} }
} }
fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
@ -316,9 +342,6 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}}
# Row-wise boolean scan # Row-wise boolean scan
def rotate_right{x:[vl]_} = shuf{x, (iota{vl}-1)%vl}
def blend_first{x:V=[vl]_, y:V} = blend{x, y, 0 < iota{vl}}
# Create masks of the given type with spacing l>=64 # Create masks of the given type with spacing l>=64
def loose_mask_gen{(u64), l} = { def loose_mask_gen{(u64), l} = {
q:usz = 0 # distance to next row boundary q:usz = 0 # distance to next row boundary
@ -407,9 +430,9 @@ def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry
{s, k}:= scan_words{x, m, ml} {s, k}:= scan_words{x, m, ml}
# Propagate carries and adjust result # Propagate carries and adjust result
p:= propagate{k, c} p:= propagate{k, c}
t:= (if (vl==2) zip{c, p, 0} else blend_first{c, rotate_right{p}}) t:= shift_first{c, p}
r = apply_carry{s, -fix_carry{t}, ml} r = apply_carry{s, -fix_carry{t}, ml}
c = shuf{V, p, vl**(vl-1)} c = broadcast_last{p}
} }
} }
def vec_loop_with_loose_mask{...a={xp, rp, nw, l, id, scan_words}, apply_carry} = { def vec_loop_with_loose_mask{...a={xp, rp, nw, l, id, scan_words}, apply_carry} = {
@ -499,14 +522,8 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, nl:usz, l:usz) : void = {
} }
fn scan_rows_neq(x:*u64, r:*u64, nl:usz, l:usz) : void = { fn scan_rows_neq(x:*u64, r:*u64, nl:usz, l:usz) : void = {
def scan_word = prefix_byshift{^, <<} def scan_word = scan_word_ne
def scan_words = { def scan_words = scan_words_ne
def vec_prefix_byshift{op, sh} = {
def pre{v:V, k} = if (k < elwidth{V}) pre{op{v, sh{v,k}}, 2*k} else v
{v:T} => pre{v, 1}
}
vec_prefix_byshift{^, <<}
}
def apply_carry{s, c, f} = s ^ (f & c) def apply_carry{s, c, f} = s ^ (f & c)
assert{l > 0} assert{l > 0}
nw := cdiv{nl, 64} nw := cdiv{nl, 64}
@ -529,7 +546,7 @@ fn scan_rows_neq(x:*u64, r:*u64, nl:usz, l:usz) : void = {
s ^ ((c & f) | (b<<l - b)) s ^ ((c & f) | (b<<l - b))
}} }}
} }
} else if (has_simd and l < (if (hasarch{'AVX2'}) 320 else 256)) { } else if (has_simd and l < (if (hasarch{'AVX2'}) 320 else 200)) {
def scan_words{x:V, m:V, ml:V} = { def scan_words{x:V, m:V, ml:V} = {
s:= scan_words{x} s:= scan_words{x}
s^= -(s<<1 & m) s^= -(s<<1 & m)

View File

@ -70,10 +70,14 @@ def make_scan_idem{T, op} = make_scan_idem{T, op, 1}
def scan_assoc_id0{op} = { def scan_assoc_id0{op} = {
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
def shl0{v:V, k==128 if hasarch{'AVX2'}} = { def shl0{v:V=[_]T, k==128 if hasarch{'AVX2'}} = {
# Broadcast end of lane 0 to entire lane 1 # Broadcast end of lane 0 to entire lane 1
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v} if (width{T} < 64) {
sel{[8]i32, l, make{[8]i32, 3*(3<iota{8})}} l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
sel{[8]i32, l, make{[8]i32, 3*(3<iota{8})}}
} else {
sel{[4]i64, v & V~~make{[4]i64,0,-1,0,0}, 1<iota{4}}
}
} }
prefix_byshift{op, shl0} prefix_byshift{op, shl0}
} }