Improve baseline for ⊣`˘ to beat non-AVX2 on length ≥64

This commit is contained in:
Marshall Lochbaum 2025-02-25 20:05:37 -05:00
parent 709712f03c
commit d85d69e759

View File

@ -579,24 +579,28 @@ fn scan_rows_left(x:*u64, r:*u64, nl:usz, l:usz) : void = {
(c & f) | apply_mask{x, m} (c & f) | apply_mask{x, m}
}} }}
} }
} else if ((hasarch{'SSE4.1'} or hasarch{'AARCH64'}) and l < (if (hasarch{'SSE4.1'} and not hasarch{'AVX2'}) 72 else 176)) { } else if ((hasarch{'AVX2'} or hasarch{'AARCH64'}) and l < 176) {
def scan_words{x:V, m:V, _} = { def scan_words{x:V, m:V, _} = {
s:= -(x & m) s:= -(x & m)
tup{s, s>>63 | (m == V**0)} tup{s, s>>63 | (m == V**0)}
} }
vec_loop_with_loose_mask{x, r, nw, l, 0, scan_words, apply_carry} vec_loop_with_loose_mask{x, r, nw, l, 0, scan_words, apply_carry}
} else { } else {
i :usz = 0 # row bit index assert{l >= 64}
wn:usz = 0 # starting word of next row k:= l/64 - 1 # at least k full aligned words in a row
c:u64 = 0 # carry i :usz = 0 # row bit index
wn:usz = 0 # starting word of next row
c:u64 = 0 # carry
we:= nl/64; while (wn < we) { we:= nl/64; while (wn < we) {
iw:= wn iw:= wn
m := u64~~1 << (i%64) m := u64~~1 << (i%64)
xw:= -(load{x, iw} & m) xw:= -(load{x, iw} & m)
store{r, iw, (c & (m-1)) | xw} r0:= (c & (m-1)) | xw
c = -(xw>>63) c = -(xw>>63)
i+= l; wn = i/64 i+= l; wn = i/64
@for (r in r over _ from iw+1 to wn) r = c store{r, wn-1, c}
store{r, iw, r0}
@for (r in r+iw+1 over k) r = c
} }
if (i%64 != 0) store{r, wn, c} if (i%64 != 0) store{r, wn, c}
} }