Improve baseline for ⊣`˘ to beat non-AVX2 on length ≥64
This commit is contained in:
parent
709712f03c
commit
d85d69e759
@ -579,24 +579,28 @@ fn scan_rows_left(x:*u64, r:*u64, nl:usz, l:usz) : void = {
|
|||||||
(c & f) | apply_mask{x, m}
|
(c & f) | apply_mask{x, m}
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
} else if ((hasarch{'SSE4.1'} or hasarch{'AARCH64'}) and l < (if (hasarch{'SSE4.1'} and not hasarch{'AVX2'}) 72 else 176)) {
|
} else if ((hasarch{'AVX2'} or hasarch{'AARCH64'}) and l < 176) {
|
||||||
def scan_words{x:V, m:V, _} = {
|
def scan_words{x:V, m:V, _} = {
|
||||||
s:= -(x & m)
|
s:= -(x & m)
|
||||||
tup{s, s>>63 | (m == V**0)}
|
tup{s, s>>63 | (m == V**0)}
|
||||||
}
|
}
|
||||||
vec_loop_with_loose_mask{x, r, nw, l, 0, scan_words, apply_carry}
|
vec_loop_with_loose_mask{x, r, nw, l, 0, scan_words, apply_carry}
|
||||||
} else {
|
} else {
|
||||||
i :usz = 0 # row bit index
|
assert{l >= 64}
|
||||||
wn:usz = 0 # starting word of next row
|
k:= l/64 - 1 # at least k full aligned words in a row
|
||||||
c:u64 = 0 # carry
|
i :usz = 0 # row bit index
|
||||||
|
wn:usz = 0 # starting word of next row
|
||||||
|
c:u64 = 0 # carry
|
||||||
we:= nl/64; while (wn < we) {
|
we:= nl/64; while (wn < we) {
|
||||||
iw:= wn
|
iw:= wn
|
||||||
m := u64~~1 << (i%64)
|
m := u64~~1 << (i%64)
|
||||||
xw:= -(load{x, iw} & m)
|
xw:= -(load{x, iw} & m)
|
||||||
store{r, iw, (c & (m-1)) | xw}
|
r0:= (c & (m-1)) | xw
|
||||||
c = -(xw>>63)
|
c = -(xw>>63)
|
||||||
i+= l; wn = i/64
|
i+= l; wn = i/64
|
||||||
@for (r in r over _ from iw+1 to wn) r = c
|
store{r, wn-1, c}
|
||||||
|
store{r, iw, r0}
|
||||||
|
@for (r in r+iw+1 over k) r = c
|
||||||
}
|
}
|
||||||
if (i%64 != 0) store{r, wn, c}
|
if (i%64 != 0) store{r, wn, c}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user