explicit 2x unroll for x86 copy.singeli

clang was unrolling ~8x anyway, but without ability to uninterleave the loads & stores
This commit is contained in:
dzaima 2025-06-01 04:50:21 +03:00
parent 9a2ecada1e
commit b768332c03

View File

@ -22,7 +22,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
rp:= *tern{R==u1, u64, R} ~~ r
def XV = [bulk]X
def RV = [bulk]R
def ur = tern{hasarch{'AARCH64'}, 4, 1}
def ur = tern{hasarch{'AARCH64'}, 4, 2}
if (X==R and R!=u1) {
if (hasarch{'X86_64'} and l<=bulk) store_narrow{rp, 0, load_widen{xp, 0, RV}, mask_first{l}}
@ -31,11 +31,11 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
# show{'R==u64', X, R}
assert{X==u8 or X==u16 or X==u32}
# TODO could maybe read 256 bits and use unpack to write >256
@for_masked{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)}
@for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{eachx{|, x, RV**(cbqn_c32Tag{}<<48)}}
} else if (X==u1 and R==u1) {
# show{'u1u1', X, R}
def V64 = [vw/64]u64
@for_masked{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
@for_mu{vcount{V64},ur}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
} else if (X==u1) {
# show{'X==u1', X, R}
copyFromBits{[bulk]R, load_expand_bits{., xp, .}, r, l}