From b768332c036a9b7ed28cbfb9494ca1c7896f9713 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 1 Jun 2025 04:50:21 +0300 Subject: [PATCH] explicit 2x unroll for x86 copy.singeli clang was unrolling ~8x anyway, but without ability to uninterleave the loads & stores --- src/singeli/src/copy.singeli | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index b8e0e871..8f12e607 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -22,7 +22,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { rp:= *tern{R==u1, u64, R} ~~ r def XV = [bulk]X def RV = [bulk]R - def ur = tern{hasarch{'AARCH64'}, 4, 1} + def ur = tern{hasarch{'AARCH64'}, 4, 2} if (X==R and R!=u1) { if (hasarch{'X86_64'} and l<=bulk) store_narrow{rp, 0, load_widen{xp, 0, RV}, mask_first{l}} @@ -31,11 +31,11 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { # show{'R==u64', X, R} assert{X==u8 or X==u16 or X==u32} # TODO could maybe read 256 bits and use unpack to write >256 - @for_masked{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)} + @for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{eachx{|, x, RV**(cbqn_c32Tag{}<<48)}} } else if (X==u1 and R==u1) { # show{'u1u1', X, R} def V64 = [vw/64]u64 - @for_masked{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x} + @for_mu{vcount{V64},ur}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x} } else if (X==u1) { # show{'X==u1', X, R} copyFromBits{[bulk]R, load_expand_bits{., xp, .}, r, l}