SIMD kernel for Reorder Axes transposing last two axes

This commit is contained in:
Marshall Lochbaum 2023-03-27 22:12:12 -04:00
parent 8413461074
commit 5b17ee44d6

View File

@ -32,6 +32,7 @@
#endif #endif
#endif #endif
typedef void (*TranspFn)(void*,void*,u64,u64);
#if SINGELI #if SINGELI
#define transposeFns simd_transpose #define transposeFns simd_transpose
#define DECL_BASE(T) \ #define DECL_BASE(T) \
@ -50,7 +51,7 @@
} }
DECL_BASE(i8) DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64) DECL_BASE(i8) DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64)
#undef DECL_BASE #undef DECL_BASE
static void (*transposeFns[])(void*,void*,u64,u64) = { static TranspFn transposeFns[] = {
transpose_i8, transpose_i16, transpose_i32, transpose_i64 transpose_i8, transpose_i16, transpose_i32, transpose_i64
}; };
#endif #endif
@ -227,36 +228,65 @@ B transp_c2(B t, B w, B x) {
u8 xe = TI(x,elType); u8 xe = TI(x,elType);
usz csz = shProd(xsh, ar, xr); usz csz = shProd(xsh, ar, xr);
if (csz >= (32*8) >> elWidthLogBits(xe)) { // cell >= 32 bytes #define AXIS_LOOP(INIT, N_AX, I_INC, DO_INNER) \
usz ria = csz * shProd(rsh, 0, na); ur a0 = N_AX - 1; \
MAKE_MUT_INIT(rm, ria, xe); MUTG_INIT(rm); for (usz i=0; i<na; i++) st[i] *= csz; \
for (usz i=0; i<na; i++) st[i] *= csz; TALLOC(usz, ri, a0); for (usz i=0; i<a0; i++) ri[i]=0; \
TALLOC(usz, ri, na-1); for (usz i=0; i<na-1; i++) ri[i]=0; INIT \
for (usz i=0, j=0;;) { for (usz i=0, j0=0;;) { \
// Hardcode one innermost loop: we know a>=0 /* Hardcode one innermost loop: assume N_AX>=1 */ \
ur a = na - 1; ur a = a0; \
usz str = st[a]; usz str = st[a]; \
usz l = rsh[a]; usz l = rsh[a]; \
for (usz k=0; k<l; k++) { for (usz k=0; k<l; k++) { \
mut_copyG(rm, i, x, j+k*str, csz); usz j=j0+k*str; DO_INNER; \
i += csz; i += I_INC; \
} } \
if (i == ria) break; if (i == ria) break; \
// Update result index, starting with last axis finished /* Update result index starting with last axis finished */\
while (1) { while (1) { \
str = st[--a]; str = st[--a]; \
j += str; j0 += str; \
if (LIKELY(++ri[a] < rsh[a])) break; if (LIKELY(++ri[a] < rsh[a])) break; \
ri[a] = 0; ri[a] = 0; \
j -= rsh[a] * str; j0 -= rsh[a] * str; \
} } \
} } \
TFREE(ri); TFREE(ri);
u8 xlw = elWidthLogBits(xe);
if (csz >= (32*8) >> xlw) { // cell >= 32 bytes
usz ria = csz * shProd(rsh, 0, na);
AXIS_LOOP(MAKE_MUT_INIT(rm, ria, xe); MUTG_INIT(rm); ,
na, csz, mut_copyG(rm, i, x, j, csz));
Arr* ra = mut_fp(rm); Arr* ra = mut_fp(rm);
shSet_alloc(ra, rr, rsh); shSet_alloc(ra, rr, rsh);
r = withFill(taga(ra), getFillQ(x)); r = withFill(taga(ra), getFillQ(x));
decG(x); goto ret_decst; decG(x); goto ret_decst;
} }
if ((csz & (csz-1))==0 && csz<=64>>xlw && csz<<xlw>=8 // CPU-sized cells
&& !dup && na>=2 && p[na-1]==ar-2 && p[na-2]==ar-1 // Last two axes transposed
&& rsh[na-2]*rsh[na-1] >= (256*8) >> xlw // And large-ish
&& xe!=el_B) {
TranspFn tran = transposeFns[CTZ(csz<<xlw)-3];
usz rf = shProd(rsh, 0, na);
Arr* ra;
u8* rp = m_tyarrlbp(&ra,xlw,rf*csz,el2t(xe));
u8* xp = tyany_ptr(x);
usz w = rsh[na-2];
usz h = rsh[na-1];
if (na == 2) {
tran(rp, xp, w, h);
} else {
csz = (csz<<xlw) / 8; // Convert to bytes
usz ria = rf*csz;
usz hw = h*w*csz;
AXIS_LOOP(, na-2, hw, tran(rp+i, xp+j, w, h));
}
shSet_alloc(ra, rr, rsh);
r = taga(ra);
decG(x); goto ret_decst;
}
#undef AXIS_LOOP
// Reshape x for selection, collapsing ar axes // Reshape x for selection, collapsing ar axes
if (ar != 1) { if (ar != 1) {