From c0aaa6f61552c8b4393639781df6aa9c02ae8294 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 19:47:02 -0400 Subject: [PATCH] SIMD transpose on 8-byte elements --- src/builtins/sfns.c | 9 +++++++-- src/singeli/src/transpose.singeli | 24 +++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 06112661..b3a3f5ce 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1245,7 +1245,8 @@ B reverse_c2(B t, B w, B x) { #endif #if SINGELI_X86_64 - static NOINLINE void base_transpose_u32(u32* rp, u32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } #endif { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; } + #endif + { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;yemit{[8]i32, '_mm256_permute2f128_si256', a,b,p}, ...t2pairs} + merge{h{16b20}, h{16b31}} +} -fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { + +fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 + def VT = [k]T def for_mult{k}{vars,begin,end,block} = { assert{begin == 0} @for (i to end/k) exec{k*i, vars, block} } - # Kernel size - def k = 8 - def VT = [k]i32 - # Cache line info def line_bytes = 64 def line_elts = line_bytes / (width{T}/8) @@ -68,7 +72,7 @@ fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { def vt{i} = vtranspose{each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}} } - ro := tail{6, -u64~~r0} / 4 # Offset to align within cache line; assume elt-aligned + ro := tail{6, -u64~~r0} / (width{T}/8) # Offset to align within cache line; assume elt-aligned wh := w*h yn := h if (ro != 0) { @@ -103,8 +107,10 @@ fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h} - if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h} + def base = if (T==i32) 'base_transpose_i32' else 'base_transpose_i64' + if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} + if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} } -export{'simd_transpose_i32', transpose{u32}} +export{'simd_transpose_i32', transpose{i32, 8}} +export{'simd_transpose_i64', transpose{i64, 4}}