Refactoring; check for small transpose width/height in Singeli, not C
This commit is contained in:
parent
ff6361e363
commit
5fccf4cda3
@ -10,12 +10,21 @@
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define TRANSPOSE_LOOP( DST, SRC, W, H) PLAINLOOP for(usz y=0;y< H;y++) NOVECTORIZE for(usz x=0;x< W;x++) DST[x*H+y] = SRC[xi++]
|
||||
#define TRANSPOSE_BLOCK(DST, SRC, BW, BH, W, H) PLAINLOOP for(usz y=0;y<BH;y++) NOVECTORIZE for(usz x=0;x<BW;x++) DST[x*H+y] = SRC[y*W+x]
|
||||
|
||||
#if SINGELI_X86_64
|
||||
static NOINLINE void base_transpose_i16(i16* rp, i16* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
|
||||
static NOINLINE void base_transpose_i32(i32* rp, i32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
|
||||
static NOINLINE void base_transpose_i64(i64* rp, i64* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
|
||||
#define DECL_BASE(T) \
|
||||
static NOINLINE void base_transpose_##T(T* rp, T* xp, u64 bw, u64 bh, u64 w, u64 h) { \
|
||||
TRANSPOSE_BLOCK(rp, xp, bw, bh, w, h); \
|
||||
}
|
||||
DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64)
|
||||
#undef DECL_BASE
|
||||
#define SINGELI_FILE transpose
|
||||
#include "../utils/includeSingeli.h"
|
||||
#define TRANSPOSE_SIMD(T, DST, SRC, W, H) simd_transpose_##T(DST, SRC, W, H)
|
||||
#else
|
||||
#define TRANSPOSE_SIMD(T, DST, SRC, W, H) TRANSPOSE_LOOP(DST, SRC, W, H)
|
||||
#endif
|
||||
|
||||
|
||||
@ -103,22 +112,10 @@ B transp_c1(B t, B x) {
|
||||
} else {
|
||||
switch(xe) { default: UD;
|
||||
case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough
|
||||
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_i16:case el_c16:
|
||||
#if SINGELI_X86_64
|
||||
if (w>=8 && h>=8) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; }
|
||||
#endif
|
||||
{ u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_i32:case el_c32:
|
||||
#if SINGELI_X86_64
|
||||
if (w>=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; }
|
||||
#endif
|
||||
{ u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_f64:
|
||||
#if SINGELI_X86_64
|
||||
if (w>=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; }
|
||||
#endif
|
||||
{ f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); TRANSPOSE_LOOP( rp, xp, w, h); break; }
|
||||
case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); TRANSPOSE_SIMD(i16, rp, xp, w, h); break; }
|
||||
case el_i32:case el_c32: { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); TRANSPOSE_SIMD(i32, rp, xp, w, h); break; }
|
||||
case el_f64: { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); TRANSPOSE_SIMD(i64, rp, xp, w, h); break; }
|
||||
case el_B: { // can't be bothered to implement a bitarr transpose
|
||||
B xf = getFillR(x);
|
||||
B* xp = TO_BPTR(x);
|
||||
|
||||
@ -36,6 +36,15 @@ def vtranspose{x & ktest{'X86_64',4,[4]i64}{x}} = {
|
||||
permute_pass{2, unpack_pass{1, x}}
|
||||
}
|
||||
|
||||
def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = {
|
||||
def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}}
|
||||
each{bind{~~,[16]i16}, r}
|
||||
}
|
||||
def load2{a:T, b:T & w128i{eltype{T}}} = {
|
||||
def V = eltype{T}
|
||||
emit{[2*vcount{V}](eltype{V}), '_mm256_loadu2_m128i', b, a}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def for_mult{k}{vars,begin,end,block} = {
|
||||
@ -43,9 +52,27 @@ def for_mult{k}{vars,begin,end,block} = {
|
||||
@for (i to end/k) exec{k*i, vars, block}
|
||||
}
|
||||
|
||||
def mat_at{rp,xp,w,h}{x,y} = tup{xp + y*w + x, rp + x*h + y}
|
||||
|
||||
# Scalar transpose defined in C
|
||||
def call_base{T} = {
|
||||
def ts = if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64'
|
||||
{...a} => emit{void, merge{'base_transpose_',ts}, ...a}
|
||||
}
|
||||
def small_transpose_out{T, k, rp, xp, w, h} = {
|
||||
if (w<k or h<k) { call_base{T}{rp, xp, w, h, w, h}; return{} }
|
||||
}
|
||||
def edge_transpose{T, k, rp, xp, w, h} = {
|
||||
def tr{...a} = call_base{T}{...a, w, h}
|
||||
wo := w%k; ws := w-wo; if (wo) tr{rp+h*ws, xp+ ws, wo, h }
|
||||
ho := h%k; hs := h-ho; if (ho) tr{rp+ hs, xp+w*hs, ws, ho}
|
||||
}
|
||||
|
||||
fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
rp:*T = *T~~r0
|
||||
xp:*T = *T~~x0
|
||||
small_transpose_out{T, k, rp, xp, w, h}
|
||||
def at = mat_at{rp,xp,w,h}
|
||||
def VT = [k]T
|
||||
|
||||
# Cache line info
|
||||
@ -56,8 +83,7 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
if (h&(line_elts-1) != 0) {
|
||||
@for_mult{k} (y to h) {
|
||||
@for_mult{k} (x to w) {
|
||||
xpo:= xp + y*w + x
|
||||
rpo:= rp + x*h + y
|
||||
{xpo,rpo} := at{x, y}
|
||||
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{k}}
|
||||
def rvs = vtranspose{xvs}
|
||||
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs}
|
||||
@ -100,44 +126,36 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
}
|
||||
@for_mult{line_elts} (y0 to yn) { y := y0 + ro
|
||||
@for_mult{k} (x to w) {
|
||||
xpo:= xp + y*w + x
|
||||
rpo:= rp + x*h + y
|
||||
{xpo,rpo} := at{x, y}
|
||||
def rls = get_lines{{i} => load{*VT~~(xpo+i*w), 0}}
|
||||
each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def base = if (T==i32) 'base_transpose_i32' else 'base_transpose_i64'
|
||||
if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h}
|
||||
if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h}
|
||||
edge_transpose{T, k, rp, xp, w, h}
|
||||
}
|
||||
|
||||
def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = {
|
||||
def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}}
|
||||
each{bind{~~,[16]i16}, r}
|
||||
}
|
||||
def load2{x, y} = emit{[16]i16, '_mm256_loadu2_m128i', *[8]i16~~y, *[8]i16~~x}
|
||||
|
||||
fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
fn transpose{T, k, m==2}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
rp:*T = *T~~r0
|
||||
xp:*T = *T~~x0
|
||||
def d = 2*k
|
||||
small_transpose_out{T, k, rp, xp, w, h}
|
||||
def at = mat_at{rp,xp,w,h}
|
||||
def d = m*k
|
||||
def VT = [d]T
|
||||
def HT = [k]T
|
||||
|
||||
@for_mult{d} (y to h) {
|
||||
@for_mult{k} (x to w) {
|
||||
xpo:= xp + y*w + x
|
||||
rpo:= rp + x*h + y
|
||||
def xvs = each{{i}=>{p:=xpo+i*w; load2{p, p+k*w}}, iota{k}}
|
||||
{xpo, rpo} := at{x, y}
|
||||
def xvs = each{{i}=>{p:=xpo+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}}
|
||||
def rvs = vtranspose2{xvs}
|
||||
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs}
|
||||
}
|
||||
}
|
||||
if ((h & k) != 0) { y := h-h%d
|
||||
@for_mult{k} (x to w) {
|
||||
xpo:= xp + y*w + x
|
||||
rpo:= rp + x*h + y
|
||||
{xpo, rpo} := at{x, y}
|
||||
def lw = k*width{T}
|
||||
def xvs = each{{i}=>loadLow{*VT~~(xpo+i*w), lw}, iota{k}}
|
||||
def rvs = vtranspose2{xvs}
|
||||
@ -145,11 +163,9 @@ fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
def base = 'base_transpose_i16'
|
||||
if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h}
|
||||
if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h}
|
||||
edge_transpose{T, k, rp, xp, w, h}
|
||||
}
|
||||
|
||||
export{'simd_transpose_i16', transpose2{i16, 8}}
|
||||
export{'simd_transpose_i16', transpose{i16, 8, 2}}
|
||||
export{'simd_transpose_i32', transpose{i32, 8}}
|
||||
export{'simd_transpose_i64', transpose{i64, 4}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user