SIMD transpose on 2-byte elements

This commit is contained in:
Marshall Lochbaum 2023-03-20 20:57:52 -04:00
parent f6c6e72661
commit 239c765998
2 changed files with 42 additions and 9 deletions

View File

@ -1245,6 +1245,7 @@ B reverse_c2(B t, B w, B x) {
#endif
#if SINGELI_X86_64
static NOINLINE void base_transpose_i16(i16* rp, i16* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
static NOINLINE void base_transpose_i32(i32* rp, i32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
static NOINLINE void base_transpose_i64(i64* rp, i64* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
#define SINGELI_FILE transpose
@ -1337,7 +1338,11 @@ B transp_c1(B t, B x) {
switch(xe) { default: UD;
case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i16:case el_c16:
#if SINGELI_X86_64
if (w>=8 && h>=16) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; }
#endif
{ u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i32:case el_c32:
#if SINGELI_X86_64
if (w>=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; }

View File

@ -38,16 +38,16 @@ def vtranspose{x & ktest{'X86_64',4,[4]i64}{x}} = {
def for_mult{k}{vars,begin,end,block} = {
assert{begin == 0}
@for (i to end/k) exec{k*i, vars, block}
}
fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
rp:*T = *T~~r0
xp:*T = *T~~x0
def VT = [k]T
def for_mult{k}{vars,begin,end,block} = {
assert{begin == 0}
@for (i to end/k) exec{k*i, vars, block}
}
# Cache line info
def line_bytes = 64
def line_elts = line_bytes / (width{T}/8)
@ -58,9 +58,9 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
@for_mult{k} (x to w) {
xpo:= xp + y*w + x
rpo:= rp + x*h + y
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}}
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{k}}
def rvs = vtranspose{xvs}
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs}
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs}
}
}
} else {
@ -113,5 +113,33 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = {
if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h}
}
def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = {
def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}}
each{bind{~~,[16]i16}, r}
}
def load2{x, y} = emit{[16]i16, '_mm256_loadu2_m128i', *[8]i16~~y, *[8]i16~~x}
fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = {
rp:*T = *T~~r0
xp:*T = *T~~x0
def d = 2*k
def VT = [d]T
@for_mult{d} (y to h) {
@for_mult{k} (x to w) {
xpo:= xp + y*w + x
rpo:= rp + x*h + y
def xvs = each{{i}=>{p:=xpo+i*w; load2{p, p+k*w}}, iota{k}}
def rvs = vtranspose2{xvs}
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs}
}
}
def base = 'base_transpose_i16'
if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h}
if (h%d) emit{void, base, rp+ (h-h%d), xp+w*(h-h%d), w-w%k, h%d, w, h}
}
export{'simd_transpose_i16', transpose2{i16, 8}}
export{'simd_transpose_i32', transpose{i32, 8}}
export{'simd_transpose_i64', transpose{i64, 4}}