SIMD i32 transpose

This commit is contained in:
dzaima 2023-02-26 01:19:54 +02:00
parent a928277e8c
commit 6d03071ae6
5 changed files with 78 additions and 2 deletions

View File

@ -562,6 +562,7 @@ cachedBin‿linkerCache ← {
"xa""src/builtins/arithd.c""dyarith", "xa""src/builtins/cmp.c""cmp", "xa""src/builtins/squeeze.c""squeeze"
"x.""src/builtins/select.c""select", "x.""src/builtins/fold.c""fold", "x.""src/builtins/scan.c""scan"
"x.""src/builtins/scan.c""neq", "x.""src/builtins/slash.c""slash", "x.""src/builtins/slash.c""constrep"
"x.""src/builtins/sfns.c""transpose"
objs

View File

@ -299,7 +299,7 @@ endif
@"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c}
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c}
@echo $(postmsg)
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
@echo $< | cut -c 17- | sed 's/^/ /'

View File

@ -1254,6 +1254,13 @@ B reverse_c2(B t, B w, B x) {
#endif
#endif
#if SINGELI_X86_64
static NOINLINE void base_transpose_u32(u32* rp, u32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
#define SINGELI_FILE transpose
#include "../utils/includeSingeli.h"
#endif
extern B rt_transp;
B transp_c1(B t, B x) {
if (RARE(isAtm(x))) return m_atomUnit(x);
@ -1316,7 +1323,11 @@ B transp_c1(B t, B x) {
case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i32:case el_c32: { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_i32:case el_c32:
#if SINGELI_X86_64
if (w>=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; }
#endif
{ u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_f64: { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
case el_B: { // can't be bothered to implement a bitarr transpose
B* xp = arr_bptr(x);

View File

@ -51,6 +51,7 @@ def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32',
def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
def unpackQ{a:[ 8]i32, b:[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
def unpackQ{a:[ 4]i64, b:[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}}
# inverse of questionable pack; these saturate the argument
def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}

View File

@ -0,0 +1,63 @@
include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'X86_64'}) {
include './sse3'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './mask'
include './bitops'
def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} = {
def t1 = merge{...each{{i} => unpackQ{tupsel{i*2,x}, tupsel{i*2+1,x}}, iota{4}}}
def t2 = merge{...each{{i} => unpackQ{tupsel{i, t1}, tupsel{i+2, t1}}, tup{0,1,4,5}}}
each{{i} => emit{[8]i32, '_mm256_permute2f128_si256', tupsel{i%4,t2}, tupsel{i%4+4,t2}, tern{i>=4,16b31,16b20}}, iota{8}}
}
transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
if (wm<=0 or hm<=0) return{}
if (scl==1) {
assert{T==u32}
def VT = [8]i32
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}}
def rvs = vtranspose{xvs}
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs}
} else {
o:= (scl+1)>>1; s:= i64~~o
transpose_rec{T}(rpo , xpo , w, h, wm , hm , o)
transpose_rec{T}(rpo + o*8, xpo + o*8*w , w, h, wm , hm-s, o)
transpose_rec{T}(rpo + o*8*h , xpo + o*8, w, h, wm-s, hm , o)
transpose_rec{T}(rpo + o*8*h + o*8, xpo + o*8*w + o*8, w, h, wm-s, hm-s, o)
}
}
transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
rp:*T = *T~~r0
xp:*T = *T~~x0
@for (y to h/8) {
@for (x to w/8) {
def VT = [8]i32
xpo:= xp + y*8*w + x*8
rpo:= rp + x*8*h + y*8
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}}
def rvs = vtranspose{xvs}
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs}
}
}
# wm:= w/8
# hm:= h/8
# transpose_rec{T}(rp, xp, w, h, i64~~wm, i64~~hm, tern{wm>hm, wm, hm})
if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+(w-w%8), w%8, h, w, h}
if (h%8) emit{void, 'base_transpose_u32', rp+(h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h}
}
'simd_transpose_i32' = transpose{u32}