Merge pull request #70 from mlochbaum/transpose
SIMD transpose on all integer widths
This commit is contained in:
commit
0fc700b4f4
@ -1,17 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
DIR="$(dirname $0)"
|
||||
PREV_BQN="$DIR/obj2/for_build"
|
||||
PREV_BQN="$DIR/obj2/for_build2"
|
||||
if [ ! -f "$PREV_BQN" ]; then
|
||||
TMP_BQN="$DIR/obj2/for_build_tmp"
|
||||
if make -C "$DIR/.." forbuild OUTPUT="$TMP_BQN"; then
|
||||
if make -C "$DIR/.." for-build OUTPUT="$TMP_BQN"; then
|
||||
if [ "4" = "$("$TMP_BQN" -p "2+2")" ]; then
|
||||
true
|
||||
else
|
||||
echo "bootstrap CBQN failed to return expected result; run 'make forbuild' with necessary configuration manually"
|
||||
echo "automatically-built bootstrap CBQN doesn't behave as expected; run 'make for-build' with necessary configuration manually"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "failed to build bootstrap CBQN; run 'make forbuild' with necessary configuration manually"
|
||||
echo "failed to build bootstrap CBQN; run 'make for-build' with necessary configuration manually"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 2b4e3b8cdf6f3a5f8ddf383a8421f012550b8c27
|
||||
Subproject commit a17f83b2db2c1e025a56af0401682c45eb883257
|
||||
@ -549,7 +549,7 @@ MakeLinkerInv ← { 𝕊 GetArgs‿cache‿name‿srcs:
|
||||
cachedBin‿linkerCache ← {
|
||||
Shorten ← {po.clangd? 𝕩; r ← {𝕩↓˜¯1-⊑'.'⊐˜⌽𝕩}¨ •file.Name¨ 𝕩 ⋄ ! ∧´ ∊r ⋄ r}
|
||||
cbqnSrc ← ∾{⌽(⊑𝕩)⊸•file.At¨ 1↓𝕩}¨ ⌽⟨
|
||||
⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩
|
||||
⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "transpose.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩
|
||||
⟨"src/core/", "tyarr.c", "harr.c", "fillarr.c", "stuff.c", "derv.c", "mm.c", "heap.c"⟩
|
||||
⟨"src/", "load.c", "main.c", "rtwrap.c", "vm.c", "ns.c", "nfns.c", "ffi.c"⟩
|
||||
⟨"src/jit/", "nvm.c"⟩
|
||||
@ -562,7 +562,7 @@ cachedBin‿linkerCache ← {
|
||||
"xa"‿"src/builtins/arithd.c"‿"dyarith", "xa"‿"src/builtins/cmp.c"‿"cmp", "xa"‿"src/builtins/squeeze.c"‿"squeeze"
|
||||
"x."‿"src/builtins/select.c"‿"select", "x."‿"src/builtins/fold.c"‿"fold", "x."‿"src/builtins/scan.c"‿"scan"
|
||||
"x."‿"src/builtins/scan.c"‿"neq", "x."‿"src/builtins/slash.c"‿"slash", "x."‿"src/builtins/slash.c"‿"constrep"
|
||||
"x."‿"src/builtins/sfns.c"‿"transpose"
|
||||
"x."‿"src/builtins/transpose.c"‿"transpose"
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
|
||||
6
makefile
6
makefile
@ -37,8 +37,8 @@ shared-o3:
|
||||
@"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 i_t=shared_o3 i_f="-O3" run_incremental_0
|
||||
shared-c:
|
||||
@"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 custom=1 run_incremental_0
|
||||
forbuild:
|
||||
@"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build run_incremental_0
|
||||
for-build:
|
||||
@"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build2 run_incremental_0
|
||||
for-bootstrap:
|
||||
@"${MAKE}" i_t=for_bootstrap i_f='-DNATIVE_COMPILER -DONLY_NATIVE_COMP -DFORMATTER=0 -DNO_RT -DNO_EXPLAIN' run_incremental_0 i_USE_BC_SUBMODULE=0 BYTECODE_DIR=bytecodeNone
|
||||
c:
|
||||
@ -283,7 +283,7 @@ ${bd}/%.o: src/jit/%.c
|
||||
@echo $< | cut -c 5-
|
||||
@$(CC_INC) $@.d -o $@ -c $<
|
||||
|
||||
builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o}
|
||||
builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o transpose.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o}
|
||||
${bd}/%.o: src/builtins/%.c
|
||||
@echo $< | cut -c 5-
|
||||
@$(CC_INC) $@.d -o $@ -c $<
|
||||
|
||||
@ -1237,150 +1237,6 @@ B reverse_c2(B t, B w, B x) {
|
||||
return withFill(mut_fcd(r, x), xf);
|
||||
}
|
||||
|
||||
#ifdef __BMI2__
|
||||
#include <immintrin.h>
|
||||
#if USE_VALGRIND
|
||||
#define _pdep_u64 vg_pdep_u64
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if SINGELI_X86_64
|
||||
static NOINLINE void base_transpose_u32(u32* rp, u32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*ro+y] = xp[y*xo+x]; }
|
||||
#define SINGELI_FILE transpose
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
|
||||
extern B rt_transp;
|
||||
B transp_c1(B t, B x) {
|
||||
if (RARE(isAtm(x))) return m_atomUnit(x);
|
||||
ur xr = RNK(x);
|
||||
if (xr<=1) return x;
|
||||
|
||||
usz ia = IA(x);
|
||||
usz* xsh = SH(x);
|
||||
usz h = xsh[0];
|
||||
if (ia==0 || h==1) {
|
||||
no_reorder:;
|
||||
Arr* r = cpyWithShape(x);
|
||||
ShArr* sh = m_shArr(xr);
|
||||
shcpy(sh->a, xsh+1, xr-1);
|
||||
sh->a[xr-1] = h;
|
||||
arr_shReplace(r, xr, sh);
|
||||
return taga(r);
|
||||
}
|
||||
usz w = xsh[1] * shProd(xsh, 2, xr);
|
||||
if (w==1) goto no_reorder;
|
||||
|
||||
Arr* r;
|
||||
usz xi = 0;
|
||||
u8 xe = TI(x,elType);
|
||||
bool toBit = false;
|
||||
if (h==2) {
|
||||
if (xe==el_B) {
|
||||
B* xp = TO_BPTR(x);
|
||||
B* x0 = xp; B* x1 = x0+w;
|
||||
HArr_p rp = m_harrUp(ia);
|
||||
for (usz i=0; i<w; i++) { rp.a[i*2] = inc(x0[i]); rp.a[i*2+1] = inc(x1[i]); }
|
||||
NOGC_E;
|
||||
r = (Arr*) rp.c;
|
||||
} else {
|
||||
#ifndef __BMI2__
|
||||
if (xe==el_bit) { x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; }
|
||||
void* rp = m_tyarrp(&r,elWidth(xe),ia,el2t(xe));
|
||||
#else
|
||||
void* rp = m_tyarrlbp(&r,elWidthLogBits(xe),ia,el2t(xe));
|
||||
#endif
|
||||
void* xp = tyany_ptr(x);
|
||||
switch(xe) { default: UD;
|
||||
#ifdef __BMI2__
|
||||
case el_bit:;
|
||||
u32* x0 = xp;
|
||||
Arr* x1o = TI(x,slice)(inc(x),w,w);
|
||||
u32* x1 = (u32*) ((TyArr*)x1o)->a;
|
||||
for (usz i=0; i<BIT_N(ia); i++) ((u64*)rp)[i] = _pdep_u64(x0[i], 0x5555555555555555) | _pdep_u64(x1[i], 0xAAAAAAAAAAAAAAAA);
|
||||
mm_free((Value*)x1o);
|
||||
break;
|
||||
#endif
|
||||
case el_i8: case el_c8: { u8* x0=xp; u8* x1=x0+w; for (usz i=0; i<w; i++) { ((u8* )rp)[i*2] = x0[i]; ((u8* )rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_i16:case el_c16: { u16* x0=xp; u16* x1=x0+w; for (usz i=0; i<w; i++) { ((u16*)rp)[i*2] = x0[i]; ((u16*)rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_i32:case el_c32: { u32* x0=xp; u32* x1=x0+w; for (usz i=0; i<w; i++) { ((u32*)rp)[i*2] = x0[i]; ((u32*)rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_f64: { u64* x0=xp; u64* x1=x0+w; for (usz i=0; i<w; i++) { ((u64*)rp)[i*2] = x0[i]; ((u64*)rp)[i*2+1] = x1[i]; } } break;
|
||||
}
|
||||
}
|
||||
} else if (w==2 && xe!=el_B) {
|
||||
#ifndef __BMI2__
|
||||
if (xe==el_bit) { x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; }
|
||||
#endif
|
||||
void* rp = m_tyarrlbp(&r,elWidthLogBits(xe),ia,el2t(xe));
|
||||
void* xp = tyany_ptr(x);
|
||||
switch(xe) { default: UD;
|
||||
#if __BMI2__
|
||||
case el_bit:;
|
||||
u64* r0 = rp; TALLOC(u64, r1, BIT_N(h));
|
||||
for (usz i=0; i<BIT_N(ia); i++) {
|
||||
u64 v = ((u64*)xp)[i];
|
||||
((u32*)r0)[i] = _pext_u64(v, 0x5555555555555555);
|
||||
((u32*)r1)[i] = _pext_u64(v, 0xAAAAAAAAAAAAAAAA);
|
||||
}
|
||||
bit_cpy(r0, h, r1, 0, h);
|
||||
TFREE(r1);
|
||||
break;
|
||||
#endif
|
||||
case el_i8: case el_c8: { u8* r0=rp; u8* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u8* )xp)[i*2]; r1[i] = ((u8* )xp)[i*2+1]; } } break;
|
||||
case el_i16:case el_c16: { u16* r0=rp; u16* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u16*)xp)[i*2]; r1[i] = ((u16*)xp)[i*2+1]; } } break;
|
||||
case el_i32:case el_c32: { u32* r0=rp; u32* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u32*)xp)[i*2]; r1[i] = ((u32*)xp)[i*2+1]; } } break;
|
||||
case el_f64: { f64* r0=rp; f64* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((f64*)xp)[i*2]; r1[i] = ((f64*)xp)[i*2+1]; } } break;
|
||||
}
|
||||
} else {
|
||||
switch(xe) { default: UD;
|
||||
case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough
|
||||
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_i32:case el_c32:
|
||||
#if SINGELI_X86_64
|
||||
if (w>=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; }
|
||||
#endif
|
||||
{ u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_f64: { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;y<h;y++) NOVECTORIZE for(usz x=0;x<w;x++) rp[x*h+y] = xp[xi++]; break; }
|
||||
case el_B: { // can't be bothered to implement a bitarr transpose
|
||||
B xf = getFillR(x);
|
||||
B* xp = TO_BPTR(x);
|
||||
|
||||
HArr_p p = m_harrUp(ia);
|
||||
for(usz y=0;y<h;y++) for(usz x=0;x<w;x++) p.a[x*h+y] = inc(xp[xi++]); // TODO inc afterwards, but don't when there's a method of freeing a HArr without freeing its elements
|
||||
NOGC_E;
|
||||
|
||||
usz* rsh = arr_shAlloc((Arr*)p.c, xr);
|
||||
if (xr==2) {
|
||||
rsh[0] = w;
|
||||
rsh[1] = h;
|
||||
} else {
|
||||
shcpy(rsh, xsh+1, xr-1);
|
||||
rsh[xr-1] = h;
|
||||
}
|
||||
decG(x); return qWithFill(p.b, xf);
|
||||
}
|
||||
}
|
||||
}
|
||||
usz* rsh = arr_shAlloc(r, xr);
|
||||
if (xr==2) {
|
||||
rsh[0] = w;
|
||||
rsh[1] = h;
|
||||
} else {
|
||||
shcpy(rsh, xsh+1, xr-1);
|
||||
rsh[xr-1] = h;
|
||||
}
|
||||
decG(x); return taga(toBit? (Arr*)cpyBitArr(taga(r)) : r);
|
||||
}
|
||||
B transp_c2(B t, B w, B x) { return c2rt(transp, w, x); }
|
||||
|
||||
B transp_im(B t, B x) {
|
||||
if (isAtm(x)) thrM("⍉⁼: 𝕩 must not be an atom");
|
||||
if (RNK(x)<=2) return transp_c1(t, x);
|
||||
return def_fn_im(bi_transp, x);
|
||||
}
|
||||
|
||||
|
||||
B pick_uc1(B t, B o, B x) { // TODO do in-place like pick_ucw; maybe just call it?
|
||||
if (isAtm(x) || IA(x)==0) return def_fn_uc1(t, o, x);
|
||||
@ -1506,7 +1362,6 @@ B shape_uc1(B t, B o, B x) {
|
||||
|
||||
B select_ucw(B t, B o, B w, B x);
|
||||
|
||||
B transp_uc1(B t, B o, B x) { return transp_im(m_f64(0), c1(o, transp_c1(t, x))); }
|
||||
B reverse_uc1(B t, B o, B x) { return reverse_c1(m_f64(0), c1(o, reverse_c1(t, x))); }
|
||||
|
||||
B reverse_ix(B t, B w, B x) {
|
||||
@ -1537,8 +1392,6 @@ void sfns_init(void) {
|
||||
c(BFn,bi_pick)->ucw = pick_ucw;
|
||||
c(BFn,bi_select)->ucw = select_ucw; // TODO move to new init fn
|
||||
c(BFn,bi_shape)->uc1 = shape_uc1;
|
||||
c(BFn,bi_transp)->uc1 = transp_uc1;
|
||||
c(BFn,bi_transp)->im = transp_im;
|
||||
c(BFn,bi_take)->ucw = take_ucw;
|
||||
c(BFn,bi_drop)->ucw = drop_ucw;
|
||||
c(BFn,bi_lt)->im = enclose_im;
|
||||
|
||||
162
src/builtins/transpose.c
Normal file
162
src/builtins/transpose.c
Normal file
@ -0,0 +1,162 @@
|
||||
#include "../core.h"
|
||||
#include "../utils/each.h"
|
||||
#include "../utils/talloc.h"
|
||||
#include "../builtins.h"
|
||||
|
||||
#ifdef __BMI2__
|
||||
#include <immintrin.h>
|
||||
#if USE_VALGRIND
|
||||
#define _pdep_u64 vg_pdep_u64
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#define TRANSPOSE_LOOP( DST, SRC, W, H) PLAINLOOP for(usz y=0;y< H;y++) NOVECTORIZE for(usz x=0;x< W;x++) DST[x*H+y] = SRC[xi++]
|
||||
#define TRANSPOSE_BLOCK(DST, SRC, BW, BH, W, H) PLAINLOOP for(usz y=0;y<BH;y++) NOVECTORIZE for(usz x=0;x<BW;x++) DST[x*H+y] = SRC[y*W+x]
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#define DECL_BASE(T) \
|
||||
static NOINLINE void base_transpose_##T(T* rp, T* xp, u64 bw, u64 bh, u64 w, u64 h) { \
|
||||
TRANSPOSE_BLOCK(rp, xp, bw, bh, w, h); \
|
||||
}
|
||||
DECL_BASE(i8) DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64)
|
||||
#undef DECL_BASE
|
||||
#define SINGELI_FILE transpose
|
||||
#include "../utils/includeSingeli.h"
|
||||
#define TRANSPOSE_SIMD(T, DST, SRC, W, H) simd_transpose_##T(DST, SRC, W, H)
|
||||
#else
|
||||
#define TRANSPOSE_SIMD(T, DST, SRC, W, H) TRANSPOSE_LOOP(DST, SRC, W, H)
|
||||
#endif
|
||||
|
||||
|
||||
extern B rt_transp;
|
||||
B transp_c1(B t, B x) {
|
||||
if (RARE(isAtm(x))) return m_atomUnit(x);
|
||||
ur xr = RNK(x);
|
||||
if (xr<=1) return x;
|
||||
|
||||
usz ia = IA(x);
|
||||
usz* xsh = SH(x);
|
||||
usz h = xsh[0];
|
||||
if (ia==0 || h==1) {
|
||||
no_reorder:;
|
||||
Arr* r = cpyWithShape(x);
|
||||
ShArr* sh = m_shArr(xr);
|
||||
shcpy(sh->a, xsh+1, xr-1);
|
||||
sh->a[xr-1] = h;
|
||||
arr_shReplace(r, xr, sh);
|
||||
return taga(r);
|
||||
}
|
||||
usz w = xsh[1] * shProd(xsh, 2, xr);
|
||||
if (w==1) goto no_reorder;
|
||||
|
||||
Arr* r;
|
||||
usz xi = 0;
|
||||
u8 xe = TI(x,elType);
|
||||
bool toBit = false;
|
||||
if (h==2) {
|
||||
if (xe==el_B) {
|
||||
B* xp = TO_BPTR(x);
|
||||
B* x0 = xp; B* x1 = x0+w;
|
||||
HArr_p rp = m_harrUp(ia);
|
||||
for (usz i=0; i<w; i++) { rp.a[i*2] = inc(x0[i]); rp.a[i*2+1] = inc(x1[i]); }
|
||||
NOGC_E;
|
||||
r = (Arr*) rp.c;
|
||||
} else {
|
||||
#ifndef __BMI2__
|
||||
if (xe==el_bit) { x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; }
|
||||
void* rp = m_tyarrp(&r,elWidth(xe),ia,el2t(xe));
|
||||
#else
|
||||
void* rp = m_tyarrlbp(&r,elWidthLogBits(xe),ia,el2t(xe));
|
||||
#endif
|
||||
void* xp = tyany_ptr(x);
|
||||
switch(xe) { default: UD;
|
||||
#ifdef __BMI2__
|
||||
case el_bit:;
|
||||
u32* x0 = xp;
|
||||
Arr* x1o = TI(x,slice)(inc(x),w,w);
|
||||
u32* x1 = (u32*) ((TyArr*)x1o)->a;
|
||||
for (usz i=0; i<BIT_N(ia); i++) ((u64*)rp)[i] = _pdep_u64(x0[i], 0x5555555555555555) | _pdep_u64(x1[i], 0xAAAAAAAAAAAAAAAA);
|
||||
mm_free((Value*)x1o);
|
||||
break;
|
||||
#endif
|
||||
case el_i8: case el_c8: { u8* x0=xp; u8* x1=x0+w; for (usz i=0; i<w; i++) { ((u8* )rp)[i*2] = x0[i]; ((u8* )rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_i16:case el_c16: { u16* x0=xp; u16* x1=x0+w; for (usz i=0; i<w; i++) { ((u16*)rp)[i*2] = x0[i]; ((u16*)rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_i32:case el_c32: { u32* x0=xp; u32* x1=x0+w; for (usz i=0; i<w; i++) { ((u32*)rp)[i*2] = x0[i]; ((u32*)rp)[i*2+1] = x1[i]; } } break;
|
||||
case el_f64: { u64* x0=xp; u64* x1=x0+w; for (usz i=0; i<w; i++) { ((u64*)rp)[i*2] = x0[i]; ((u64*)rp)[i*2+1] = x1[i]; } } break;
|
||||
}
|
||||
}
|
||||
} else if (w==2 && xe!=el_B) {
|
||||
#ifndef __BMI2__
|
||||
if (xe==el_bit) { x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; }
|
||||
#endif
|
||||
void* rp = m_tyarrlbp(&r,elWidthLogBits(xe),ia,el2t(xe));
|
||||
void* xp = tyany_ptr(x);
|
||||
switch(xe) { default: UD;
|
||||
#if __BMI2__
|
||||
case el_bit:;
|
||||
u64* r0 = rp; TALLOC(u64, r1, BIT_N(h));
|
||||
for (usz i=0; i<BIT_N(ia); i++) {
|
||||
u64 v = ((u64*)xp)[i];
|
||||
((u32*)r0)[i] = _pext_u64(v, 0x5555555555555555);
|
||||
((u32*)r1)[i] = _pext_u64(v, 0xAAAAAAAAAAAAAAAA);
|
||||
}
|
||||
bit_cpy(r0, h, r1, 0, h);
|
||||
TFREE(r1);
|
||||
break;
|
||||
#endif
|
||||
case el_i8: case el_c8: { u8* r0=rp; u8* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u8* )xp)[i*2]; r1[i] = ((u8* )xp)[i*2+1]; } } break;
|
||||
case el_i16:case el_c16: { u16* r0=rp; u16* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u16*)xp)[i*2]; r1[i] = ((u16*)xp)[i*2+1]; } } break;
|
||||
case el_i32:case el_c32: { u32* r0=rp; u32* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((u32*)xp)[i*2]; r1[i] = ((u32*)xp)[i*2+1]; } } break;
|
||||
case el_f64: { f64* r0=rp; f64* r1=r0+h; for (usz i=0; i<h; i++) { r0[i] = ((f64*)xp)[i*2]; r1[i] = ((f64*)xp)[i*2+1]; } } break;
|
||||
}
|
||||
} else {
|
||||
switch(xe) { default: UD;
|
||||
case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough
|
||||
case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); TRANSPOSE_SIMD( i8, rp, xp, w, h); break; }
|
||||
case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); TRANSPOSE_SIMD(i16, rp, xp, w, h); break; }
|
||||
case el_i32:case el_c32: { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); TRANSPOSE_SIMD(i32, rp, xp, w, h); break; }
|
||||
case el_f64: { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); TRANSPOSE_SIMD(i64, rp, xp, w, h); break; }
|
||||
case el_B: { // can't be bothered to implement a bitarr transpose
|
||||
B xf = getFillR(x);
|
||||
B* xp = TO_BPTR(x);
|
||||
|
||||
HArr_p p = m_harrUp(ia);
|
||||
for(usz y=0;y<h;y++) for(usz x=0;x<w;x++) p.a[x*h+y] = inc(xp[xi++]); // TODO inc afterwards, but don't when there's a method of freeing a HArr without freeing its elements
|
||||
NOGC_E;
|
||||
|
||||
usz* rsh = arr_shAlloc((Arr*)p.c, xr);
|
||||
if (xr==2) {
|
||||
rsh[0] = w;
|
||||
rsh[1] = h;
|
||||
} else {
|
||||
shcpy(rsh, xsh+1, xr-1);
|
||||
rsh[xr-1] = h;
|
||||
}
|
||||
decG(x); return qWithFill(p.b, xf);
|
||||
}
|
||||
}
|
||||
}
|
||||
usz* rsh = arr_shAlloc(r, xr);
|
||||
if (xr==2) {
|
||||
rsh[0] = w;
|
||||
rsh[1] = h;
|
||||
} else {
|
||||
shcpy(rsh, xsh+1, xr-1);
|
||||
rsh[xr-1] = h;
|
||||
}
|
||||
decG(x); return taga(toBit? (Arr*)cpyBitArr(taga(r)) : r);
|
||||
}
|
||||
B transp_c2(B t, B w, B x) { return c2rt(transp, w, x); }
|
||||
|
||||
B transp_im(B t, B x) {
|
||||
if (isAtm(x)) thrM("⍉⁼: 𝕩 must not be an atom");
|
||||
if (RNK(x)<=2) return transp_c1(t, x);
|
||||
return def_fn_im(bi_transp, x);
|
||||
}
|
||||
|
||||
B transp_uc1(B t, B o, B x) { return transp_im(m_f64(0), c1(o, transp_c1(t, x))); }
|
||||
|
||||
void transp_init(void) {
|
||||
c(BFn,bi_transp)->uc1 = transp_uc1;
|
||||
c(BFn,bi_transp)->im = transp_im;
|
||||
}
|
||||
@ -8,7 +8,7 @@
|
||||
#define PRECOMPILED_FILE1(X) PRECOMPILED_FILE0(X)
|
||||
#define PRECOMPILED_FILE(END) PRECOMPILED_FILE1(../build/BYTECODE_DIR/gen/END)
|
||||
|
||||
#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished)
|
||||
#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(transp) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished)
|
||||
#define F(X) NOINLINE void X##_init(void);
|
||||
FOR_INIT(F)
|
||||
#undef F
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
#include "../builtins/sort.c"
|
||||
#include "../builtins/search.c"
|
||||
#include "../builtins/selfsearch.c"
|
||||
#include "../builtins/transpose.c"
|
||||
#include "../builtins/fold.c"
|
||||
#include "../builtins/scan.c"
|
||||
#include "../builtins/arithm.c"
|
||||
|
||||
@ -11,53 +11,163 @@ if (hasarch{'X86_64'}) {
|
||||
include './mask'
|
||||
include './bitops'
|
||||
|
||||
def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} = {
|
||||
def t1 = merge{...each{{i} => unpackQ{tupsel{i*2,x}, tupsel{i*2+1,x}}, iota{4}}}
|
||||
def t2 = merge{...each{{i} => unpackQ{tupsel{i, t1}, tupsel{i+2, t1}}, tup{0,1,4,5}}}
|
||||
each{{i} => emit{[8]i32, '_mm256_permute2f128_si256', tupsel{i%4,t2}, tupsel{i%4+4,t2}, tern{i>=4,16b31,16b20}}, iota{8}}
|
||||
# Group l (power of 2) elements into paired groups of length o
|
||||
# e.g. pairs{2, iota{8}} = {{0,1,4,5}, {2,3,6,7}}
|
||||
def pairs{o, x} = {
|
||||
def i = iota{tuplen{x}/2}
|
||||
def g = 2*i - i%o
|
||||
tupsel{tup{g, g+o}, x}
|
||||
}
|
||||
def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}}
|
||||
def permute_pass{o, x} = {
|
||||
def p = pairs{o, x}
|
||||
def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p}
|
||||
merge{h{16b20}, h{16b31}}
|
||||
}
|
||||
def unpack_to{f, l, x} = {
|
||||
def pass = if (f) permute_pass else unpack_pass
|
||||
pass{l, if (l==1) x else unpack_to{0, l/2, x}}
|
||||
}
|
||||
# Last pass for square kernel packed in halves
|
||||
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
|
||||
|
||||
# Square kernel where width is a full vector
|
||||
def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x}
|
||||
|
||||
def load2{a:T, b:T} = pair{load{a}, load{b}}
|
||||
def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = {
|
||||
each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}}
|
||||
}
|
||||
def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
|
||||
def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{l}, x}
|
||||
def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
|
||||
def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
|
||||
|
||||
# Transpose kernel of size kw,kh in size w,h array
|
||||
def kernel{src:P, dst:P, kw, kh, w, h} = {
|
||||
def T = eltype{P}
|
||||
def n = (kw*kh*width{T}) / 256 # Number of vectors
|
||||
def xvs = load_k{[kw]T, src, n, w}
|
||||
def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n
|
||||
def rvs = if (n==kw) xt else shuf_pass{xt} # To kh by kh for packed square
|
||||
store_k{[kh]T, dst, rvs, n, h}
|
||||
}
|
||||
|
||||
|
||||
|
||||
fn transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = {
|
||||
if (wm<=0 or hm<=0) return{}
|
||||
|
||||
if (scl==1) {
|
||||
assert{T==u32}
|
||||
def VT = [8]i32
|
||||
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}}
|
||||
def rvs = vtranspose{xvs}
|
||||
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs}
|
||||
} else {
|
||||
o:= (scl+1)>>1; s:= i64~~o
|
||||
transpose_rec{T}(rpo , xpo , w, h, wm , hm , o)
|
||||
transpose_rec{T}(rpo + o*8, xpo + o*8*w , w, h, wm , hm-s, o)
|
||||
transpose_rec{T}(rpo + o*8*h , xpo + o*8, w, h, wm-s, hm , o)
|
||||
transpose_rec{T}(rpo + o*8*h + o*8, xpo + o*8*w + o*8, w, h, wm-s, hm-s, o)
|
||||
def for_mult{k}{vars,begin,end,block} = {
|
||||
assert{begin == 0}
|
||||
@for (i to end/k) exec{k*i, vars, block}
|
||||
}
|
||||
def for_mult_max{k, m}{vars,begin,end,block} = {
|
||||
@for_mult{k} (i0 to end) {
|
||||
i:=i0; if (i>m) i = m
|
||||
exec{i, vars, block}
|
||||
}
|
||||
}
|
||||
|
||||
fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = {
|
||||
# Scalar transpose defined in C
|
||||
def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64'
|
||||
def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a, w, h}
|
||||
|
||||
rp:*T = *T~~r0
|
||||
xp:*T = *T~~x0
|
||||
if (w<k or h<k) { call_base{rp, xp, w, h}; return{} }
|
||||
|
||||
@for (y to h/8) {
|
||||
@for (x to w/8) {
|
||||
def VT = [8]i32
|
||||
xpo:= xp + y*8*w + x*8
|
||||
rpo:= rp + x*8*h + y*8
|
||||
def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}}
|
||||
def rvs = vtranspose{xvs}
|
||||
each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs}
|
||||
}
|
||||
def at{x,y} = tup{xp + y*w + x, rp + x*h + y}
|
||||
|
||||
# Cache line info
|
||||
def line_bytes = 64
|
||||
def line_elts = line_bytes / (width{T}/8)
|
||||
|
||||
def use_overlap{o} = o >= 2 # For overlapped SIMD instead of scalar
|
||||
wo := w%k
|
||||
# Effective width: number of columns read, counting overlap twice
|
||||
# Just use base transpose for short overhang; otherwise round up
|
||||
we := w; if (use_overlap{wo}) we += k - wo
|
||||
wm := w - k
|
||||
|
||||
if (line_elts > 2*k or h&(line_elts-1) != 0) {
|
||||
ho := h%k
|
||||
# Effective height, like we for w
|
||||
he := h; if (use_overlap{ho}) he += k - ho
|
||||
def has_half = 2*k == kh
|
||||
if (has_half and he==kh and h<he) he = k # Skip main loop; caught with he<h tests later
|
||||
# Main transpose
|
||||
@for_mult_max{kh, h-kh} (y to he) {
|
||||
@for_mult_max{k, wm} (x to we) {
|
||||
kernel{...at{x,y}, k, kh, w, h}
|
||||
}
|
||||
}
|
||||
# Half-row(s) for non-square i16 case
|
||||
if (has_half and (he & k) != 0) {
|
||||
n := 1 + cast_i{u64, he < h} # 2 for overlapped halves
|
||||
e := h%kh; if (he<h or e<k) e = k
|
||||
@for (yi to n) {
|
||||
y:u64 = 0; if (yi == n-1) y = h - e
|
||||
@for_mult_max{k, wm} (x to we) {
|
||||
kernel{...at{x,y}, k, k, w, h}
|
||||
}
|
||||
}
|
||||
}
|
||||
# Base transpose used if overlap wasn't
|
||||
if (ho!=0 and he==h) { hs := h-ho; call_base{rp+hs, xp+w*hs, w, ho} }
|
||||
} else {
|
||||
# Result rows are aligned with each other so it's possible to
|
||||
# write a full cache line at a time
|
||||
# This case is here to mitigate cache associativity problems at
|
||||
# at multiples of 256 or so, but it's faster whenever it applies
|
||||
assert{k == kh}
|
||||
def VT = [k]T
|
||||
def line_vecs = line_bytes / (width{VT}/8)
|
||||
def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs}
|
||||
def get_lines{loadx} = {
|
||||
def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}}
|
||||
each{tup, ...each{vt, iota{line_vecs}}}
|
||||
}
|
||||
ro := tail{6, -u64~~r0} / (width{T}/8) # Offset to align within cache line; assume elt-aligned
|
||||
wh := w*h
|
||||
yn := h
|
||||
if (ro != 0) {
|
||||
ra := line_elts - ro
|
||||
y := h - ra
|
||||
rpe := rp + y + (w-1)*h # Cache aligned
|
||||
# Part of first and last result row aren't covered by the split loop
|
||||
def trtail{dst, src, len} = @for (i to len) store{dst, i, load{src, w*i}}
|
||||
trtail{rp, xp, ro}
|
||||
trtail{rpe, xp + y*w + w-1, ra}
|
||||
# Transpose first few rows and last few rows together
|
||||
@for_mult_max{k, wm} (x to we) {
|
||||
{xpo,rpo} := at{x, y}
|
||||
o := w*y + x
|
||||
def loadx{_} = {
|
||||
l:=load{*VT~~(xp+o)}
|
||||
o+=w; if (o>wh-k) o -= wh-1 # Jump from last source row to first, shifting right 1
|
||||
l
|
||||
}
|
||||
def rls = get_lines{loadx} # 4 rows of 2 vectors each
|
||||
each{{i,v} => {p:=rpo+i*h; if (i<3 or p<rpe) store_line{*VT~~p, v}}, iota{k}, rls}
|
||||
}
|
||||
--yn # One strip handled
|
||||
}
|
||||
@for_mult{line_elts} (y0 to yn) { y := y0 + ro
|
||||
@for_mult_max{k, wm} (x to we) {
|
||||
{xpo,rpo} := at{x, y}
|
||||
def rls = get_lines{{i} => load{*VT~~(xpo+i*w), 0}}
|
||||
each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# wm:= w/8
|
||||
# hm:= h/8
|
||||
# transpose_rec{T}(rp, xp, w, h, i64~~wm, i64~~hm, tern{wm>hm, wm, hm})
|
||||
|
||||
if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h}
|
||||
if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h}
|
||||
if (we==w) @for(ws from w-wo to w) {
|
||||
xpo:=xp+ws; rpo:=rp+h*ws
|
||||
@for (i to h) store{rpo, i, load{xpo, w*i}}
|
||||
}
|
||||
}
|
||||
|
||||
export{'simd_transpose_i32', transpose{u32}}
|
||||
def transpose{T, k} = transpose{T, k, k}
|
||||
|
||||
export{'simd_transpose_i8', transpose{i8 , 16}}
|
||||
export{'simd_transpose_i16', transpose{i16, 8, 16}}
|
||||
export{'simd_transpose_i32', transpose{i32, 8}}
|
||||
export{'simd_transpose_i64', transpose{i64, 4}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user