diff --git a/build/build b/build/build index 997d6ec9..3f5b4cc6 100755 --- a/build/build +++ b/build/build @@ -1,17 +1,17 @@ #!/usr/bin/env bash DIR="$(dirname $0)" -PREV_BQN="$DIR/obj2/for_build" +PREV_BQN="$DIR/obj2/for_build2" if [ ! -f "$PREV_BQN" ]; then TMP_BQN="$DIR/obj2/for_build_tmp" - if make -C "$DIR/.." forbuild OUTPUT="$TMP_BQN"; then + if make -C "$DIR/.." for-build OUTPUT="$TMP_BQN"; then if [ "4" = "$("$TMP_BQN" -p "2+2")" ]; then true else - echo "bootstrap CBQN failed to return expected result; run 'make forbuild' with necessary configuration manually" + echo "automatically-built bootstrap CBQN doesn't behave as expected; run 'make for-build' with necessary configuration manually" exit 1 fi else - echo "failed to build bootstrap CBQN; run 'make forbuild' with necessary configuration manually" + echo "failed to build bootstrap CBQN; run 'make for-build' with necessary configuration manually" exit 1 fi diff --git a/build/singeliSubmodule b/build/singeliSubmodule index 2b4e3b8c..a17f83b2 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit 2b4e3b8cdf6f3a5f8ddf383a8421f012550b8c27 +Subproject commit a17f83b2db2c1e025a56af0401682c45eb883257 diff --git a/build/src/build.bqn b/build/src/build.bqn index aa554f4c..ce035fe7 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -549,7 +549,7 @@ MakeLinkerInv ← { 𝕊 GetArgs‿cache‿name‿srcs: cachedBin‿linkerCache ← { Shorten ← {po.clangd? 𝕩; r ← {𝕩↓˜¯1-⊑'.'⊐˜⌽𝕩}¨ •file.Name¨ 𝕩 ⋄ ! ∧´ ∊r ⋄ r} cbqnSrc ← ∾{⌽(⊑𝕩)⊸•file.At¨ 1↓𝕩}¨ ⌽⟨ - ⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩ + ⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "transpose.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩ ⟨"src/core/", "tyarr.c", "harr.c", "fillarr.c", "stuff.c", "derv.c", "mm.c", "heap.c"⟩ ⟨"src/", "load.c", "main.c", "rtwrap.c", "vm.c", "ns.c", "nfns.c", "ffi.c"⟩ ⟨"src/jit/", "nvm.c"⟩ @@ -562,7 +562,7 @@ cachedBin‿linkerCache ← { "xa"‿"src/builtins/arithd.c"‿"dyarith", "xa"‿"src/builtins/cmp.c"‿"cmp", "xa"‿"src/builtins/squeeze.c"‿"squeeze" "x."‿"src/builtins/select.c"‿"select", "x."‿"src/builtins/fold.c"‿"fold", "x."‿"src/builtins/scan.c"‿"scan" "x."‿"src/builtins/scan.c"‿"neq", "x."‿"src/builtins/slash.c"‿"slash", "x."‿"src/builtins/slash.c"‿"constrep" - "x."‿"src/builtins/sfns.c"‿"transpose" + "x."‿"src/builtins/transpose.c"‿"transpose" ⟩ objs ← ⟨⟩ diff --git a/makefile b/makefile index c57a422f..36fc3b1b 100644 --- a/makefile +++ b/makefile @@ -37,8 +37,8 @@ shared-o3: @"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 i_t=shared_o3 i_f="-O3" run_incremental_0 shared-c: @"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 custom=1 run_incremental_0 -forbuild: - @"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build run_incremental_0 +for-build: + @"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build2 run_incremental_0 for-bootstrap: @"${MAKE}" i_t=for_bootstrap i_f='-DNATIVE_COMPILER -DONLY_NATIVE_COMP -DFORMATTER=0 -DNO_RT -DNO_EXPLAIN' run_incremental_0 i_USE_BC_SUBMODULE=0 BYTECODE_DIR=bytecodeNone c: @@ -283,7 +283,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o transpose.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 06112661..af3c092d 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1237,150 +1237,6 @@ B reverse_c2(B t, B w, B x) { return withFill(mut_fcd(r, x), xf); } -#ifdef __BMI2__ - #include - #if USE_VALGRIND - #define _pdep_u64 vg_pdep_u64 - #endif -#endif - -#if SINGELI_X86_64 - static NOINLINE void base_transpose_u32(u32* rp, u32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;ya, xsh+1, xr-1); - sh->a[xr-1] = h; - arr_shReplace(r, xr, sh); - return taga(r); - } - usz w = xsh[1] * shProd(xsh, 2, xr); - if (w==1) goto no_reorder; - - Arr* r; - usz xi = 0; - u8 xe = TI(x,elType); - bool toBit = false; - if (h==2) { - if (xe==el_B) { - B* xp = TO_BPTR(x); - B* x0 = xp; B* x1 = x0+w; - HArr_p rp = m_harrUp(ia); - for (usz i=0; ia; - for (usz i=0; i=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } - #endif - { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;yucw = pick_ucw; c(BFn,bi_select)->ucw = select_ucw; // TODO move to new init fn c(BFn,bi_shape)->uc1 = shape_uc1; - c(BFn,bi_transp)->uc1 = transp_uc1; - c(BFn,bi_transp)->im = transp_im; c(BFn,bi_take)->ucw = take_ucw; c(BFn,bi_drop)->ucw = drop_ucw; c(BFn,bi_lt)->im = enclose_im; diff --git a/src/builtins/transpose.c b/src/builtins/transpose.c new file mode 100644 index 00000000..73b39f1c --- /dev/null +++ b/src/builtins/transpose.c @@ -0,0 +1,162 @@ +#include "../core.h" +#include "../utils/each.h" +#include "../utils/talloc.h" +#include "../builtins.h" + +#ifdef __BMI2__ + #include + #if USE_VALGRIND + #define _pdep_u64 vg_pdep_u64 + #endif +#endif + +#define TRANSPOSE_LOOP( DST, SRC, W, H) PLAINLOOP for(usz y=0;y< H;y++) NOVECTORIZE for(usz x=0;x< W;x++) DST[x*H+y] = SRC[xi++] +#define TRANSPOSE_BLOCK(DST, SRC, BW, BH, W, H) PLAINLOOP for(usz y=0;ya, xsh+1, xr-1); + sh->a[xr-1] = h; + arr_shReplace(r, xr, sh); + return taga(r); + } + usz w = xsh[1] * shProd(xsh, 2, xr); + if (w==1) goto no_reorder; + + Arr* r; + usz xi = 0; + u8 xe = TI(x,elType); + bool toBit = false; + if (h==2) { + if (xe==el_B) { + B* xp = TO_BPTR(x); + B* x0 = xp; B* x1 = x0+w; + HArr_p rp = m_harrUp(ia); + for (usz i=0; ia; + for (usz i=0; iuc1 = transp_uc1; + c(BFn,bi_transp)->im = transp_im; +} diff --git a/src/load.c b/src/load.c index d569dc90..f6485267 100644 --- a/src/load.c +++ b/src/load.c @@ -8,7 +8,7 @@ #define PRECOMPILED_FILE1(X) PRECOMPILED_FILE0(X) #define PRECOMPILED_FILE(END) PRECOMPILED_FILE1(../build/BYTECODE_DIR/gen/END) -#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished) +#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(transp) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished) #define F(X) NOINLINE void X##_init(void); FOR_INIT(F) #undef F diff --git a/src/opt/single.c b/src/opt/single.c index 36f62ddf..382eb099 100644 --- a/src/opt/single.c +++ b/src/opt/single.c @@ -25,6 +25,7 @@ #include "../builtins/sort.c" #include "../builtins/search.c" #include "../builtins/selfsearch.c" +#include "../builtins/transpose.c" #include "../builtins/fold.c" #include "../builtins/scan.c" #include "../builtins/arithm.c" diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 99ff84c9..bbdb9057 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -11,53 +11,163 @@ if (hasarch{'X86_64'}) { include './mask' include './bitops' -def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} = { - def t1 = merge{...each{{i} => unpackQ{tupsel{i*2,x}, tupsel{i*2+1,x}}, iota{4}}} - def t2 = merge{...each{{i} => unpackQ{tupsel{i, t1}, tupsel{i+2, t1}}, tup{0,1,4,5}}} - each{{i} => emit{[8]i32, '_mm256_permute2f128_si256', tupsel{i%4,t2}, tupsel{i%4+4,t2}, tern{i>=4,16b31,16b20}}, iota{8}} +# Group l (power of 2) elements into paired groups of length o +# e.g. pairs{2, iota{8}} = {{0,1,4,5}, {2,3,6,7}} +def pairs{o, x} = { + def i = iota{tuplen{x}/2} + def g = 2*i - i%o + tupsel{tup{g, g+o}, x} +} +def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} +def permute_pass{o, x} = { + def p = pairs{o, x} + def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p} + merge{h{16b20}, h{16b31}} +} +def unpack_to{f, l, x} = { + def pass = if (f) permute_pass else unpack_pass + pass{l, if (l==1) x else unpack_to{0, l/2, x}} +} +# Last pass for square kernel packed in halves +def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} + +# Square kernel where width is a full vector +def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x} + +def load2{a:T, b:T} = pair{load{a}, load{b}} +def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { + each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} +} +def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} +def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{l}, x} +def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} +def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} + +# Transpose kernel of size kw,kh in size w,h array +def kernel{src:P, dst:P, kw, kh, w, h} = { + def T = eltype{P} + def n = (kw*kh*width{T}) / 256 # Number of vectors + def xvs = load_k{[kw]T, src, n, w} + def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n + def rvs = if (n==kw) xt else shuf_pass{xt} # To kh by kh for packed square + store_k{[kh]T, dst, rvs, n, h} } - -fn transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = { - if (wm<=0 or hm<=0) return{} - - if (scl==1) { - assert{T==u32} - def VT = [8]i32 - def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} - def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} - } else { - o:= (scl+1)>>1; s:= i64~~o - transpose_rec{T}(rpo , xpo , w, h, wm , hm , o) - transpose_rec{T}(rpo + o*8, xpo + o*8*w , w, h, wm , hm-s, o) - transpose_rec{T}(rpo + o*8*h , xpo + o*8, w, h, wm-s, hm , o) - transpose_rec{T}(rpo + o*8*h + o*8, xpo + o*8*w + o*8, w, h, wm-s, hm-s, o) +def for_mult{k}{vars,begin,end,block} = { + assert{begin == 0} + @for (i to end/k) exec{k*i, vars, block} +} +def for_mult_max{k, m}{vars,begin,end,block} = { + @for_mult{k} (i0 to end) { + i:=i0; if (i>m) i = m + exec{i, vars, block} } } -fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { +fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { + # Scalar transpose defined in C + def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' + def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a, w, h} + rp:*T = *T~~r0 xp:*T = *T~~x0 + if (wload{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} - def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} - } + def at{x,y} = tup{xp + y*w + x, rp + x*h + y} + + # Cache line info + def line_bytes = 64 + def line_elts = line_bytes / (width{T}/8) + + def use_overlap{o} = o >= 2 # For overlapped SIMD instead of scalar + wo := w%k + # Effective width: number of columns read, counting overlap twice + # Just use base transpose for short overhang; otherwise round up + we := w; if (use_overlap{wo}) we += k - wo + wm := w - k + + if (line_elts > 2*k or h&(line_elts-1) != 0) { + ho := h%k + # Effective height, like we for w + he := h; if (use_overlap{ho}) he += k - ho + def has_half = 2*k == kh + if (has_half and he==kh and hwh-k) o -= wh-1 # Jump from last source row to first, shifting right 1 + l + } + def rls = get_lines{loadx} # 4 rows of 2 vectors each + each{{i,v} => {p:=rpo+i*h; if (i<3 or p load{*VT~~(xpo+i*w), 0}} + each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} + } + } } - # wm:= w/8 - # hm:= h/8 - # transpose_rec{T}(rp, xp, w, h, i64~~wm, i64~~hm, tern{wm>hm, wm, hm}) - - if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h} - if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h} + if (we==w) @for(ws from w-wo to w) { + xpo:=xp+ws; rpo:=rp+h*ws + @for (i to h) store{rpo, i, load{xpo, w*i}} + } } -export{'simd_transpose_i32', transpose{u32}} \ No newline at end of file +def transpose{T, k} = transpose{T, k, k} + +export{'simd_transpose_i8', transpose{i8 , 16}} +export{'simd_transpose_i16', transpose{i16, 8, 16}} +export{'simd_transpose_i32', transpose{i32, 8}} +export{'simd_transpose_i64', transpose{i64, 4}}