From bfb2519aeda0ce95c14c1382b7dcf9fcef31c9a1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 19 Mar 2023 17:18:09 -0400 Subject: [PATCH 01/23] Remove transpose_rec (was unused): the whole cache-oblivious thing seems to be useless --- src/singeli/src/transpose.singeli | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 99ff84c9..9c13eff2 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -19,24 +19,6 @@ def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} -fn transpose_rec{T}(rpo:*T, xpo:*T, w:u64, h:u64, wm:i64, hm:i64, scl:u64) : void = { - if (wm<=0 or hm<=0) return{} - - if (scl==1) { - assert{T==u32} - def VT = [8]i32 - def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} - def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} - } else { - o:= (scl+1)>>1; s:= i64~~o - transpose_rec{T}(rpo , xpo , w, h, wm , hm , o) - transpose_rec{T}(rpo + o*8, xpo + o*8*w , w, h, wm , hm-s, o) - transpose_rec{T}(rpo + o*8*h , xpo + o*8, w, h, wm-s, hm , o) - transpose_rec{T}(rpo + o*8*h + o*8, xpo + o*8*w + o*8, w, h, wm-s, hm-s, o) - } -} - fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 @@ -52,12 +34,8 @@ fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - # wm:= w/8 - # hm:= h/8 - # transpose_rec{T}(rp, xp, w, h, i64~~wm, i64~~hm, tern{wm>hm, wm, hm}) - if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h} if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h} } -export{'simd_transpose_i32', transpose{u32}} \ No newline at end of file +export{'simd_transpose_i32', transpose{u32}} From ca44d41daa82208771529ea3fb6eb433979ac8cb Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 19 Mar 2023 17:23:18 -0400 Subject: [PATCH 02/23] Faster i32 transpose when height is a multiple of 16 by writing cache lines --- src/singeli/src/transpose.singeli | 76 +++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 9c13eff2..dbd0478a 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -22,16 +22,74 @@ def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 + + def for_mult{k}{vars,begin,end,block} = { + assert{begin == 0} + @for (i to end/k) exec{k*i, vars, block} + } + + # Kernel size + def k = 8 + def VT = [k]i32 + + # Cache line info + def line_bytes = 64 + def line_elts = line_bytes / (width{T}/8) + def line_vecs = line_bytes / (width{VT}/8) - @for (y to h/8) { - @for (x to w/8) { - def VT = [8]i32 - xpo:= xp + y*8*w + x*8 - rpo:= rp + x*8*h + y*8 - def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} - def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} - } + if (h&(line_elts-1) != 0) { + @for_mult{k} (y to h) { + @for_mult{k} (x to w) { + xpo:= xp + y*w + x + rpo:= rp + x*h + y + def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} + def rvs = vtranspose{xvs} + each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} + } + } + } else { + # Result rows are aligned with each other so it's possible to + # write a full cache line at a time + # This case is here to mitigate cache associativity problems at + # at multiples of 256 or so, but it's faster whenever it applies + def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs} + def get_lines{loadx} = { + def vt{i} = vtranspose{each{loadx, k*i + iota{k}}} + each{tup, ...each{vt, iota{line_vecs}}} + } + ro := tail{6, -u64~~r0} / 4 # Offset to align within cache line; assume elt-aligned + wh := w*h + yn := h + if (ro != 0) { + ra := line_elts - ro + y := h - ra + rpo := rp + y # Cache aligned + rpe := rpo + (w-1)*h + # Part of first and last result row aren't covered by the split loop + def trtail{dst, src, len} = @for (i to len) store{dst, i, load{src, w*i}} + trtail{rp, xp, ro} + trtail{rpe, xp + y*w + w-1, ra} + # Transpose first few rows and last few rows together + @for_mult{k} (x to w) { + o := w*y + x + def loadx{_} = { + l:=load{*VT~~(xp+o)} + o+=w; if (o>wh-k) o -= wh-1 # Jump from last source row to first, shifting right 1 + l + } + def rls = get_lines{loadx} # 4 rows of 2 vectors each + each{{i,v} => {if (i<3 or rpo load{*VT~~(xpo+i*w), 0}} + each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} + } + } } if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h} From a0e85db702afee0a4ef8a6a093e902aaa837cf9e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 18:14:10 -0400 Subject: [PATCH 03/23] Refactor vtranspose with more general code --- src/singeli/src/transpose.singeli | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index dbd0478a..1124123a 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -11,10 +11,21 @@ if (hasarch{'X86_64'}) { include './mask' include './bitops' +# Group l (power of 2) elements into paired groups of length o +# e.g. pairs{2, iota{8}} = {{0,1,4,5}, {2,3,6,7}} +def pairs{o, x} = { + def i = iota{tuplen{x}/2} + def g = 2*i - i%o + tupsel{tup{g, g+o}, x} +} +def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} + def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} = { - def t1 = merge{...each{{i} => unpackQ{tupsel{i*2,x}, tupsel{i*2+1,x}}, iota{4}}} - def t2 = merge{...each{{i} => unpackQ{tupsel{i, t1}, tupsel{i+2, t1}}, tup{0,1,4,5}}} - each{{i} => emit{[8]i32, '_mm256_permute2f128_si256', tupsel{i%4,t2}, tupsel{i%4+4,t2}, tern{i>=4,16b31,16b20}}, iota{8}} + def t1 = unpack_pass{1, x} + def t2 = unpack_pass{2, t1} + def t2pairs = pairs{4, t2} + def h{p} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,p}, ...t2pairs} + merge{h{16b20}, h{16b31}} } From c0aaa6f61552c8b4393639781df6aa9c02ae8294 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 19:47:02 -0400 Subject: [PATCH 04/23] SIMD transpose on 8-byte elements --- src/builtins/sfns.c | 9 +++++++-- src/singeli/src/transpose.singeli | 24 +++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 06112661..b3a3f5ce 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1245,7 +1245,8 @@ B reverse_c2(B t, B w, B x) { #endif #if SINGELI_X86_64 - static NOINLINE void base_transpose_u32(u32* rp, u32* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } #endif { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; } + #endif + { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;yemit{[8]i32, '_mm256_permute2f128_si256', a,b,p}, ...t2pairs} + merge{h{16b20}, h{16b31}} +} -fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { + +fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 + def VT = [k]T def for_mult{k}{vars,begin,end,block} = { assert{begin == 0} @for (i to end/k) exec{k*i, vars, block} } - # Kernel size - def k = 8 - def VT = [k]i32 - # Cache line info def line_bytes = 64 def line_elts = line_bytes / (width{T}/8) @@ -68,7 +72,7 @@ fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { def vt{i} = vtranspose{each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}} } - ro := tail{6, -u64~~r0} / 4 # Offset to align within cache line; assume elt-aligned + ro := tail{6, -u64~~r0} / (width{T}/8) # Offset to align within cache line; assume elt-aligned wh := w*h yn := h if (ro != 0) { @@ -103,8 +107,10 @@ fn transpose{T}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - if (w%8) emit{void, 'base_transpose_u32', rp+h*(w-w%8), xp+ (w-w%8), w%8, h, w, h} - if (h%8) emit{void, 'base_transpose_u32', rp+ (h-h%8), xp+w*(h-h%8), w-w%8, h%8, w, h} + def base = if (T==i32) 'base_transpose_i32' else 'base_transpose_i64' + if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} + if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} } -export{'simd_transpose_i32', transpose{u32}} +export{'simd_transpose_i32', transpose{i32, 8}} +export{'simd_transpose_i64', transpose{i64, 4}} From f6c6e7266194df9fc090532c70972b769d5eb1d7 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 20:01:39 -0400 Subject: [PATCH 05/23] More transpose kernel refactoring --- src/singeli/src/transpose.singeli | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index c489c528..49cbd910 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -19,20 +19,21 @@ def pairs{o, x} = { tupsel{tup{g, g+o}, x} } def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} - -def vtranspose{x & tuplen{x}==8 & type{tupsel{0,x}}==[8]i32 & hasarch{'X86_64'}} = { - def t1 = unpack_pass{1, x} - def t2 = unpack_pass{2, t1} - def t2pairs = pairs{4, t2} - def h{p} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,p}, ...t2pairs} +def permute_pass{o, x} = { + def p = pairs{o, x} + def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p} merge{h{16b20}, h{16b31}} } -def vtranspose{x & tuplen{x}==4 & type{tupsel{0,x}}==[4]i64 & hasarch{'X86_64'}} = { - def t1 = unpack_pass{1, x} - def t2pairs = pairs{2, t1} - def h{p} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,p}, ...t2pairs} - merge{h{16b20}, h{16b31}} +def ktest{a,l,T}{x} = { + if (hasarch{a} and tuplen{x}==l and type{tupsel{0,x}}==T) 1 else 0 +} + +def vtranspose{x & ktest{'X86_64',8,[8]i32}{x}} = { + permute_pass{4, unpack_pass{2, unpack_pass{1, x}}} +} +def vtranspose{x & ktest{'X86_64',4,[4]i64}{x}} = { + permute_pass{2, unpack_pass{1, x}} } From 239c765998a52e5f5ee25b2bc49634c940f0ade1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 20:57:52 -0400 Subject: [PATCH 06/23] SIMD transpose on 2-byte elements --- src/builtins/sfns.c | 7 ++++- src/singeli/src/transpose.singeli | 44 +++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index b3a3f5ce..a9ecab2b 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1245,6 +1245,7 @@ B reverse_c2(B t, B w, B x) { #endif #if SINGELI_X86_64 + static NOINLINE void base_transpose_i16(i16* rp, i16* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;y=8 && h>=16) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } + #endif + { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 49cbd910..7c88be5b 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -38,16 +38,16 @@ def vtranspose{x & ktest{'X86_64',4,[4]i64}{x}} = { +def for_mult{k}{vars,begin,end,block} = { + assert{begin == 0} + @for (i to end/k) exec{k*i, vars, block} +} + fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 def VT = [k]T - - def for_mult{k}{vars,begin,end,block} = { - assert{begin == 0} - @for (i to end/k) exec{k*i, vars, block} - } - + # Cache line info def line_bytes = 64 def line_elts = line_bytes / (width{T}/8) @@ -58,9 +58,9 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { @for_mult{k} (x to w) { xpo:= xp + y*w + x rpo:= rp + x*h + y - def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{vcount{VT}}} + def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{k}} def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{vcount{VT}}, rvs} + each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} } } } else { @@ -113,5 +113,33 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} } +def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = { + def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} + each{bind{~~,[16]i16}, r} +} +def load2{x, y} = emit{[16]i16, '_mm256_loadu2_m128i', *[8]i16~~y, *[8]i16~~x} + +fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = { + rp:*T = *T~~r0 + xp:*T = *T~~x0 + def d = 2*k + def VT = [d]T + + @for_mult{d} (y to h) { + @for_mult{k} (x to w) { + xpo:= xp + y*w + x + rpo:= rp + x*h + y + def xvs = each{{i}=>{p:=xpo+i*w; load2{p, p+k*w}}, iota{k}} + def rvs = vtranspose2{xvs} + each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} + } + } + + def base = 'base_transpose_i16' + if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} + if (h%d) emit{void, base, rp+ (h-h%d), xp+w*(h-h%d), w-w%k, h%d, w, h} +} + +export{'simd_transpose_i16', transpose2{i16, 8}} export{'simd_transpose_i32', transpose{i32, 8}} export{'simd_transpose_i64', transpose{i64, 4}} From 0e2e315e512993584c46fd868a615d9f7fe9ca05 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 20 Mar 2023 21:43:34 -0400 Subject: [PATCH 07/23] Allow an 8x8 row for 2-byte transpose --- src/builtins/sfns.c | 2 +- src/singeli/src/transpose.singeli | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index a9ecab2b..4c177d11 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1340,7 +1340,7 @@ B transp_c1(B t, B x) { case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=8 && h>=16) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } + if (w>=8 && h>=8) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } #endif { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;yload{*VT~~(xpo+i*w), 0}, iota{k}} def rvs = vtranspose{xvs} each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} - } + } } } else { # Result rows are aligned with each other so it's possible to @@ -132,12 +132,22 @@ fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = { def xvs = each{{i}=>{p:=xpo+i*w; load2{p, p+k*w}}, iota{k}} def rvs = vtranspose2{xvs} each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} - } + } + } + if ((h & k) != 0) { y := h-h%d + @for_mult{k} (x to w) { + xpo:= xp + y*w + x + rpo:= rp + x*h + y + def lw = k*width{T} + def xvs = each{{i}=>loadLow{*VT~~(xpo+i*w), lw}, iota{k}} + def rvs = vtranspose2{xvs} + each{{i,v}=>storeLow{*VT~~(rpo+i*h), lw, v}, iota{k}, rvs} + } } def base = 'base_transpose_i16' if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} - if (h%d) emit{void, base, rp+ (h-h%d), xp+w*(h-h%d), w-w%k, h%d, w, h} + if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} } export{'simd_transpose_i16', transpose2{i16, 8}} From ff6361e36394870e90bb4a12c9f4baccd8fcd820 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 11:01:09 -0400 Subject: [PATCH 08/23] Move tranpose to its own file --- build/src/build.bqn | 4 +- makefile | 2 +- src/builtins/sfns.c | 157 ------------------------------------- src/builtins/transpose.c | 165 +++++++++++++++++++++++++++++++++++++++ src/load.c | 2 +- src/opt/single.c | 1 + 6 files changed, 170 insertions(+), 161 deletions(-) create mode 100644 src/builtins/transpose.c diff --git a/build/src/build.bqn b/build/src/build.bqn index aa554f4c..ce035fe7 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -549,7 +549,7 @@ MakeLinkerInv ← { 𝕊 GetArgs‿cache‿name‿srcs: cachedBin‿linkerCache ← { Shorten ← {po.clangd? 𝕩; r ← {𝕩↓˜¯1-⊑'.'⊐˜⌽𝕩}¨ •file.Name¨ 𝕩 ⋄ ! ∧´ ∊r ⋄ r} cbqnSrc ← ∾{⌽(⊑𝕩)⊸•file.At¨ 1↓𝕩}¨ ⌽⟨ - ⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩ + ⟨"src/builtins/", "arithd.c", "arithm.c", "cmp.c", "sfns.c", "squeeze.c", "select.c", "slash.c", "group.c", "sort.c", "search.c", "selfsearch.c", "transpose.c", "fold.c", "scan.c", "md1.c", "md2.c", "fns.c", "sysfn.c", "internal.c", "inverse.c"⟩ ⟨"src/core/", "tyarr.c", "harr.c", "fillarr.c", "stuff.c", "derv.c", "mm.c", "heap.c"⟩ ⟨"src/", "load.c", "main.c", "rtwrap.c", "vm.c", "ns.c", "nfns.c", "ffi.c"⟩ ⟨"src/jit/", "nvm.c"⟩ @@ -562,7 +562,7 @@ cachedBin‿linkerCache ← { "xa"‿"src/builtins/arithd.c"‿"dyarith", "xa"‿"src/builtins/cmp.c"‿"cmp", "xa"‿"src/builtins/squeeze.c"‿"squeeze" "x."‿"src/builtins/select.c"‿"select", "x."‿"src/builtins/fold.c"‿"fold", "x."‿"src/builtins/scan.c"‿"scan" "x."‿"src/builtins/scan.c"‿"neq", "x."‿"src/builtins/slash.c"‿"slash", "x."‿"src/builtins/slash.c"‿"constrep" - "x."‿"src/builtins/sfns.c"‿"transpose" + "x."‿"src/builtins/transpose.c"‿"transpose" ⟩ objs ← ⟨⟩ diff --git a/makefile b/makefile index c57a422f..dd07b8c9 100644 --- a/makefile +++ b/makefile @@ -283,7 +283,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o tranpose.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 4c177d11..af3c092d 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1237,160 +1237,6 @@ B reverse_c2(B t, B w, B x) { return withFill(mut_fcd(r, x), xf); } -#ifdef __BMI2__ - #include - #if USE_VALGRIND - #define _pdep_u64 vg_pdep_u64 - #endif -#endif - -#if SINGELI_X86_64 - static NOINLINE void base_transpose_i16(i16* rp, i16* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;ya, xsh+1, xr-1); - sh->a[xr-1] = h; - arr_shReplace(r, xr, sh); - return taga(r); - } - usz w = xsh[1] * shProd(xsh, 2, xr); - if (w==1) goto no_reorder; - - Arr* r; - usz xi = 0; - u8 xe = TI(x,elType); - bool toBit = false; - if (h==2) { - if (xe==el_B) { - B* xp = TO_BPTR(x); - B* x0 = xp; B* x1 = x0+w; - HArr_p rp = m_harrUp(ia); - for (usz i=0; ia; - for (usz i=0; i=8 && h>=8) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } - #endif - { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } - #endif - { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; } - #endif - { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;yucw = pick_ucw; c(BFn,bi_select)->ucw = select_ucw; // TODO move to new init fn c(BFn,bi_shape)->uc1 = shape_uc1; - c(BFn,bi_transp)->uc1 = transp_uc1; - c(BFn,bi_transp)->im = transp_im; c(BFn,bi_take)->ucw = take_ucw; c(BFn,bi_drop)->ucw = drop_ucw; c(BFn,bi_lt)->im = enclose_im; diff --git a/src/builtins/transpose.c b/src/builtins/transpose.c new file mode 100644 index 00000000..02a8df8c --- /dev/null +++ b/src/builtins/transpose.c @@ -0,0 +1,165 @@ +#include "../core.h" +#include "../utils/each.h" +#include "../utils/talloc.h" +#include "../builtins.h" + +#ifdef __BMI2__ + #include + #if USE_VALGRIND + #define _pdep_u64 vg_pdep_u64 + #endif +#endif + +#if SINGELI_X86_64 + static NOINLINE void base_transpose_i16(i16* rp, i16* xp, u64 w, u64 h, u64 xo, u64 ro) { PLAINLOOP for(usz y=0;ya, xsh+1, xr-1); + sh->a[xr-1] = h; + arr_shReplace(r, xr, sh); + return taga(r); + } + usz w = xsh[1] * shProd(xsh, 2, xr); + if (w==1) goto no_reorder; + + Arr* r; + usz xi = 0; + u8 xe = TI(x,elType); + bool toBit = false; + if (h==2) { + if (xe==el_B) { + B* xp = TO_BPTR(x); + B* x0 = xp; B* x1 = x0+w; + HArr_p rp = m_harrUp(ia); + for (usz i=0; ia; + for (usz i=0; i=8 && h>=8) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } + #endif + { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } + #endif + { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; } + #endif + { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;yuc1 = transp_uc1; + c(BFn,bi_transp)->im = transp_im; +} diff --git a/src/load.c b/src/load.c index d569dc90..f6485267 100644 --- a/src/load.c +++ b/src/load.c @@ -8,7 +8,7 @@ #define PRECOMPILED_FILE1(X) PRECOMPILED_FILE0(X) #define PRECOMPILED_FILE(END) PRECOMPILED_FILE1(../build/BYTECODE_DIR/gen/END) -#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished) +#define FOR_INIT(F) F(base) F(harr) F(mutF) F(cmpA) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(slash) F(search) F(transp) F(load) F(sysfnPost) F(dervPost) F(ryu) F(ffi) F(mmap) F(typesFinished) #define F(X) NOINLINE void X##_init(void); FOR_INIT(F) #undef F diff --git a/src/opt/single.c b/src/opt/single.c index 36f62ddf..382eb099 100644 --- a/src/opt/single.c +++ b/src/opt/single.c @@ -25,6 +25,7 @@ #include "../builtins/sort.c" #include "../builtins/search.c" #include "../builtins/selfsearch.c" +#include "../builtins/transpose.c" #include "../builtins/fold.c" #include "../builtins/scan.c" #include "../builtins/arithm.c" From 5fccf4cda3baa728ea6b47df35e3d77f391ce998 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 11:26:44 -0400 Subject: [PATCH 09/23] Refactoring; check for small transpose width/height in Singeli, not C --- src/builtins/transpose.c | 35 ++++++++--------- src/singeli/src/transpose.singeli | 64 +++++++++++++++++++------------ 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/builtins/transpose.c b/src/builtins/transpose.c index 02a8df8c..c3fa025b 100644 --- a/src/builtins/transpose.c +++ b/src/builtins/transpose.c @@ -10,12 +10,21 @@ #endif #endif +#define TRANSPOSE_LOOP( DST, SRC, W, H) PLAINLOOP for(usz y=0;y< H;y++) NOVECTORIZE for(usz x=0;x< W;x++) DST[x*H+y] = SRC[xi++] +#define TRANSPOSE_BLOCK(DST, SRC, BW, BH, W, H) PLAINLOOP for(usz y=0;y=8 && h>=8) { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i16(rp, xp, w, h); break; } - #endif - { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=8 && h>=8) { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); simd_transpose_i32(rp, xp, w, h); break; } - #endif - { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); PLAINLOOP for(usz y=0;y=4 && h>=4) { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); simd_transpose_i64(rp, xp, w, h); break; } - #endif - { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); PLAINLOOP for(usz y=0;y emit{void, merge{'base_transpose_',ts}, ...a} +} +def small_transpose_out{T, k, rp, xp, w, h} = { + if (wload{*VT~~(xpo+i*w), 0}, iota{k}} def rvs = vtranspose{xvs} each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} @@ -100,44 +126,36 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { } @for_mult{line_elts} (y0 to yn) { y := y0 + ro @for_mult{k} (x to w) { - xpo:= xp + y*w + x - rpo:= rp + x*h + y + {xpo,rpo} := at{x, y} def rls = get_lines{{i} => load{*VT~~(xpo+i*w), 0}} each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} } } } - def base = if (T==i32) 'base_transpose_i32' else 'base_transpose_i64' - if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} - if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} + edge_transpose{T, k, rp, xp, w, h} } -def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = { - def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} - each{bind{~~,[16]i16}, r} -} -def load2{x, y} = emit{[16]i16, '_mm256_loadu2_m128i', *[8]i16~~y, *[8]i16~~x} - -fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = { +fn transpose{T, k, m==2}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 - def d = 2*k + small_transpose_out{T, k, rp, xp, w, h} + def at = mat_at{rp,xp,w,h} + def d = m*k def VT = [d]T + def HT = [k]T @for_mult{d} (y to h) { @for_mult{k} (x to w) { - xpo:= xp + y*w + x - rpo:= rp + x*h + y - def xvs = each{{i}=>{p:=xpo+i*w; load2{p, p+k*w}}, iota{k}} + {xpo, rpo} := at{x, y} + def xvs = each{{i}=>{p:=xpo+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}} def rvs = vtranspose2{xvs} each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} } } if ((h & k) != 0) { y := h-h%d @for_mult{k} (x to w) { - xpo:= xp + y*w + x - rpo:= rp + x*h + y + {xpo, rpo} := at{x, y} def lw = k*width{T} def xvs = each{{i}=>loadLow{*VT~~(xpo+i*w), lw}, iota{k}} def rvs = vtranspose2{xvs} @@ -145,11 +163,9 @@ fn transpose2{T, k & T < i32}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - def base = 'base_transpose_i16' - if (w%k) emit{void, base, rp+h*(w-w%k), xp+ (w-w%k), w%k, h, w, h} - if (h%k) emit{void, base, rp+ (h-h%k), xp+w*(h-h%k), w-w%k, h%k, w, h} + edge_transpose{T, k, rp, xp, w, h} } -export{'simd_transpose_i16', transpose2{i16, 8}} +export{'simd_transpose_i16', transpose{i16, 8, 2}} export{'simd_transpose_i32', transpose{i32, 8}} export{'simd_transpose_i64', transpose{i64, 4}} From 7d800bf3498901dfe8dfe00e133e0e64cfd0058c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 15:17:17 -0400 Subject: [PATCH 10/23] SIMD transpose on 1-byte elements --- src/builtins/transpose.c | 4 ++-- src/singeli/src/transpose.singeli | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/builtins/transpose.c b/src/builtins/transpose.c index c3fa025b..73b39f1c 100644 --- a/src/builtins/transpose.c +++ b/src/builtins/transpose.c @@ -18,7 +18,7 @@ static NOINLINE void base_transpose_##T(T* rp, T* xp, u64 bw, u64 bh, u64 w, u64 h) { \ TRANSPOSE_BLOCK(rp, xp, bw, bh, w, h); \ } - DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64) + DECL_BASE(i8) DECL_BASE(i16) DECL_BASE(i32) DECL_BASE(i64) #undef DECL_BASE #define SINGELI_FILE transpose #include "../utils/includeSingeli.h" @@ -112,7 +112,7 @@ B transp_c1(B t, B x) { } else { switch(xe) { default: UD; case el_bit: x = taga(cpyI8Arr(x)); xsh=SH(x); xe=el_i8; toBit=true; // fallthough - case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); TRANSPOSE_LOOP( rp, xp, w, h); break; } + case el_i8: case el_c8: { u8* xp=tyany_ptr(x); u8* rp = m_tyarrp(&r,1,ia,el2t(xe)); TRANSPOSE_SIMD( i8, rp, xp, w, h); break; } case el_i16:case el_c16: { u16* xp=tyany_ptr(x); u16* rp = m_tyarrp(&r,2,ia,el2t(xe)); TRANSPOSE_SIMD(i16, rp, xp, w, h); break; } case el_i32:case el_c32: { u32* xp=tyany_ptr(x); u32* rp = m_tyarrp(&r,4,ia,el2t(xe)); TRANSPOSE_SIMD(i32, rp, xp, w, h); break; } case el_f64: { f64* xp=f64any_ptr(x); f64* rp; r=m_f64arrp(&rp,ia); TRANSPOSE_SIMD(i64, rp, xp, w, h); break; } diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 471f9ff3..c77cbba9 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -44,6 +44,9 @@ def load2{a:T, b:T & w128i{eltype{T}}} = { def V = eltype{T} emit{[2*vcount{V}](eltype{V}), '_mm256_loadu2_m128i', b, a} } +def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { + each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} +} @@ -56,7 +59,7 @@ def mat_at{rp,xp,w,h}{x,y} = tup{xp + y*w + x, rp + x*h + y} # Scalar transpose defined in C def call_base{T} = { - def ts = if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' + def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' {...a} => emit{void, merge{'base_transpose_',ts}, ...a} } def small_transpose_out{T, k, rp, xp, w, h} = { @@ -166,6 +169,31 @@ fn transpose{T, k, m==2}(r0:*void, x0:*void, w:u64, h:u64) : void = { edge_transpose{T, k, rp, xp, w, h} } +def vtranspose{x & ktest{'X86_64',8,[32]i8}{x}} = { + def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} + each{{v}=>[32]i8~~shuf{[4]i64, v, 4b3120}, r} +} +fn transpose{T==i8, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { + rp:*T = *T~~r0 + xp:*T = *T~~x0 + small_transpose_out{T, k, rp, xp, w, h} + def at = mat_at{rp,xp,w,h} + def VT = [k]T + + @for_mult{k} (y to h) { + @for_mult{k} (x to w) { + {xpo, rpo} := at{x, y} + def s = k/2 + def xvs = each{{i}=>{p:=xpo+i*w; load2{*VT~~p, *VT~~(p+s*w)}}, iota{s}} + def rvs = vtranspose{xvs} + each{{i,v}=>{p:=rpo+2*i*h; store2{*VT~~p, *VT~~(p+h), v}}, iota{s}, rvs} + } + } + + edge_transpose{T, k, rp, xp, w, h} +} + +export{'simd_transpose_i8', transpose{i8 , 16}} export{'simd_transpose_i16', transpose{i16, 8, 2}} export{'simd_transpose_i32', transpose{i32, 8}} export{'simd_transpose_i64', transpose{i64, 4}} From 2a8e6e9c8a013a6bfa2934acdae8f452a248f3c7 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 18:42:43 -0400 Subject: [PATCH 11/23] Separate most transpose kernel logic from loops and move i8 case into main --- src/singeli/src/transpose.singeli | 100 +++++++++++++++--------------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index c77cbba9..82843b76 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -40,6 +40,11 @@ def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = { def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} each{bind{~~,[16]i16}, r} } +# Transpose 16x16 packed as halves +def vtranspose{x & ktest{'X86_64',8,[32]i8}{x}} = { + def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} + each{{v}=>[32]i8~~shuf{[4]i64, v, 4b3120}, r} +} def load2{a:T, b:T & w128i{eltype{T}}} = { def V = eltype{T} emit{[2*vcount{V}](eltype{V}), '_mm256_loadu2_m128i', b, a} @@ -48,6 +53,35 @@ def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} } +def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==256} = { + def VT = [k](eltype{P}) + def xvs = each{{i}=>load{*VT~~(src+i*w), 0}, iota{k}} + def rvs = vtranspose{xvs} + each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} +} +def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==128} = { + def VT = [k](eltype{P}) + def s = k/2 + def xvs = each{{i}=>{p:=src+i*w; load2{*VT~~p, *VT~~(p+s*w)}}, iota{s}} + def rvs = vtranspose{xvs} + each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+h), v}}, iota{s}, rvs} +} +def kernel{src:P, dst:P, k, d, w, h & d==2*k & d*width{eltype{P}}==256} = { + def HT = [k](eltype{P}) + def VT = [d](eltype{P}) + def xvs = each{{i}=>{p:=src+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}} + def rvs = vtranspose2{xvs} + each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} +} +# 8x8 i16 with a wasted half: would be better to store as 4x16 +def kernel{src:P, dst:P, k==8, k, w, h & eltype{P}==i16} = { + def T = i16 + def VT = [2*k]i16 + def lw = k*width{T} + def xvs = each{{i}=>loadLow{*VT~~(src+i*w), lw}, iota{k}} + def rvs = vtranspose2{xvs} + each{{i,v}=>storeLow{*VT~~(dst+i*h), lw, v}, iota{k}, rvs} +} def for_mult{k}{vars,begin,end,block} = { @@ -57,6 +91,14 @@ def for_mult{k}{vars,begin,end,block} = { def mat_at{rp,xp,w,h}{x,y} = tup{xp + y*w + x, rp + x*h + y} +def transpose_kernels{kw, kh, rp, xp, w, h} = { + @for_mult{kh} (y to h) { + @for_mult{kw} (x to w) { + kernel{...mat_at{rp,xp,w,h}{x,y}, kw, kh, w, h} + } + } +} + # Scalar transpose defined in C def call_base{T} = { def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' @@ -75,7 +117,6 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 small_transpose_out{T, k, rp, xp, w, h} - def at = mat_at{rp,xp,w,h} def VT = [k]T # Cache line info @@ -83,15 +124,9 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { def line_elts = line_bytes / (width{T}/8) def line_vecs = line_bytes / (width{VT}/8) - if (h&(line_elts-1) != 0) { - @for_mult{k} (y to h) { - @for_mult{k} (x to w) { - {xpo,rpo} := at{x, y} - def xvs = each{{i}=>load{*VT~~(xpo+i*w), 0}, iota{k}} - def rvs = vtranspose{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} - } - } + if (line_elts > 2*k or h&(line_elts-1) != 0) { + # Main transpose + transpose_kernels{k, k, rp, xp, w, h} } else { # Result rows are aligned with each other so it's possible to # write a full cache line at a time @@ -129,7 +164,7 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { } @for_mult{line_elts} (y0 to yn) { y := y0 + ro @for_mult{k} (x to w) { - {xpo,rpo} := at{x, y} + {xpo,rpo} := mat_at{rp,xp,w,h}{x, y} def rls = get_lines{{i} => load{*VT~~(xpo+i*w), 0}} each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} } @@ -143,50 +178,13 @@ fn transpose{T, k, m==2}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 small_transpose_out{T, k, rp, xp, w, h} - def at = mat_at{rp,xp,w,h} def d = m*k - def VT = [d]T - def HT = [k]T - @for_mult{d} (y to h) { - @for_mult{k} (x to w) { - {xpo, rpo} := at{x, y} - def xvs = each{{i}=>{p:=xpo+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}} - def rvs = vtranspose2{xvs} - each{{i,v}=>store{*VT~~(rpo+i*h), 0, v}, iota{k}, rvs} - } - } + transpose_kernels{k, d, rp, xp, w, h} + if ((h & k) != 0) { y := h-h%d @for_mult{k} (x to w) { - {xpo, rpo} := at{x, y} - def lw = k*width{T} - def xvs = each{{i}=>loadLow{*VT~~(xpo+i*w), lw}, iota{k}} - def rvs = vtranspose2{xvs} - each{{i,v}=>storeLow{*VT~~(rpo+i*h), lw, v}, iota{k}, rvs} - } - } - - edge_transpose{T, k, rp, xp, w, h} -} - -def vtranspose{x & ktest{'X86_64',8,[32]i8}{x}} = { - def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} - each{{v}=>[32]i8~~shuf{[4]i64, v, 4b3120}, r} -} -fn transpose{T==i8, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { - rp:*T = *T~~r0 - xp:*T = *T~~x0 - small_transpose_out{T, k, rp, xp, w, h} - def at = mat_at{rp,xp,w,h} - def VT = [k]T - - @for_mult{k} (y to h) { - @for_mult{k} (x to w) { - {xpo, rpo} := at{x, y} - def s = k/2 - def xvs = each{{i}=>{p:=xpo+i*w; load2{*VT~~p, *VT~~(p+s*w)}}, iota{s}} - def rvs = vtranspose{xvs} - each{{i,v}=>{p:=rpo+2*i*h; store2{*VT~~p, *VT~~(p+h), v}}, iota{s}, rvs} + kernel{...mat_at{rp,xp,w,h}{x,y}, k, k, w, h} } } From 923c485cc2881cf6806e7d8a55a2d645452d5eb4 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 18:54:46 -0400 Subject: [PATCH 12/23] Use packed-halves transpose kernel for 8x8 i16 --- src/singeli/src/transpose.singeli | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 82843b76..ae41b891 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -40,11 +40,16 @@ def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = { def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} each{bind{~~,[16]i16}, r} } -# Transpose 16x16 packed as halves +# Transpose square packed as halves def vtranspose{x & ktest{'X86_64',8,[32]i8}{x}} = { def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} each{{v}=>[32]i8~~shuf{[4]i64, v, 4b3120}, r} } +def vtranspose{x & ktest{'X86_64',4,[16]i16}{x}} = { + def r = unpack_pass{2, unpack_pass{1, x}} + each{{v}=>[16]i16~~shuf{[4]i64, v, 4b3120}, r} +} + def load2{a:T, b:T & w128i{eltype{T}}} = { def V = eltype{T} emit{[2*vcount{V}](eltype{V}), '_mm256_loadu2_m128i', b, a} @@ -73,15 +78,6 @@ def kernel{src:P, dst:P, k, d, w, h & d==2*k & d*width{eltype{P}}==256} = { def rvs = vtranspose2{xvs} each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} } -# 8x8 i16 with a wasted half: would be better to store as 4x16 -def kernel{src:P, dst:P, k==8, k, w, h & eltype{P}==i16} = { - def T = i16 - def VT = [2*k]i16 - def lw = k*width{T} - def xvs = each{{i}=>loadLow{*VT~~(src+i*w), lw}, iota{k}} - def rvs = vtranspose2{xvs} - each{{i,v}=>storeLow{*VT~~(dst+i*h), lw, v}, iota{k}, rvs} -} def for_mult{k}{vars,begin,end,block} = { From e40feaa81cdc3e9aa86877124c360766a76905bd Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 19:17:00 -0400 Subject: [PATCH 13/23] Make transpose kernel code type-generic --- src/singeli/src/transpose.singeli | 46 ++++++++++++++----------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index ae41b891..538b89e6 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -19,35 +19,31 @@ def pairs{o, x} = { tupsel{tup{g, g+o}, x} } def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} +def unpack_to{l, x} = unpack_pass{l, if (l==1) x else unpack_to{l/2, x}} def permute_pass{o, x} = { def p = pairs{o, x} def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p} merge{h{16b20}, h{16b31}} } -def ktest{a,l,T}{x} = { - if (hasarch{a} and tuplen{x}==l and type{tupsel{0,x}}==T) 1 else 0 +# Square kernel where width is a full vector +def transpose_square{x & hasarch{'X86_64'}} = { + def k = tuplen{x} + def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{k==vcount{T}} + def rvs = permute_pass{k/2, unpack_to{k/4, x}} } - -def vtranspose{x & ktest{'X86_64',8,[8]i32}{x}} = { - permute_pass{4, unpack_pass{2, unpack_pass{1, x}}} +# Square kernel where width is half a vector; top half next to bottom +def transpose_square_halves{x & hasarch{'X86_64'}} = { + def l = tuplen{x} + def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{l==vcount{T}/4} + def r = unpack_to{l/2, x} + each{{v} => T~~shuf{[4]i64, v, 4b3120}, r} } -def vtranspose{x & ktest{'X86_64',4,[4]i64}{x}} = { - permute_pass{2, unpack_pass{1, x}} -} - -def vtranspose2{x & ktest{'X86_64',8,[16]i16}{x}} = { - def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} - each{bind{~~,[16]i16}, r} -} -# Transpose square packed as halves -def vtranspose{x & ktest{'X86_64',8,[32]i8}{x}} = { - def r = unpack_pass{4, unpack_pass{2, unpack_pass{1, x}}} - each{{v}=>[32]i8~~shuf{[4]i64, v, 4b3120}, r} -} -def vtranspose{x & ktest{'X86_64',4,[16]i16}{x}} = { - def r = unpack_pass{2, unpack_pass{1, x}} - each{{v}=>[16]i16~~shuf{[4]i64, v, 4b3120}, r} +# Same with 2*k by k rectangle +def transpose_rect_halves{x & hasarch{'X86_64'}} = { + def l = tuplen{x} + def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{l==vcount{T}/2} + each{bind{~~,T}, unpack_to{l/2, x}} } def load2{a:T, b:T & w128i{eltype{T}}} = { @@ -61,21 +57,21 @@ def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==256} = { def VT = [k](eltype{P}) def xvs = each{{i}=>load{*VT~~(src+i*w), 0}, iota{k}} - def rvs = vtranspose{xvs} + def rvs = transpose_square{xvs} each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} } def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==128} = { def VT = [k](eltype{P}) def s = k/2 def xvs = each{{i}=>{p:=src+i*w; load2{*VT~~p, *VT~~(p+s*w)}}, iota{s}} - def rvs = vtranspose{xvs} + def rvs = transpose_square_halves{xvs} each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+h), v}}, iota{s}, rvs} } def kernel{src:P, dst:P, k, d, w, h & d==2*k & d*width{eltype{P}}==256} = { def HT = [k](eltype{P}) def VT = [d](eltype{P}) def xvs = each{{i}=>{p:=src+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}} - def rvs = vtranspose2{xvs} + def rvs = transpose_rect_halves{xvs} each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} } @@ -130,7 +126,7 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { # at multiples of 256 or so, but it's faster whenever it applies def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs} def get_lines{loadx} = { - def vt{i} = vtranspose{each{loadx, k*i + iota{k}}} + def vt{i} = transpose_square{each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}} } ro := tail{6, -u64~~r0} / (width{T}/8) # Offset to align within cache line; assume elt-aligned From 967377118725db1dbbad2b444127f999fcb9cfaa Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 21:52:31 -0400 Subject: [PATCH 14/23] Combine code for the different transpose kernel geometries --- src/singeli/src/transpose.singeli | 59 +++++++++++-------------------- 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 538b89e6..80f66679 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -19,32 +19,20 @@ def pairs{o, x} = { tupsel{tup{g, g+o}, x} } def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}} -def unpack_to{l, x} = unpack_pass{l, if (l==1) x else unpack_to{l/2, x}} def permute_pass{o, x} = { def p = pairs{o, x} def h{s} = each{{a,b}=>emit{[8]i32, '_mm256_permute2f128_si256', a,b,s}, ...p} merge{h{16b20}, h{16b31}} } +def unpack_to{f, l, x} = { + def pass = if (f) permute_pass else unpack_pass + pass{l, if (l==1) x else unpack_to{0, l/2, x}} +} +# Last pass for square kernel packed in halves +def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} # Square kernel where width is a full vector -def transpose_square{x & hasarch{'X86_64'}} = { - def k = tuplen{x} - def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{k==vcount{T}} - def rvs = permute_pass{k/2, unpack_to{k/4, x}} -} -# Square kernel where width is half a vector; top half next to bottom -def transpose_square_halves{x & hasarch{'X86_64'}} = { - def l = tuplen{x} - def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{l==vcount{T}/4} - def r = unpack_to{l/2, x} - each{{v} => T~~shuf{[4]i64, v, 4b3120}, r} -} -# Same with 2*k by k rectangle -def transpose_rect_halves{x & hasarch{'X86_64'}} = { - def l = tuplen{x} - def T = type{tupsel{0,x}}; assert{256==width{T}}; assert{l==vcount{T}/2} - each{bind{~~,T}, unpack_to{l/2, x}} -} +def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x} def load2{a:T, b:T & w128i{eltype{T}}} = { def V = eltype{T} @@ -53,26 +41,19 @@ def load2{a:T, b:T & w128i{eltype{T}}} = { def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} } +def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} +def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{l}, x} +def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} +def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} -def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==256} = { - def VT = [k](eltype{P}) - def xvs = each{{i}=>load{*VT~~(src+i*w), 0}, iota{k}} - def rvs = transpose_square{xvs} - each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} -} -def kernel{src:P, dst:P, k, k, w, h & k*width{eltype{P}}==128} = { - def VT = [k](eltype{P}) - def s = k/2 - def xvs = each{{i}=>{p:=src+i*w; load2{*VT~~p, *VT~~(p+s*w)}}, iota{s}} - def rvs = transpose_square_halves{xvs} - each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+h), v}}, iota{s}, rvs} -} -def kernel{src:P, dst:P, k, d, w, h & d==2*k & d*width{eltype{P}}==256} = { - def HT = [k](eltype{P}) - def VT = [d](eltype{P}) - def xvs = each{{i}=>{p:=src+i*w; load2{*HT~~p, *HT~~(p+k*w)}}, iota{k}} - def rvs = transpose_rect_halves{xvs} - each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{k}, rvs} +# Transpose kernel of size kw,kh in size w,h array +def kernel{src:P, dst:P, kw, kh, w, h} = { + def T = eltype{P} + def n = (kw*kh*width{T}) / 256 # Number of vectors + def xvs = load_k{[kw]T, src, n, w} + def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n + def rvs = if (n==kw) xt else shuf_pass{xt} # To kh by kh for packed square + store_k{[kh]T, dst, rvs, n, h} } @@ -126,7 +107,7 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { # at multiples of 256 or so, but it's faster whenever it applies def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs} def get_lines{loadx} = { - def vt{i} = transpose_square{each{loadx, k*i + iota{k}}} + def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}} } ro := tail{6, -u64~~r0} / (width{T}/8) # Offset to align within cache line; assume elt-aligned From dd141add3f5b5db31661df6fc3d764e30292291f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 22:01:21 -0400 Subject: [PATCH 15/23] Merge i16 SIMD transpose with the rest of them --- src/singeli/src/transpose.singeli | 34 ++++++++++++------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 80f66679..a4ebf416 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -86,25 +86,32 @@ def edge_transpose{T, k, rp, xp, w, h} = { ho := h%k; hs := h-ho; if (ho) tr{rp+ hs, xp+w*hs, ws, ho} } -fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { +fn transpose{T, k, d}(r0:*void, x0:*void, w:u64, h:u64) : void = { rp:*T = *T~~r0 xp:*T = *T~~x0 small_transpose_out{T, k, rp, xp, w, h} - def VT = [k]T # Cache line info def line_bytes = 64 def line_elts = line_bytes / (width{T}/8) - def line_vecs = line_bytes / (width{VT}/8) if (line_elts > 2*k or h&(line_elts-1) != 0) { # Main transpose - transpose_kernels{k, k, rp, xp, w, h} + transpose_kernels{k, d, rp, xp, w, h} + # Extra column for uneven i16 case + if (2*k == d and (h & k) != 0) { y := h-h%d + @for_mult{k} (x to w) { + kernel{...mat_at{rp,xp,w,h}{x,y}, k, k, w, h} + } + } } else { # Result rows are aligned with each other so it's possible to # write a full cache line at a time # This case is here to mitigate cache associativity problems at # at multiples of 256 or so, but it's faster whenever it applies + assert{k == d} + def VT = [k]T + def line_vecs = line_bytes / (width{VT}/8) def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs} def get_lines{loadx} = { def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}} @@ -147,24 +154,9 @@ fn transpose{T, k}(r0:*void, x0:*void, w:u64, h:u64) : void = { edge_transpose{T, k, rp, xp, w, h} } -fn transpose{T, k, m==2}(r0:*void, x0:*void, w:u64, h:u64) : void = { - rp:*T = *T~~r0 - xp:*T = *T~~x0 - small_transpose_out{T, k, rp, xp, w, h} - def d = m*k - - transpose_kernels{k, d, rp, xp, w, h} - - if ((h & k) != 0) { y := h-h%d - @for_mult{k} (x to w) { - kernel{...mat_at{rp,xp,w,h}{x,y}, k, k, w, h} - } - } - - edge_transpose{T, k, rp, xp, w, h} -} +def transpose{T, k} = transpose{T, k, k} export{'simd_transpose_i8', transpose{i8 , 16}} -export{'simd_transpose_i16', transpose{i16, 8, 2}} +export{'simd_transpose_i16', transpose{i16, 8, 16}} export{'simd_transpose_i32', transpose{i32, 8}} export{'simd_transpose_i64', transpose{i64, 4}} From fad7f3aa8b9bd264c2246cf4b19e43a3aab5f082 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 21 Mar 2023 22:04:51 -0400 Subject: [PATCH 16/23] Simplify --- src/singeli/src/transpose.singeli | 52 ++++++++++++------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index a4ebf416..e13bf7d5 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -62,34 +62,16 @@ def for_mult{k}{vars,begin,end,block} = { @for (i to end/k) exec{k*i, vars, block} } -def mat_at{rp,xp,w,h}{x,y} = tup{xp + y*w + x, rp + x*h + y} - -def transpose_kernels{kw, kh, rp, xp, w, h} = { - @for_mult{kh} (y to h) { - @for_mult{kw} (x to w) { - kernel{...mat_at{rp,xp,w,h}{x,y}, kw, kh, w, h} - } - } -} - -# Scalar transpose defined in C -def call_base{T} = { +fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { + # Scalar transpose defined in C def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' - {...a} => emit{void, merge{'base_transpose_',ts}, ...a} -} -def small_transpose_out{T, k, rp, xp, w, h} = { - if (w 2*k or h&(line_elts-1) != 0) { # Main transpose - transpose_kernels{k, d, rp, xp, w, h} - # Extra column for uneven i16 case - if (2*k == d and (h & k) != 0) { y := h-h%d + @for_mult{kh} (y to h) { @for_mult{k} (x to w) { - kernel{...mat_at{rp,xp,w,h}{x,y}, k, k, w, h} + kernel{...at{x,y}, k, kh, w, h} + } + } + # Extra row for uneven i16 case + if (2*k == kh and (h & k) != 0) { y := h-h%kh + @for_mult{k} (x to w) { + kernel{...at{x,y}, k, k, w, h} } } } else { @@ -109,7 +95,7 @@ fn transpose{T, k, d}(r0:*void, x0:*void, w:u64, h:u64) : void = { # write a full cache line at a time # This case is here to mitigate cache associativity problems at # at multiples of 256 or so, but it's faster whenever it applies - assert{k == d} + assert{k == kh} def VT = [k]T def line_vecs = line_bytes / (width{VT}/8) def store_line{p, vs} = each{bind{store,p}, iota{line_vecs}, vs} @@ -144,14 +130,16 @@ fn transpose{T, k, d}(r0:*void, x0:*void, w:u64, h:u64) : void = { } @for_mult{line_elts} (y0 to yn) { y := y0 + ro @for_mult{k} (x to w) { - {xpo,rpo} := mat_at{rp,xp,w,h}{x, y} + {xpo,rpo} := at{x, y} def rls = get_lines{{i} => load{*VT~~(xpo+i*w), 0}} each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} } } } - edge_transpose{T, k, rp, xp, w, h} + def edge_tr{...a} = call_base{...a, w, h} + wo := w%k; ws := w-wo; if (wo) edge_tr{rp+h*ws, xp+ ws, wo, h } + ho := h%k; hs := h-ho; if (ho) edge_tr{rp+ hs, xp+w*hs, ws, ho} } def transpose{T, k} = transpose{T, k, k} From 8ff40cb2e58d62741b8d33dc6387ee1632c156b5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 22 Mar 2023 11:35:57 -0400 Subject: [PATCH 17/23] Overlap SIMD transpose to handle uneven heights (not widths yet) --- src/singeli/src/transpose.singeli | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index e13bf7d5..c01a3329 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -65,31 +65,43 @@ def for_mult{k}{vars,begin,end,block} = { fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { # Scalar transpose defined in C def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' - def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a} + def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a, w, h} rp:*T = *T~~r0 xp:*T = *T~~x0 - if (w= 2 # For overlapped SIMD instead of scalar + # Handle uneven height (extra rows) here, but not uneven width if (line_elts > 2*k or h&(line_elts-1) != 0) { + ho := h%k + # Effective height: number of rows read, counting overlap twice + # Just use base transpose for short overhang; otherwise round up + he := h; if (use_overlap{ho}) he += k - ho # Main transpose - @for_mult{kh} (y to h) { + hm := h - kh + @for_mult{kh} (y0 to he) { y:=y0; if (y>hm) y = hm @for_mult{k} (x to w) { kernel{...at{x,y}, k, kh, w, h} } } - # Extra row for uneven i16 case - if (2*k == kh and (h & k) != 0) { y := h-h%kh + # Half-row for non-square i16 case + if (2*k == kh and (he & k) != 0) { + e := h%kh; if (e Date: Wed, 22 Mar 2023 13:58:54 -0400 Subject: [PATCH 18/23] Fix writing past end in i16 transpose with overlapped halves --- src/singeli/src/transpose.singeli | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index c01a3329..33c4148e 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -85,6 +85,8 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { # Effective height: number of rows read, counting overlap twice # Just use base transpose for short overhang; otherwise round up he := h; if (use_overlap{ho}) he += k - ho + def has_half = 2*k == kh + if (has_half and he==kh and hhm) y = hm @@ -92,12 +94,15 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { kernel{...at{x,y}, k, kh, w, h} } } - # Half-row for non-square i16 case - if (2*k == kh and (he & k) != 0) { - e := h%kh; if (e Date: Wed, 22 Mar 2023 15:13:18 -0400 Subject: [PATCH 19/23] Overlap SIMD transpose on width too --- src/singeli/src/transpose.singeli | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 33c4148e..2ac55e64 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -61,6 +61,12 @@ def for_mult{k}{vars,begin,end,block} = { assert{begin == 0} @for (i to end/k) exec{k*i, vars, block} } +def for_mult_max{k, m}{vars,begin,end,block} = { + @for_mult{k} (i0 to end) { + i:=i0; if (i>m) i = m + exec{i, vars, block} + } +} fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { # Scalar transpose defined in C @@ -78,19 +84,21 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { def line_elts = line_bytes / (width{T}/8) def use_overlap{o} = o >= 2 # For overlapped SIMD instead of scalar + wo := w%k + # Effective width: number of columns read, counting overlap twice + # Just use base transpose for short overhang; otherwise round up + we := w; if (use_overlap{wo}) we += k - wo + wm := w - k - # Handle uneven height (extra rows) here, but not uneven width if (line_elts > 2*k or h&(line_elts-1) != 0) { ho := h%k - # Effective height: number of rows read, counting overlap twice - # Just use base transpose for short overhang; otherwise round up + # Effective height, like we for w he := h; if (use_overlap{ho}) he += k - ho def has_half = 2*k == kh if (has_half and he==kh and hhm) y = hm - @for_mult{k} (x to w) { + @for_mult_max{kh, h-kh} (y to he) { + @for_mult_max{k, wm} (x to we) { kernel{...at{x,y}, k, kh, w, h} } } @@ -100,13 +108,13 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { e := h%kh; if (he {if (i<3 or rpo {p:=rpo+i*h; if (i<3 or p load{*VT~~(xpo+i*w), 0}} each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} @@ -154,7 +162,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - wo := w%k; if (wo!=0) { ws := w-wo; call_base{rp+h*ws, xp+ws, wo, h } } + if (wo!=0 and we==w) { ws := w-wo; call_base{rp+h*ws, xp+ws, wo, h } } } def transpose{T, k} = transpose{T, k, k} From 699faea69cc4d6bf12d081f4c0d1e7e7d237d770 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 22 Mar 2023 17:30:02 -0400 Subject: [PATCH 20/23] Dedicated width-overflow loop for SIMD transpose --- src/singeli/src/transpose.singeli | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 2ac55e64..aa196453 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -162,7 +162,10 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - if (wo!=0 and we==w) { ws := w-wo; call_base{rp+h*ws, xp+ws, wo, h } } + if (we==w) @for(ws from w-wo to w) { + xpo:=xp+ws; rpo:=rp+h*ws + @for (i to h) store{rpo, i, load{xpo, w*i}} + } } def transpose{T, k} = transpose{T, k, k} From 662fd83c406899217e3a6303fcf64bbb3adf184c Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 23 Mar 2023 17:58:50 +0200 Subject: [PATCH 21/23] fix typo --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index dd07b8c9..c00dfcb5 100644 --- a/makefile +++ b/makefile @@ -283,7 +283,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o tranpose.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o transpose.o fold.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< From 3178241d9a3e91dd2c519e5b419523bb071564ca Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 23 Mar 2023 18:08:28 +0200 Subject: [PATCH 22/23] update Singeli, move "make forbuild" to "make for-build" --- build/build | 8 ++++---- build/singeliSubmodule | 2 +- makefile | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build/build b/build/build index 997d6ec9..3f5b4cc6 100755 --- a/build/build +++ b/build/build @@ -1,17 +1,17 @@ #!/usr/bin/env bash DIR="$(dirname $0)" -PREV_BQN="$DIR/obj2/for_build" +PREV_BQN="$DIR/obj2/for_build2" if [ ! -f "$PREV_BQN" ]; then TMP_BQN="$DIR/obj2/for_build_tmp" - if make -C "$DIR/.." forbuild OUTPUT="$TMP_BQN"; then + if make -C "$DIR/.." for-build OUTPUT="$TMP_BQN"; then if [ "4" = "$("$TMP_BQN" -p "2+2")" ]; then true else - echo "bootstrap CBQN failed to return expected result; run 'make forbuild' with necessary configuration manually" + echo "automatically-built bootstrap CBQN doesn't behave as expected; run 'make for-build' with necessary configuration manually" exit 1 fi else - echo "failed to build bootstrap CBQN; run 'make forbuild' with necessary configuration manually" + echo "failed to build bootstrap CBQN; run 'make for-build' with necessary configuration manually" exit 1 fi diff --git a/build/singeliSubmodule b/build/singeliSubmodule index 2b4e3b8c..a17f83b2 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit 2b4e3b8cdf6f3a5f8ddf383a8421f012550b8c27 +Subproject commit a17f83b2db2c1e025a56af0401682c45eb883257 diff --git a/makefile b/makefile index c00dfcb5..36fc3b1b 100644 --- a/makefile +++ b/makefile @@ -37,8 +37,8 @@ shared-o3: @"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 i_t=shared_o3 i_f="-O3" run_incremental_0 shared-c: @"${MAKE}" i_OUTPUT=libcbqn.so i_SHARED=1 custom=1 run_incremental_0 -forbuild: - @"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build run_incremental_0 +for-build: + @"${MAKE}" i_singeli=0 i_CC=cc i_t=forbuild i_f="-O2 -DFOR_BUILD" i_FFI=0 i_OUTPUT=build/obj2/for_build2 run_incremental_0 for-bootstrap: @"${MAKE}" i_t=for_bootstrap i_f='-DNATIVE_COMPILER -DONLY_NATIVE_COMP -DFORMATTER=0 -DNO_RT -DNO_EXPLAIN' run_incremental_0 i_USE_BC_SUBMODULE=0 BYTECODE_DIR=bytecodeNone c: From fa8437ca25532312e7e2aa3ed6956225903e7b19 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 23 Mar 2023 18:21:34 +0200 Subject: [PATCH 23/23] don't use _mm256_loadu2_m128i doesn't exist on old gcc --- src/singeli/src/transpose.singeli | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index aa196453..bbdb9057 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -34,10 +34,7 @@ def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} # Square kernel where width is a full vector def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x} -def load2{a:T, b:T & w128i{eltype{T}}} = { - def V = eltype{T} - emit{[2*vcount{V}](eltype{V}), '_mm256_loadu2_m128i', b, a} -} +def load2{a:T, b:T} = pair{load{a}, load{b}} def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} }