From 5a2bc15f2a3b68d53142e6653f85e9836193604b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 22 Mar 2023 15:13:18 -0400 Subject: [PATCH] Overlap SIMD transpose on width too --- src/singeli/src/transpose.singeli | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 33c4148e..2ac55e64 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -61,6 +61,12 @@ def for_mult{k}{vars,begin,end,block} = { assert{begin == 0} @for (i to end/k) exec{k*i, vars, block} } +def for_mult_max{k, m}{vars,begin,end,block} = { + @for_mult{k} (i0 to end) { + i:=i0; if (i>m) i = m + exec{i, vars, block} + } +} fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { # Scalar transpose defined in C @@ -78,19 +84,21 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { def line_elts = line_bytes / (width{T}/8) def use_overlap{o} = o >= 2 # For overlapped SIMD instead of scalar + wo := w%k + # Effective width: number of columns read, counting overlap twice + # Just use base transpose for short overhang; otherwise round up + we := w; if (use_overlap{wo}) we += k - wo + wm := w - k - # Handle uneven height (extra rows) here, but not uneven width if (line_elts > 2*k or h&(line_elts-1) != 0) { ho := h%k - # Effective height: number of rows read, counting overlap twice - # Just use base transpose for short overhang; otherwise round up + # Effective height, like we for w he := h; if (use_overlap{ho}) he += k - ho def has_half = 2*k == kh if (has_half and he==kh and hhm) y = hm - @for_mult{k} (x to w) { + @for_mult_max{kh, h-kh} (y to he) { + @for_mult_max{k, wm} (x to we) { kernel{...at{x,y}, k, kh, w, h} } } @@ -100,13 +108,13 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { e := h%kh; if (he {if (i<3 or rpo {p:=rpo+i*h; if (i<3 or p load{*VT~~(xpo+i*w), 0}} each{{i,v} => store_line{*VT~~(rpo+i*h), v}, iota{k}, rls} @@ -154,7 +162,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { } } - wo := w%k; if (wo!=0) { ws := w-wo; call_base{rp+h*ws, xp+ws, wo, h } } + if (wo!=0 and we==w) { ws := w-wo; call_base{rp+h*ws, xp+ws, wo, h } } } def transpose{T, k} = transpose{T, k, k}