From 8ff40cb2e58d62741b8d33dc6387ee1632c156b5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 22 Mar 2023 11:35:57 -0400 Subject: [PATCH] Overlap SIMD transpose to handle uneven heights (not widths yet) --- src/singeli/src/transpose.singeli | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index e13bf7d5..c01a3329 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -65,31 +65,43 @@ def for_mult{k}{vars,begin,end,block} = { fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64) : void = { # Scalar transpose defined in C def ts = if (T==i8) 'i8' else if (T==i16) 'i16' else if (T==i32) 'i32' else 'i64' - def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a} + def call_base{...a} = emit{void, merge{'base_transpose_',ts}, ...a, w, h} rp:*T = *T~~r0 xp:*T = *T~~x0 - if (w= 2 # For overlapped SIMD instead of scalar + # Handle uneven height (extra rows) here, but not uneven width if (line_elts > 2*k or h&(line_elts-1) != 0) { + ho := h%k + # Effective height: number of rows read, counting overlap twice + # Just use base transpose for short overhang; otherwise round up + he := h; if (use_overlap{ho}) he += k - ho # Main transpose - @for_mult{kh} (y to h) { + hm := h - kh + @for_mult{kh} (y0 to he) { y:=y0; if (y>hm) y = hm @for_mult{k} (x to w) { kernel{...at{x,y}, k, kh, w, h} } } - # Extra row for uneven i16 case - if (2*k == kh and (h & k) != 0) { y := h-h%kh + # Half-row for non-square i16 case + if (2*k == kh and (he & k) != 0) { + e := h%kh; if (e