From db16b8fa6e4e85bc8d24bf6b8e82510db6febf8e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 6 Nov 2024 14:52:18 -0500 Subject: [PATCH] =?UTF-8?q?Fix=20dropped=20high=20bit=20in=20=E2=8A=A3?= =?UTF-8?q?=CB=9D=CB=98=20due=20to=20signed=20rather=20than=20unsigned=20s?= =?UTF-8?q?aturating=20pack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/fold.singeli | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 16a2064f..4b5f1073 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -79,10 +79,9 @@ def extract_column_pow2{T, x0, r0, nv, k} = { xv := *V~~x0 @for (r in *V~~r0 over i to nv) { xs := each{load{xv, .}, iota{k}} - def unzip0 = if (not hasarch{'X86_64'}) { + def unzip0{w} = if (not hasarch{'X86_64'}) { unzip{..., 0} # Sane instruction set } else { - def w = width{T} if (w <= 16) { # Pack instructions m := make{V, - (iota{vcount{V}}%k == 0)} @@ -98,7 +97,19 @@ def extract_column_pow2{T, x0, r0, nv, k} = { {...ab} => sh{ab} } } - r = tree_fold{unzip0, xs} + if (not hasarch{'X86_64'} or T != u16 or hasarch{'SSE4.1'}) { + r = tree_fold{unzip0{width{T}}, xs} + } else { + # No unsigned saturation: sign-extend then use unsigned + def D = [4]i32 + def f = tree_fold{unzip0{32}, .} + def proc{hx} = { + ri := D~~f{hx} + top := D**(1<<15); m := D**(1<<16 - 1) + (ri & m) | (D~~(ri&top == top) &~ m) + } + r = V~~packQ{...each{proc, split{k/2, xs}}} + } if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom def tr{E,a, r} = shuf{[1< 4 @@ -234,7 +245,7 @@ fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = { } # 10 loops: i8 2,4,8,16; i16 2,4,8; i32 2,4; i64 2 @unroll (ek to 4) if (el == ek) { - def T = ty_s{8<