Fix dropped high bit in ⊣˝˘ due to signed rather than unsigned saturating pack
This commit is contained in:
parent
67d3fe3f81
commit
db16b8fa6e
@ -79,10 +79,9 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
|
|||||||
xv := *V~~x0
|
xv := *V~~x0
|
||||||
@for (r in *V~~r0 over i to nv) {
|
@for (r in *V~~r0 over i to nv) {
|
||||||
xs := each{load{xv, .}, iota{k}}
|
xs := each{load{xv, .}, iota{k}}
|
||||||
def unzip0 = if (not hasarch{'X86_64'}) {
|
def unzip0{w} = if (not hasarch{'X86_64'}) {
|
||||||
unzip{..., 0} # Sane instruction set
|
unzip{..., 0} # Sane instruction set
|
||||||
} else {
|
} else {
|
||||||
def w = width{T}
|
|
||||||
if (w <= 16) {
|
if (w <= 16) {
|
||||||
# Pack instructions
|
# Pack instructions
|
||||||
m := make{V, - (iota{vcount{V}}%k == 0)}
|
m := make{V, - (iota{vcount{V}}%k == 0)}
|
||||||
@ -98,7 +97,19 @@ def extract_column_pow2{T, x0, r0, nv, k} = {
|
|||||||
{...ab} => sh{ab}
|
{...ab} => sh{ab}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
r = tree_fold{unzip0, xs}
|
if (not hasarch{'X86_64'} or T != u16 or hasarch{'SSE4.1'}) {
|
||||||
|
r = tree_fold{unzip0{width{T}}, xs}
|
||||||
|
} else {
|
||||||
|
# No unsigned saturation: sign-extend then use unsigned
|
||||||
|
def D = [4]i32
|
||||||
|
def f = tree_fold{unzip0{32}, .}
|
||||||
|
def proc{hx} = {
|
||||||
|
ri := D~~f{hx}
|
||||||
|
top := D**(1<<15); m := D**(1<<16 - 1)
|
||||||
|
(ri & m) | (D~~(ri&top == top) &~ m)
|
||||||
|
}
|
||||||
|
r = V~~packQ{...each{proc, split{k/2, xs}}}
|
||||||
|
}
|
||||||
if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom
|
if (width{V} > 128) { # Lane axis wasn't packed, need to shuffle to bottom
|
||||||
def tr{E,a, r} = shuf{[1<<a]E, r, tr_iota{shiftright{a-1, iota{a}}}}
|
def tr{E,a, r} = shuf{[1<<a]E, r, tr_iota{shiftright{a-1, iota{a}}}}
|
||||||
def lc = k > 4
|
def lc = k > 4
|
||||||
@ -234,7 +245,7 @@ fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = {
|
|||||||
}
|
}
|
||||||
# 10 loops: i8 2,4,8,16; i16 2,4,8; i32 2,4; i64 2
|
# 10 loops: i8 2,4,8,16; i16 2,4,8; i32 2,4; i64 2
|
||||||
@unroll (ek to 4) if (el == ek) {
|
@unroll (ek to 4) if (el == ek) {
|
||||||
def T = ty_s{8<<ek}
|
def T = ty_u{8<<ek}
|
||||||
@unroll (p from 1 to 5-ek) try_unzip{T, 1<<p}
|
@unroll (p from 1 to 5-ek) try_unzip{T, 1<<p}
|
||||||
}
|
}
|
||||||
return{0}
|
return{0}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user