From 791e7d122faeea9213053d32253a3d3b1366a308 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 15 Dec 2022 21:44:48 +0200 Subject: [PATCH] =?UTF-8?q?ucvt=E2=86=92narrow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/avx2.singeli | 19 ++++++++----------- src/singeli/src/base.singeli | 3 ++- src/singeli/src/mask.singeli | 5 +++-- src/singeli/src/squeeze.singeli | 10 ++++++---- src/singeli/src/sse3.singeli | 3 +++ 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 46107bd4..63817db6 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -156,23 +156,20 @@ def widen{T==[ 4]u64, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; de def widen{T==[ 4]u64, x:X & X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X & X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x} def widen{T==[ 4]u64, x:X & X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x} -# undo cvt; assumes elements of X fit in T, otherwise may be saturated or otherwise corrupted; but keeps the width of x, with trailing elements undefined -def ucvt{T, x:X & w256i{X,32} & width{T}==8} = { +def narrow{T, x:X & w256i{X,32} & width{T}==8} = { a:= packQ{x, x} b:= packQ{a, a} to_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}} } -def ucvt{T, x:X & w256i{X,32} & width{T}==16} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} -def ucvt{T, x:X & w256i{X,16} & width{T}== 8} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X & w256i{X,32} & width{T}==16} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X & w256i{X,16} & width{T}== 8} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} -def ucvt{T, x:X & w256f{X,64} & T>1<<2) | (iota{32}&1)}}} -def ucvt{T, x:X & w256u{X,64} & T== u8} = to_el{T, sel{[16]i8, ucvt{u32,x}, make{[32]i8, 4*iota{32}}}} - -def ucvt{T, x:X & w256{X} & width{T}==elwidth{X}} = to_el{T, x} # TODO check for not being f64/i64 +def narrow{T, x:X & w256u{X,64} & T==u32} = to_el{T, sel{[8]i32, x, make{[8]i32, 2*iota{8}}}} +def narrow{T, x:X & w256u{X,64} & T==u16} = to_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}} +def narrow{T, x:X & w256u{X,64} & T== u8} = to_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}} def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index a1bbd756..f790b731 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -85,7 +85,8 @@ def ty_dbl {T & isvec{T}} = to_el{ty_dbl {eltype{T}}, T} def ty_half{T & isvec{T}} = to_el{ty_half{eltype{T}}, T} # base vector utility definitions -def widen{T, x:X & T==X} = x +def widen {T, x:X & T==X} = x +def narrow{T, x:X & T==eltype{X}} = x def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def f{i,l & i==l} = 0 diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 650dc690..64c43f27 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -44,9 +44,10 @@ def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n def storeBatch{ptr:P, n, x:T, M} = { def rpos = ptr + n*vcount{T} def E0 = eltype{P} - xu:= ucvt{E0, x} def TF = to_el{E0, T} - if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, xu} + xu:= narrow{E0, x} + + if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, undefPromote{TF, xu}} else storeLow{rpos, vcount{T}*width{E0}, xu} } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 1cbc74b0..1795fac6 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -97,18 +97,20 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { maskedLoop{bulk, len, {i, M} => { v0:= loadBatch{xp, i, XV} def toint{x:T & isint{eltype{T}}} = x - def toint{flt:T & X==f64} = { - int:= cvt2{i32, flt} + def toint{flt:T & isfloat{eltype{T}}} = { + int:= narrow{i32, flt} + assert{width{type{int}} == width{XV}/2} # we'll be doing operations over it def conv{x} = tern{preserve_negative_zero, ty_u{x}, x} - if (anynePositive{conv{flt}, conv{cvt2{f64, int}}, M}) { # is any not an integer + if (anynePositive{conv{flt}, conv{widen{T, int}}, M}) { # is any not an integer if (B) { iCont=i; goto{case_B} } # if B, need to give an even more special result else return{0xffff_ffff} # float } int } - v1:= toint{v0} + assert{XV == type{v0}} + v1:= toint{v0} # TODO do with an if instead r1|= M{((EV ** ~E~~1) & EV~~v1) ^ EV~~(v1 >> (width{X}-1))} }} diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index a0dbf1d1..8d38044d 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -162,6 +162,9 @@ def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} +def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} + # blend by sign bit; no 16-bit case def blend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} def blend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}