diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index f27c7878..91064a3d 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -102,6 +102,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion def half{x:T, i & w256{T} & knum{i}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} +def half{x:T, i==0 & w256{T}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} def pair{x} = pair{tupsel{0,x},tupsel{1,x}} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 468f0dff..57d4214b 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -107,8 +107,8 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}} def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x} -def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} -def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} +def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} +def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m} def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} @@ -186,4 +186,4 @@ def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO che def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} -def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} \ No newline at end of file +def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index c7c41d53..67f59d26 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -24,6 +24,13 @@ def wrapChk{cw0, VI,xlf, M} = { cw } +def storeExp{dst, ind, val, M, ext, rd, wl} = { + def s{M} = storeBatch{dst, ind, val, M} + if (ext==1 or not M{0}) s{M} + else if (ind*rd+rd <= wl) s{maskNone} + else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } +} + def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { def VI = [ri]TI def ext = ri/rd @@ -39,18 +46,24 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { 2*o + iota{2} } } - def se{e==ext, c, o} = { - io:= is+o - got:= selx{c} - def s{M} = storeBatch{r, io, got, M} - if (ext==1 or not M{0}) s{M} - else if (io*rd+rd <= wl) s{maskNone} - else { if (io*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } - } + def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl} se{1, cw, 0} }} } +def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { + def VI = [ri]TI + def ext = ri/rd + xlf:= broadcast{VI, cast_i{TI, xl}} + maskedLoop{ri, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + is:= (if (ext>1) i<sel{VD, xd, c}} + } + if (wi==8 and wd==32 and xl*wd<=256 ) { perm_select{ } } + else if (wi==8 and wd<=16 and xl*wd<=128 ) { shuf_select{0} } else if (wi==8 and wd<=16 and xl*wd<=128<<1) { shuf_select{1} } else if (wi==8 and wd<=16 and xl*wd<=128<<2) { shuf_select{2} } else if (wi==8 and wd<= 8 and xl*wd<=128<<3) { shuf_select{3} }