singeli select (disabled)

This commit is contained in:
dzaima 2022-04-12 16:21:07 +03:00
parent c54e63099f
commit a4a72a78c4
9 changed files with 297 additions and 52 deletions

View File

@ -3,6 +3,13 @@
#include "../utils/mut.h"
#include "../builtins.h"
// #include <immintrin.h>
// #if SINGELI
// #pragma GCC diagnostic push
// #pragma GCC diagnostic ignored "-Wunused-variable"
// #include "../singeli/gen/select.c"
// #pragma GCC diagnostic pop
// #endif
extern B rt_select;
B select_c1(B t, B x) {
@ -37,9 +44,34 @@ B select_c2(B t, B w, B x) {
usz wia = a(w)->ia;
if (xr==1) {
if (wia==0) {
decG(x);
B r;
if (isVal(xf)) {
Arr* ra = m_fillarrp(0);
arr_shCopy(ra, w);
fillarr_setFill(ra, xf);
r = taga(ra);
} else if (rnk(w)==1) {
r = isNum(xf)? emptyIVec() : emptyCVec();
} else {
Arr* ra = m_arr(sizeof(TyArr), isNum(xf)? t_i8arr : t_c8arr, 0);
arr_shCopy(ra, w);
r = taga(ra);
}
decG(w);
return r;
}
usz xia = a(x)->ia;
u8 xe = TI(x,elType);
u8 we = TI(w,elType);
#if SINGELI
// if (we==el_i8 && xe==el_i32) { i32* rp; B r = m_i32arrc(&rp, w); if (!avx2_select_i8_32 ((u8*)i8any_ptr (w), (u8*)i32any_ptr(x), (u8*)rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); decG(w); decG(x); return r; }
// if (we==el_i16 && xe==el_i32) { i32* rp; B r = m_i32arrc(&rp, w); if (!avx2_select_i16_32((u8*)i16any_ptr(w), (u8*)i32any_ptr(x), (u8*)rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); decG(w); decG(x); return r; }
// if (we==el_i32 && xe==el_i8 ) { i8* rp; B r = m_i8arrc (&rp, w); if (!avx2_select_i32_8 ((u8*)i32any_ptr(w), (u8*)i8any_ptr (x), (u8*)rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); decG(w); decG(x); return r; }
// if (we==el_i32 && xe==el_i32) { i32* rp; B r = m_i32arrc(&rp, w); if (!avx2_select_i32_32((u8*)i32any_ptr(w), (u8*)i32any_ptr(x), (u8*)rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); decG(w); decG(x); return r; }
// if (we==el_i32 && xe==el_f64) { f64* rp; B r = m_f64arrc(&rp, w); if (!avx2_select_i32_64((u8*)i32any_ptr(w), (u8*)f64any_ptr(x), (u8*)rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); decG(w); decG(x); return r; }
#endif
#define CASE(T,E) if (xe==el_##T) { \
E* rp; B r = m_##T##arrc(&rp, w); \
E* xp = T##any_ptr(x); \

View File

@ -27,9 +27,8 @@ def v2i{x:T & w256{T}} = cast_v{[32]u8, x} # for compact casting for the annoyin
def v2f{x:T & w256{T}} = cast_v{[8]f32, x}
def v2d{x:T & w256{T}} = cast_v{[4]f64, x}
def ty_vu{T & w256s{T}} = [vcount{T}](ty_u{eltype{T}})
def ty_vs{T & w256u{T}} = [vcount{T}](ty_s{eltype{T}})
def to_el{E, x:T} = cast_v{[width{T}/width{E}]E, x}
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
def to_el{E, x:T} = cast_v{to_el{E,T}, x}
def forv{T & w256{T}} = forc{{v}=>cast_vp{T,v}}
# load & store
@ -108,8 +107,7 @@ def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
# conversion
def cvt_f2i{x:[8]f32} = emit{[8]i32, '_mm256_cvtps_epi32', x}
def cvt_i2f{x:[8]i32} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a}
# structural operations

View File

@ -2,13 +2,17 @@
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', cast_p{i32, a+n}, m, v}
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', cast_p{f64, a+n}, m, v}
# maskstore with all cases defined, at the cost of not being a single instruction
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
# min & max
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}
def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}
def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b}
def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b}
def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b}
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}
def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b}
def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b}
def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b}
def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b}
# signed comparison
def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
@ -58,6 +62,8 @@ def unpackQ{a:[ 8]i32, b:[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32',
# inverse of questionable pack; these saturate the argument
def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
# super questionable pack - takes assumes high halves are zero
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
@ -113,8 +119,8 @@ def blend{I==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 4]f64,
def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = cast_v{T, emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}}
# assumes all bits are the same in each mask item
def blendf = blend
def blendf{f:T, t:T, m:M & w256{T} & w256i{M,16}} = blend{f, t, cast_v{[32]i8,m}}
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & width{eltype{M}}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w256{T} & w256{M,16}} = blend{f, t, cast_v{[32]i8,m}}
def shuf{I==[4]u32, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}}
def shuf{I==[4]u64, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}}
@ -129,8 +135,32 @@ def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_ep
def getmask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
def getmask{x:T & w256{T, 16}} = {
msk:u32 = getmask{emit{[32]u8, '_mm256_packs_epi16', x, broadcast{[16]u16, 0}}}
(msk&255) | (msk>>8) # TODO try out permute4x64 for fixing up
(msk&255) | (msk>>8)
}
def any{x:T & w256i{T, 16}} = getmask{cast_v{[32]u8,x}}!=0
def all{x:T & w256i{T, 16}} = getmask{cast_v{[32]u8,x}} == (1<<32)-1
def anyneg{x:T & w256s{T, 16}} = getmask{cast_v{[32]u8, cast_v{[16]i16,x} < broadcast{[16]i16, 0}}}!=0
# conversion
# convert packed elements of type F to a result T
def cvt{F==u8, T==[16]u16, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi16', a}
def cvt{F==u8, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi32', a}
def cvt{F==u16, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu16_epi32', a}
def cvt{F==i8, T==[16]i16, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi16', a}
def cvt{F==i8, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi32', a}
def cvt{F==i16, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi16_epi32', a}
def cvt{F , T==[4]f64, a:A & w128i{A} & F!=i32} = cvt{i32, T, cvt{F, [4]i32, a}}
def cvt{F, T, a:A & eltype{T}==F} = T~~a
# undo cvt; assumes elements of X fit in T, otherwise may be saturated; but keeps the width of x, with trailing elements undefined
def ucvt{T, x:X & w256i{X,32} & width{T}==8} = {
a:= packQ{x, x}
b:= packQ{a, a}
to_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}}
}
def ucvt{T, x:X & w256i{X,32} & width{T}==16} = {
to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
}
def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x}

View File

@ -1,5 +1,6 @@
include 'skin/c'
include 'arch/c'
oper infix right ~~ reinterpret 55
# TODO move these to a more base file
def knum{x} = match{'number',kind{x}}
@ -22,9 +23,15 @@ def anyNum{x:T} = match{'primitive',typekind{T}}
def anyInt{x} = knum{x}
def anyInt{x & match{'register',kind{x}} | match{'constant',kind{x}}} = isint{x}
def __add{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a, b}
def __sub{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a, b}
def tern{c, T, F & anyInt{c}} = {
if(c) T
else F
}
def ty_u{T==i8 } = u8; def ty_u{w== 8} = u8
def ty_u{T==i16} = u16; def ty_u{w==16} = u16
@ -35,6 +42,11 @@ def ty_s{T==u16} = i16; def ty_s{w==16} = i16
def ty_s{T==u32} = i32; def ty_s{w==32} = i32
def ty_s{T==u64} = i64; def ty_s{w==64} = i64
def ty_vu{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
def ty_vs{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
def ty_vs{x:T} = ty_vs{T}~~x
def ty_vu{x:T} = ty_vu{T}~~x
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
def f{i,l & i==l} = 0
def f{i,l & i!=l} = {
@ -85,4 +97,11 @@ def broadcast{T, v & match{typekind{T},'primitive'}} = v
def any{v:T & match{typekind{T},'primitive'}} = v
def anyneg{v:T & match{typekind{T},'primitive'}} = v<0
def iota{n & knum{n}} = @collect(i to n) i
def broadcast{n, v & knum{n}} = @collect(n) v
def broadcast{n, v & knum{n}} = @collect(n) v
# debug stuff
def print{x & match{kind{x},'symbol'}} = emit{void, 'printf', '"%s"', merge{'"', x, '"'}}
def print{x & anyNum{x}} = emit{void, 'printf', '"%d"', x}
def print{x:T & anyNum{x} & width{T}==64} = emit{void, 'printf', '"%ld"', x}
def println{x} = { print{x}; print{'\n'} }

View File

@ -18,9 +18,6 @@ def ty_dbl{T & i32==T} = i64
def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}})
def dcast_i{x} = ext{ty_dbl{type{x}}, x}
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
def maskstoreF{p, m, n, x:T} = store{p, n, blendf{load{p,n}, x, m}}
# + & -
def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}}
def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub}} = anyneg{M{(w^x) & (w^r)}}

View File

@ -6,28 +6,6 @@ include './avx'
include './avx2'
include './mask'
def cvt{F==i8, T==[8]i16, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi16', a}
def cvt{F==i8, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi32', a}
def cvt{F==i16, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi16_epi32', a}
def cvt{F==u8, T==[16]u16, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi16', a}
def cvt{F==u8, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi32', a}
def cvt{F==u16, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu16_epi32', a}
def cvt{F==i8, T==[16]i16, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi16', a}
def cvt{F==i8, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi32', a}
def cvt{F==i16, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi16_epi32', a}
def cvt{F , T==[4]f64, a:A & w128i{A}} = cvt{i32, T, cvt{F, [4]i32, a}}
def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a}
def cvt{F, T, a & eltype{T}==F} = reinterpret{T, a}
def cif{C, T, F} = {
if(C) T
else F
}
def swap{w,x} = {
t:= w
w = x
@ -84,7 +62,7 @@ equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = {
def fac = width{X}/width{W}
maskedLoop{bulk, l, {i, M} => {
cw:= load{cast_p{ww{cif{fac==1, 256, 128}, u8}, w + i*32/fac}, 0}
cw:= load{cast_p{ww{tern{fac==1, 256, 128}, u8}, w + i*32/fac}, 0}
cx:= load{cast_p{ww{256, X}, x}, i}
cwc:= cvt{W, ww{256, X}, cw}
if (anyne{cwc,cx,M}) return{0}

View File

@ -1,16 +1,19 @@
# get mask of first n items; n>0 & n<vcount{T}
def maskInit{T} = {
def n = 256/width{T} - 1
merge{broadcast{n, -1}, broadcast{n, 0}}
def maskInit{w, T} = {
def n = w/width{T} - 1
merge{broadcast{n, -1}, broadcast{n, 0}}
}
mask256_8 :*i8 = maskInit{i8}
mask256_16:*i16 = maskInit{i16}
mask256_32:*i32 = maskInit{i32}
mask256_64:*i64 = maskInit{i64}
mask256_1:*u8 = apply{merge, each{{x} => {
merge{broadcast{31, 255}, (1<<x)-1, broadcast{32, 0}}
}, iota{8}}}
def maskInit1{w} = {
apply{merge, each{{x} => {
merge{broadcast{w/8-1, 255}, (1<<x)-1, broadcast{w/8, 0}}
}, iota{8}}}
}
mask256_8 :*i8 = maskInit{256,i8 }; mask128_8 :*i8 = maskInit{128,i8 }
mask256_16:*i16 = maskInit{256,i16}; mask128_16:*i16 = maskInit{128,i16}
mask256_32:*i32 = maskInit{256,i32}; mask128_32:*i32 = maskInit{128,i32}
mask256_64:*i64 = maskInit{256,i64}; mask128_64:*i64 = maskInit{128,i64}
mask256_1 :*u8 = maskInit1{256}; mask128_1 :*u8 = maskInit1{128}
def maskOf{T,n & w256{T, 8}} = load{cast_p{[32]u8, mask256_8 + (n^31)}, 0}
@ -19,6 +22,12 @@ def maskOf{T,n & w256{T,32}} = load{cast_p{[ 8]u32, mask256_32 + (n^7)}, 0}
def maskOf{T,n & w256{T,64}} = load{cast_p{[ 4]u64, mask256_64 + (n^3)}, 0}
def maskOfBit{T,n & w256{T}} = load{cast_p{[32]u8, mask256_1 + (n>>3)^31 + 64*(n&7)}, 0}
def maskOf{T,n & w128{T, 8}} = load{cast_p{[16]u8, mask128_8 + (n^15)}, 0}
def maskOf{T,n & w128{T,16}} = load{cast_p{[ 8]u16, mask128_16 + (n^7)}, 0}
def maskOf{T,n & w128{T,32}} = load{cast_p{[ 4]u32, mask128_32 + (n^3)}, 0}
def maskOf{T,n & w128{T,64}} = load{cast_p{[ 2]u64, mask128_64 + (n^1)}, 0}
def maskOfBit{T,n & w128{T}} = load{cast_p{[16]u8, mask128_1 + (n>>3)^15 + 32*(n&7)}, 0}
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y}
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}}
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
@ -31,5 +40,7 @@ def maskAfter{n} = {
def mask{x:X & isvec{X}} = x&cast_v{X,maskOf{X,n}}
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X, mode=='all bits zeroes'} = andIsZero{x, cast_v{X,maskOfBit{X,n}}}
def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{mode=='count'} = n
def mask{x==0} = 1
}

View File

@ -0,0 +1,88 @@
include './base'
include './cbqnDefs'
include './sse3'
include './avx'
include './avx2'
include './mask'
def maskedLoop{bulk, l, step} = {
m:u64 = l/bulk
@for (i from 0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def minBulk{w, A, B & width{A}< width{B}} = w/width{B}
def minBulk{w, A, B & width{A}>=width{B}} = w/width{A}
# def:T - masked original content
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
# idx - actual (unscaled) index list
def gather{def:T, b:B, idx:[8]i32, M & w256{T,32}} = {
if (M{0}) cast_v{T, emit{[8]i32, '_mm256_mask_i32gather_epi32', def, cast_p{i32,b}, idx, M{T,'to sign bits'}, width{eltype{B}}/8}}
else cast_v{T, emit{[8]i32, '_mm256_i32gather_epi32', cast_p{i32,b}, idx, width{eltype{B}}/8}}
}
def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
if (M{0}) cast_v{T, emit{[4]i64, '_mm256_mask_i32gather_epi64', def, cast_p{i64,b}, idx, M{T,'to sign bits'}, width{eltype{B}}/8}}
else cast_v{T, emit{[4]i64, '_mm256_i32gather_epi64', cast_p{i64,b}, idx, width{eltype{B}}/8}}
}
# (sign/zero)-extend n'th batch to T
def loadx{ptr:P, n, T} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0}
else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}}
}
# store low packed elements of x to P
def storeL{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x}
def storeL{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}}
# store extended data x to n'th batch in ptr
def storex{ptr:P, n, x:T, M} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
xu:= ucvt{eltype{P}, x}
def TF = to_el{E0, T}
if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu}
else storeL{rpos, vcount{T}*width{E0}, xu}
}
select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
def TIE = i32
def TDE = tern{width{TD}<32, u32, TD}
def bulk = minBulk{rw,TIE,TDE}
def TIF = [bulk]TIE
def TDF = [bulk]TDE
def xlf = broadcast{TIF, cast_i{eltype{TIF}, xl}}
w:=cast_p{TI,w0}
x:=cast_p{TD,x0}
r:=cast_p{TD,r0}
maskedLoop{bulk, wl, {i, M} => {
cw0:= loadx{w, i, TIF}
cw1:= cw0+xlf
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
if (any{M{ty_vu{cw} >= ty_vu{xlf}}}) return{0}
got:= gather{broadcast{TDF,0}, x, cw, M}
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
storex{r, i, got, M}
}}
1
}
'avx2_select_i32_8' = select{256, i32, u8}
'avx2_select_i32_16' = select{256, i32, u16}
'avx2_select_i32_32' = select{256, i32, u32}
'avx2_select_i32_64' = select{256, i32, u64}
'avx2_select_i8_32' = select{256, i8, u32}
'avx2_select_i16_32' = select{256, i16, u32}

View File

@ -36,4 +36,96 @@ def storea{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_store_si128',
def store {a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_storeu_pd', cast_p{f64, a+n}, v}
def storea{a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_store_pd', cast_p{f64, a+n}, v}
def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', cast_p{f32, a+n}, v}
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', cast_p{f32, a+n}, v}
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', cast_p{f32, a+n}, v}
# broadcast
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', ext{eltype{T},v}}
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', ext{eltype{T},v}}
def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', ext{eltype{T},v}}
def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',ext{eltype{T},v}}
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
# signed comparison
def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __gt{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __eq{a:T,b:T & T==[16]u8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]u16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]u32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T}} = ~(a<=b)
# rest of comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
# mask stuff
def getmask{x:T & w128{T, 8}} = emit{u32, '_mm_movemask_epi8', x}
def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, broadcast{[8]u16, 0}}}
def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def any{x:T & w128i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits)
def all{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1 # same assumption
def anyneg{x:T & w128s{T}} = getmask{x}!=0
def any{x:T & w128i{T, 16}} = getmask{cast_v{[32]u8,x}}!=0
def all{x:T & w128i{T, 16}} = getmask{cast_v{[32]u8,x}} == (1<<32)-1
def anyneg{x:T & w128s{T, 16}} = getmask{cast_v{[32]u8, cast_v{[16]i16,x} < broadcast{[16]i16, 0}}}!=0
# the lone SSE2 extract
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
# debug stuff
def printGen{x, s, n} = apply{emit, merge{tup{void, 'printf', merge{'"', apply{merge,each{{c}=>{if(c>0) merge{',',s}; else s}, iota{n}}}, '"'}}, each{{c}=>extract{x,c}, iota{n}}}}
def print{x:T & isvec{T}} = printGen{x, merge{'%', tern{width{eltype{T}}==64, 'l', ''}, tern{issigned{x}, 'd', 'u'}}, vcount{T}}
### SSE4.1; TODO either move to own file or rename this file
# rest of extract
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
# conversion
def cvt{F==i8, T==[8]i16, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi16', a}
def cvt{F==i8, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi32', a}
def cvt{F==i16, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi16_epi32', a}
# mask stuff
def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# blend by sign bit; no 16-bit case
def blend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = cast_v{T, emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}}
def blend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = cast_v{T, emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}}
def blend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = cast_v{T, emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}}
# assumes all bits are the same in each mask item
def blendF{f:T, t:T, m:M & w128{T} & w128{M} & width{eltype{M}}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w128{T} & w128{M,16}} = blend{f, t, cast_v{[16]i8,m}}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def __eq{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
# SSE4.2
def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}