singeli ≡
This commit is contained in:
parent
9c9a91afa5
commit
49ff5a04ff
2
makefile
2
makefile
@ -136,7 +136,7 @@ preSingeliBin:
|
||||
@mv BQN obj/presingeli/BQN
|
||||
|
||||
|
||||
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c}
|
||||
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c}
|
||||
@echo $(postmsg)
|
||||
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
||||
@echo $< | cut -c 17- | sed 's/^/ /'
|
||||
|
||||
@ -6,11 +6,6 @@
|
||||
#define BCALL(N, X) N(b(X))
|
||||
#define interp_f64(X) b(X).f
|
||||
|
||||
static i8 mask8[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
|
||||
static i16 mask16[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
|
||||
static i32 mask32[] = {-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0};
|
||||
static i64 mask64[] = {-1,-1,-1,0,0,0};
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#include "../singeli/gen/dyarith.c"
|
||||
@ -235,10 +230,6 @@ static f64 pfmod(f64 a, f64 b) {
|
||||
if (we==el_f64) { Rf64(w) PF(w) DOF(EXPR,x,wp[i],x.f) dec(w); return num_squeeze(r); }
|
||||
|
||||
#if SINGELI
|
||||
static void* tyany_ptr(B x) { // TODO extract to some header file
|
||||
u8 t = v(x)->type;
|
||||
return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a;
|
||||
}
|
||||
#define SI_AA(N,S,BASE) R##S(x); usz rlen=avx2_##N##AA##_##S((void*)wp, (void*)xp, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r;
|
||||
#define SI_SA_I(N,S,W,BASE) R##S(x); usz rlen=avx2_##N##SA##_##S((W).u, (void*)xp, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r;
|
||||
#define SI_AS_I(N,S,X,BASE) R##S(w); usz rlen=avx2_##N##AS##_##S((void*)wp, (X).u, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r;
|
||||
|
||||
@ -33,6 +33,10 @@ static B* arr_bptr(B x) { assert(isArr(x));
|
||||
if (v(x)->type==t_fillslice) return c(FillSlice,x)->a;
|
||||
return NULL;
|
||||
}
|
||||
static void* tyany_ptr(B x) {
|
||||
u8 t = v(x)->type;
|
||||
return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a;
|
||||
}
|
||||
|
||||
|
||||
typedef struct BFn {
|
||||
|
||||
@ -402,6 +402,41 @@ NOINLINE bool atomEqualR(B w, B x) {
|
||||
{ dec(wd);dec(xd); return false; }
|
||||
dec(wd);dec(xd); return true;
|
||||
}
|
||||
|
||||
#if SINGELI
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#include "../singeli/gen/equal.c"
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
typedef bool (*EqFn)(u8* a, u8* b, u64 l, u64 data);
|
||||
bool notEq(u8* a, u8* b, u64 l, u64 data) { return false; }
|
||||
|
||||
#define F(X) avx2_equal_##X
|
||||
EqFn eqFns[] = {
|
||||
F(1_1), F(1_8), F(1_16), F(1_32), F(1_f64), notEq, notEq, notEq,
|
||||
F(1_8), F(8_8), F(s8_16), F(s8_32), F(s8_f64), notEq, notEq, notEq,
|
||||
F(1_16), F(s8_16), F(8_8), F(s16_32), F(s16_f64), notEq, notEq, notEq,
|
||||
F(1_32), F(s8_32), F(s16_32), F(8_8), F(s32_f64), notEq, notEq, notEq,
|
||||
F(1_f64), F(s8_f64), F(s16_f64), F(s32_f64), F(f64_f64), notEq, notEq, notEq,
|
||||
notEq, notEq, notEq, notEq, notEq, F(8_8), F(u8_16), F(u8_32),
|
||||
notEq, notEq, notEq, notEq, notEq, F(u8_16), F(8_8), F(u16_32),
|
||||
notEq, notEq, notEq, notEq, notEq, F(u8_32), F(u16_32), F(8_8),
|
||||
};
|
||||
#undef F
|
||||
static const u8 n = 99;
|
||||
u8 eqFnData[] = { // for the main diagonal, amount to shift length by; otherwise, whether to swap arguments
|
||||
0,0,0,0,0,n,n,n,
|
||||
1,0,0,0,0,n,n,n,
|
||||
1,1,1,0,0,n,n,n,
|
||||
1,1,1,2,0,n,n,n,
|
||||
1,1,1,1,0,n,n,n,
|
||||
n,n,n,n,n,0,0,0,
|
||||
n,n,n,n,n,1,1,0,
|
||||
n,n,n,n,n,1,1,2,
|
||||
};
|
||||
#endif
|
||||
|
||||
NOINLINE bool equal(B w, B x) { // doesn't consume
|
||||
bool wa = isAtm(w);
|
||||
bool xa = isAtm(x);
|
||||
@ -409,24 +444,36 @@ NOINLINE bool equal(B w, B x) { // doesn't consume
|
||||
if (wa) return atomEqual(w, x);
|
||||
if (!eqShape(w,x)) return false;
|
||||
usz ia = a(x)->ia;
|
||||
if (ia==0) return true;
|
||||
u8 we = TI(w,elType);
|
||||
u8 xe = TI(x,elType);
|
||||
if (((we==el_f64 | we==el_i32) && (xe==el_f64 | xe==el_i32))) {
|
||||
if (we==el_i32) { i32* wp = i32any_ptr(w);
|
||||
if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
} else { f64* wp = f64any_ptr(w);
|
||||
if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
|
||||
#if SINGELI
|
||||
if (we<=el_c32 && xe<=el_c32) {
|
||||
u8* wp = tyany_ptr(w);
|
||||
u8* xp = tyany_ptr(x);
|
||||
u64 idx = we*8 + xe;
|
||||
return eqFns[idx](wp, xp, ia, eqFnData[idx]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (we==el_c32 && xe==el_c32) {
|
||||
u32* wp = c32any_ptr(w);
|
||||
u32* xp = c32any_ptr(x);
|
||||
for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false;
|
||||
return true;
|
||||
}
|
||||
#else
|
||||
if (((we==el_f64 | we==el_i32) && (xe==el_f64 | xe==el_i32))) {
|
||||
if (we==el_i32) { i32* wp = i32any_ptr(w);
|
||||
if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
} else { f64* wp = f64any_ptr(w);
|
||||
if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; }
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (we==el_c32 && xe==el_c32) {
|
||||
u32* wp = c32any_ptr(w);
|
||||
u32* xp = c32any_ptr(x);
|
||||
for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false;
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
SLOW2("equal", w, x);
|
||||
SGetU(x)
|
||||
SGetU(w)
|
||||
for (usz i = 0; i < ia; i++) if(!equal(GetU(w,i),GetU(x,i))) return false;
|
||||
|
||||
@ -36,11 +36,6 @@ FN_LUT(avx2, ge, AS); FN_LUT(avx2, ge, AA);
|
||||
FN_LUT(avx2, lt, AS);
|
||||
FN_LUT(avx2, le, AS);
|
||||
|
||||
static void* tyany_ptr(B x) {
|
||||
u8 t = v(x)->type;
|
||||
return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a;
|
||||
}
|
||||
|
||||
#define AL(X) u64* rp; B r = m_bitarrc(&rp, X); usz ria=a(r)->ia
|
||||
#define CMP_IMPL(CHR, NAME, RNAME, PNAME, L, R, OP, FC, CF, BX) \
|
||||
if (isF64(w)&isF64(x)) return m_i32(w.f OP x.f); \
|
||||
|
||||
@ -1,15 +1,18 @@
|
||||
# various utilities
|
||||
|
||||
def w256{T} = 0
|
||||
def w256{T & isvec{T}} = width{T}==256
|
||||
|
||||
def wgen256{F} = {
|
||||
def r{T} = 0
|
||||
def r{T & width{T}==256} = F{eltype{T}}
|
||||
def r{T & w256{T}} = F{eltype{T}}
|
||||
def r{T,w} = 0
|
||||
def r{T,w & width{T}==256} = F{eltype{T}} & (width{eltype{T}}==w)
|
||||
def r{T,w & w256{T}} = F{eltype{T}} & (width{eltype{T}}==w)
|
||||
def r{T & ~isvec{T}} = 0
|
||||
r
|
||||
}
|
||||
def w256{T} = width{T}==256
|
||||
def w256{T, w} = 0
|
||||
def w256{T, w & width{T}==256 & width{eltype{T}}==w} = 1
|
||||
def w256{T,w} = 0
|
||||
def w256{T,w & w256{T}} = width{eltype{T}}==w
|
||||
def w256i = wgen256{{T} => isint{T}}
|
||||
def w256s = wgen256{{T} => isint{T} & issigned{T}}
|
||||
def w256u = wgen256{{T} => isint{T} & isunsigned{T}}
|
||||
@ -24,8 +27,9 @@ def v2i{x:T & w256{T}} = cast_v{[32]u8, x} # for compact casting for the annoyin
|
||||
def v2f{x:T & w256{T}} = cast_v{[8]f32, x}
|
||||
def v2d{x:T & w256{T}} = cast_v{[4]f64, x}
|
||||
|
||||
def ty_vu{T & w256s{T}} = [vcount{T}](ty_iu{eltype{T}})
|
||||
def ty_vs{T & w256u{T}} = [vcount{T}](ty_is{eltype{T}})
|
||||
def ty_vu{T & w256s{T}} = [vcount{T}](ty_u{eltype{T}})
|
||||
def ty_vs{T & w256u{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def to_el{E, x:T} = cast_v{[width{T}/width{E}]E, x}
|
||||
def forv{T & w256{T}} = forc{{v}=>cast_vp{T,v}}
|
||||
|
||||
# load & store
|
||||
@ -122,6 +126,10 @@ def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x
|
||||
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
|
||||
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
|
||||
|
||||
# blend by sign bit
|
||||
def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = cast_v{T, emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}}
|
||||
def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = cast_v{T, emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}}
|
||||
|
||||
# mixed-width operations
|
||||
def half{x:T, i & w256{T} & knum{i}} = cast_v{[vcount{T}/2](eltype{T}), emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}}
|
||||
def pair{a:T,b:T & width{T}==128} = cast_v{[vcount{T}*2](eltype{T}), emit{[8]i32, '_mm256_setr_m128i', a, b}}
|
||||
@ -130,6 +138,7 @@ def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
|
||||
# mask stuff
|
||||
def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
||||
def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
||||
def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
||||
def any{x:T & w256i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits)
|
||||
def all{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1 # same assumption
|
||||
def anyneg{x:T & w256s{T}} = getmask{x}!=0
|
||||
@ -109,14 +109,12 @@ def blend{I==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[16]i16,
|
||||
def blend{I==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}}
|
||||
def blend{I==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}}
|
||||
|
||||
# blends by sign bit; no 16-bit case
|
||||
def blend{a:T, b:T, m:M & w256{T} & w256i{M, 8}} = cast_v{T, emit{[32]i8, '_mm256_blendv_epi8', v2i{a}, v2i{b}, v2i{m}}}
|
||||
def blend{a:T, b:T, m:M & w256{T} & w256i{M,32}} = cast_v{T, emit{[8]f32, '_mm256_blendv_ps', v2f{a}, v2f{b}, v2f{m}}}
|
||||
def blend{a:T, b:T, m:M & w256{T} & w256i{M,64}} = cast_v{T, emit{[4]f64, '_mm256_blendv_pd', v2d{a}, v2d{b}, v2d{m}}}
|
||||
# blend by sign bit; still no 16-bit case
|
||||
def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = cast_v{T, emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}}
|
||||
|
||||
# assumes all bits are the same in each mask item
|
||||
def blendf = blend
|
||||
def blendf{a:T, b:T, m:M & w256{T} & w256i{M,16}} = blend{a, b, cast_v{[32]i8,m}}
|
||||
def blendf{f:T, t:T, m:M & w256{T} & w256i{M,16}} = blend{f, t, cast_v{[32]i8,m}}
|
||||
|
||||
def shuf{I==[4]u32, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}}
|
||||
def shuf{I==[4]u64, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}}
|
||||
|
||||
@ -26,10 +26,14 @@ def __add{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a,
|
||||
def __sub{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a, b}
|
||||
|
||||
|
||||
def ty_iu{T & T==i8 } = u8; def ty_is{T & T==i8 } = u8
|
||||
def ty_iu{T & T==i16} = u16; def ty_is{T & T==i16} = u16
|
||||
def ty_iu{T & T==i32} = u32; def ty_is{T & T==i32} = u32
|
||||
def ty_iu{T & T==i64} = u64; def ty_is{T & T==i64} = u64
|
||||
def ty_u{T==i8 } = u8; def ty_u{w== 8} = u8
|
||||
def ty_u{T==i16} = u16; def ty_u{w==16} = u16
|
||||
def ty_u{T==i32} = u32; def ty_u{w==32} = u32
|
||||
def ty_u{T==i64} = u64; def ty_u{w==64} = u64
|
||||
def ty_s{T==u8 } = i8; def ty_s{w== 8} = i8
|
||||
def ty_s{T==u16} = i16; def ty_s{w==16} = i16
|
||||
def ty_s{T==u32} = i32; def ty_s{w==32} = i32
|
||||
def ty_s{T==u64} = i64; def ty_s{w==64} = i64
|
||||
|
||||
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
|
||||
def f{i,l & i==l} = 0
|
||||
@ -71,10 +75,14 @@ def isintv{T} = isint{eltype{T}}
|
||||
def isfloatv{T} = isfloat{eltype{T}}
|
||||
def issignedv{T} = issigned{eltype{T}}
|
||||
def isunsignedv{T} = isunsigned{eltype{T}}
|
||||
def isvec{T} = match{typekind{T},'vector'}
|
||||
def istype{T} = match{kind{T},'type'}
|
||||
def isvec{T} = 0
|
||||
def isvec{T & istype{T}} = match{typekind{T},'vector'}
|
||||
|
||||
|
||||
# non-vector variants of vector defs
|
||||
def broadcast{T, v & match{typekind{T},'primitive'}} = v
|
||||
def any{v:T & match{typekind{T},'primitive'}} = v
|
||||
def anyneg{v:T & match{typekind{T},'primitive'}} = v<0
|
||||
def anyneg{v:T & match{typekind{T},'primitive'}} = v<0
|
||||
def iota{n & knum{n}} = @collect(i to n) i
|
||||
def broadcast{n, v & knum{n}} = @collect(n) v
|
||||
@ -4,6 +4,7 @@ include './cbqnDefs'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './bitops'
|
||||
include './mask'
|
||||
|
||||
|
||||
def rootty{T & match{typekind{T},'primitive'}} = T
|
||||
@ -15,20 +16,10 @@ def ty_dbl{T & i16==T} = i32
|
||||
def ty_dbl{T & i32==T} = i64
|
||||
def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}})
|
||||
def dcast_i{x} = ext{ty_dbl{type{x}}, x}
|
||||
def to_el{E, x:T} = cast_v{[width{T}/width{E}]E, x}
|
||||
|
||||
# get mask of first n items; n>0 & n<vcount{T}
|
||||
def genmask{T,n & w256{T, 8}} = load{cast_p{[32]i8, emit{*i8, '', 'mask8' } + (n^31)}, 0}
|
||||
def genmask{T,n & w256{T,16}} = load{cast_p{[16]i16, emit{*i16, '', 'mask16'} + (n^15)}, 0}
|
||||
def genmask{T,n & w256{T,32}} = load{cast_p{[ 8]i32, emit{*i32, '', 'mask32'} + (n^7)}, 0}
|
||||
def genmask{T,n & w256{T,64}} = load{cast_p{[ 4]i64, emit{*i64, '', 'mask64'} + (n^3)}, 0}
|
||||
|
||||
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendf{load{p,n}, x, m}}
|
||||
|
||||
def anyne{x:T, y:T, M & M{0}==0} = ~all{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1} = any{M{x!=y}}
|
||||
|
||||
# + & -
|
||||
def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}}
|
||||
def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub}} = anyneg{M{(w^x) & (w^r)}}
|
||||
@ -94,10 +85,10 @@ def arithAny{VT, F, W, X, r, len} = {
|
||||
@forv{VT} (r over i from 0 to vv) r = arithChk3{F, {x}=>x, W{i}, X{i}, i*bam}
|
||||
left:= len&(bam-1)
|
||||
if (left!=0) {
|
||||
m:= genmask{VT, left}
|
||||
m:= maskOf{VT, left}
|
||||
def mask{x:X} = x&cast_v{X,m}
|
||||
def mask{x==0} = 1
|
||||
rv:= arithChk3{F, mask, W{vv}, X{vv}, vv*bam}
|
||||
rv:= arithChk3{F, maskAfter{left}, W{vv}, X{vv}, vv*bam}
|
||||
maskstoreF{cast_p{VT,r}, m, vv, rv}
|
||||
}
|
||||
len
|
||||
|
||||
115
src/singeli/src/equal.singeli
Normal file
115
src/singeli/src/equal.singeli
Normal file
@ -0,0 +1,115 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
|
||||
def cvt{F==i8, T==[8]i16, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi16', a}
|
||||
def cvt{F==i8, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi32', a}
|
||||
def cvt{F==i16, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi16_epi32', a}
|
||||
|
||||
|
||||
def cvt{F==u8, T==[16]u16, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi16', a}
|
||||
def cvt{F==u8, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi32', a}
|
||||
def cvt{F==u16, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu16_epi32', a}
|
||||
|
||||
def cvt{F==i8, T==[16]i16, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi16', a}
|
||||
def cvt{F==i8, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi32', a}
|
||||
def cvt{F==i16, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi16_epi32', a}
|
||||
|
||||
def cvt{F , T==[4]f64, a:A & w128i{A}} = cvt{i32, T, cvt{F, [4]i32, a}}
|
||||
def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a}
|
||||
|
||||
def cvt{F, T, a & eltype{T}==F} = reinterpret{T, a}
|
||||
|
||||
def cif{C, T, F} = {
|
||||
if(C) T
|
||||
else F
|
||||
}
|
||||
def swap{w,x} = {
|
||||
t:= w
|
||||
w = x
|
||||
x = t
|
||||
}
|
||||
|
||||
def maskedLoop{bulk, l, step} = {
|
||||
m:u64 = l/bulk
|
||||
@for (i from 0 to m) step{i, maskNone}
|
||||
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) step{m, maskAfter{left}}
|
||||
}
|
||||
|
||||
equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = {
|
||||
def bulk = 256 / width{X}
|
||||
if (W!=X) if (d!=0) swap{w,x}
|
||||
if (W==u1) {
|
||||
if (X==u1) { # bitarr ≡ bitarr
|
||||
maskedLoop{256, l, {i, M} => {
|
||||
cw:= load{cast_p{[32]u8, w}, i}
|
||||
cx:= load{cast_p{[32]u8, x}, i}
|
||||
if (anyneBit{cw,cx,M}) return{0}
|
||||
}}
|
||||
} else if (X==f64) { # bitarr ≡ f64arr
|
||||
def T = [4]f64
|
||||
def bulk = 4
|
||||
f0:= broadcast{T, 0.0}
|
||||
f1:= broadcast{T, 1.0}
|
||||
maskedLoop{4, l, {i, M} => {
|
||||
cw:= load{cast_p{u8,w}, i>>1} >> cast_i{u8, 4*(i&1)}
|
||||
cx:= load{cast_p{T, x}, i}
|
||||
wu:= blend{f0, f1, broadcast{[4]u64, cw} << make{[4]u64,63,62,61,60}}
|
||||
if (anyne{wu, cx, M}) return{0}
|
||||
}}
|
||||
} else { # bitarr ≡ i8/i16/i32arr
|
||||
def T = [256/width{X}]X
|
||||
def sh{c} = c << (width{X}-1)
|
||||
def sh{c & X==u8} = cast_v{T, to_el{u16,c}<<7}
|
||||
|
||||
# TODO compare with doing the comparison in vector registers
|
||||
badBits:= broadcast{T, ~cast{X,1}}
|
||||
maskedLoop{bulk, l, {i, M} => {
|
||||
cw:= load{cast_p{ty_u{bulk}, w}, i}
|
||||
cx:= load{cast_p{T,x}, i}
|
||||
if (~andIsZero{M{cx}, badBits}) return{0}
|
||||
if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0}
|
||||
}}
|
||||
1
|
||||
}
|
||||
} else { # everything not involving bitarrs
|
||||
if (W==i8 and X==i8) l<<= d
|
||||
def ww{gw, E} = [gw/width{E}]E
|
||||
def fac = width{X}/width{W}
|
||||
|
||||
maskedLoop{bulk, l, {i, M} => {
|
||||
cw:= load{cast_p{ww{cif{fac==1, 256, 128}, u8}, w + i*32/fac}, 0}
|
||||
cx:= load{cast_p{ww{256, X}, x}, i}
|
||||
cwc:= cvt{W, ww{256, X}, cw}
|
||||
if (anyne{cwc,cx,M}) return{0}
|
||||
}}
|
||||
}
|
||||
1
|
||||
}
|
||||
|
||||
'avx2_equal_1_1' = equal{u1, u1} # todo maybe unify with 8_8 and make the non-bit diagonal shift by 3
|
||||
'avx2_equal_1_8' = equal{u1, u8}
|
||||
'avx2_equal_1_16' = equal{u1, u16}
|
||||
'avx2_equal_1_32' = equal{u1, u32}
|
||||
'avx2_equal_1_f64' = equal{u1, f64}
|
||||
|
||||
'avx2_equal_8_8' = equal{i8, i8}
|
||||
|
||||
'avx2_equal_s8_16' = equal{i8, i16}
|
||||
'avx2_equal_s8_32' = equal{i8, i32}
|
||||
'avx2_equal_s16_32' = equal{i16, i32}
|
||||
|
||||
'avx2_equal_s8_f64' = equal{i8, f64}
|
||||
'avx2_equal_s16_f64'= equal{i16, f64}
|
||||
'avx2_equal_s32_f64'= equal{i32, f64}
|
||||
'avx2_equal_f64_f64'= equal{f64, f64}
|
||||
|
||||
'avx2_equal_u8_16' = equal{u8, u16}
|
||||
'avx2_equal_u8_32' = equal{u8, u32}
|
||||
'avx2_equal_u16_32' = equal{u16, u32}
|
||||
35
src/singeli/src/mask.singeli
Normal file
35
src/singeli/src/mask.singeli
Normal file
@ -0,0 +1,35 @@
|
||||
# get mask of first n items; n>0 & n<vcount{T}
|
||||
|
||||
def maskInit{T} = {
|
||||
def n = 256/width{T} - 1
|
||||
merge{broadcast{n, -1}, broadcast{n, 0}}
|
||||
}
|
||||
mask256_8 :*i8 = maskInit{i8}
|
||||
mask256_16:*i16 = maskInit{i16}
|
||||
mask256_32:*i32 = maskInit{i32}
|
||||
mask256_64:*i64 = maskInit{i64}
|
||||
mask256_1:*u8 = apply{merge, each{{x} => {
|
||||
merge{broadcast{31, 255}, (1<<x)-1, broadcast{32, 0}}
|
||||
}, iota{8}}}
|
||||
|
||||
|
||||
def maskOf{T,n & w256{T, 8}} = load{cast_p{[32]u8, mask256_8 + (n^31)}, 0}
|
||||
def maskOf{T,n & w256{T,16}} = load{cast_p{[16]u16, mask256_16 + (n^15)}, 0}
|
||||
def maskOf{T,n & w256{T,32}} = load{cast_p{[ 8]u32, mask256_32 + (n^7)}, 0}
|
||||
def maskOf{T,n & w256{T,64}} = load{cast_p{[ 4]u64, mask256_64 + (n^3)}, 0}
|
||||
def maskOfBit{T,n & w256{T}} = load{cast_p{[32]u8, mask256_1 + (n>>3)^31 + 64*(n&7)}, 0}
|
||||
|
||||
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y}
|
||||
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}}
|
||||
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
|
||||
def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
|
||||
def maskNone{x} = x
|
||||
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
|
||||
def maskAfter{n} = {
|
||||
def mask{x:X & isvec{X}} = x&cast_v{X,maskOf{X,n}}
|
||||
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
|
||||
def mask{x:X, mode=='all bits zeroes'} = andIsZero{x, cast_v{X,maskOfBit{X,n}}}
|
||||
def mask{x==0} = 1
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user