separate SSE2, implement SSE2 dyarith & bitsel

also separate maskstore to homogeneous & top bit mask variants, and make blend require equal element widths
This commit is contained in:
dzaima 2023-04-01 18:52:23 +03:00
parent 8c76561883
commit 3a45eac1aa
30 changed files with 273 additions and 217 deletions

View File

@ -73,6 +73,7 @@ _assert_ ← { 𝔾𝕩?𝕩; Log 𝕨𝔽𝕩 ⋄ OnExit@ ⋄ •Exit 1}
SepArgs {' ' ((¬-˜×·+`»>)) 𝕩}
Lowercase {𝕩 - (-´"Aa")×(𝕩'A')𝕩'Z'}
Uppercase {𝕩 - (-´"aA")×(𝕩'a')𝕩'z'}
getOpt {
args ר/ •args
@ -146,7 +147,7 @@ AtRoot ← rootDir⊸•file.At
po { # parsed options
verbose, J, rebuildAll, rebuildSingeli, tui, output, clangd, versionName,
os, arch, native, has,
os, arch, native, has, avx2
emcc, wasm, replxx, singeli,
ffi, o3, debug, rtverify, heapverify,
bytecodeDir, replxxDir, singeliDir,
@ -200,15 +201,16 @@ po ← { # parsed options
{𝕊: "Error: Unsupported 'has' options; options:"1", "¨𝕩}_assert_(´has) "avx2""bmi2""slow-pdep"
{𝕊: "Error: Cannot have 'has' options on architecture '"arch"'; add an argument of "compat"arch=x86-64""target_arch=x86-64"}_assert_¬ (arch"x86-64") 0has
avx2 (arch"x86-64") singeli native "avx2"<has
{𝕊: "Target: "os" "arch 1 ", "¨ hasnative/"native"} _verboseLog@
{𝕊:
arch"generic"
Log((0=has) @GetOpt"native")
"Note: on x86-64, a generic build doesn't include many optimizations; ",
compat"add an argument of 'native' or 'has=avx2' if possible; add 'native=0' to hide""use 'make o3n-singeli' (targets the current CPU, assumes AVX2) or 'make o3-singeli has=avx2' if possible"
} (arch"x86-64") singeli (¬native) ¬"avx2"<has
} (arch"x86-64") singeli ¬avx2
pie {(¬shared)linuxarch"x86-64"? 0; 1} DOpt "pie"
@ -262,9 +264,9 @@ po ← { # parsed options
args GetOpt "f"
args GetOpt "CCFLAGS"
args (singeliarch "x86-64") / "-DSINGELI_X86_64"
args (singeliarch"aarch64") / "-DSINGELI_NEON"
args (singeliarch"generic") / "-DSINGELI_SIMD"
args (singeli arch "x86-64") / "-DSINGELI_X86_64" avx2 / "-DSINGELI_AVX2"
args (singeli arch"aarch64") / "-DSINGELI_NEON"
args (singeli avx2arch"aarch64") / "-DSINGELI_SIMD"
args ( wasm) / "-DWASM"
args ( wasi) / "-DWASI", "-DNO_MMAP", "-DCATCH_ERRORS=0", "-D_WASI_EMULATED_MMAN", "--target=wasm32-wasi"
args ( emcc) / "-DEMCC", "-O3"
@ -587,13 +589,16 @@ cachedBin‿linkerCache ← {
"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"
cbqnSrc cbqnSrc clangd.Files "src"
singeliMap 1¨ (´¨ (po.arch) = ¨)/
"xa.""src/builtins/arithm.c""monarith",
"xa.""src/core/stuff.c""equal", "xa.""src/utils/mut.c""copy", "xa.""src/utils/bits.c""bits"
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp", "xa.""src/builtins/squeeze.c""squeeze"
"x..""src/builtins/select.c""select", "x..""src/builtins/fold.c""fold", "x..""src/builtins/scan.c""scan"
"x..""src/builtins/scan.c""neq", "x..""src/builtins/slash.c""slash", "x..""src/builtins/slash.c""constrep"
"xag""src/builtins/transpose.c""transpose"
singeliMap 1¨ ({´ ((po.avx2/"X")po.arch) 𝕩}¨ ¨)/
"Xa.""src/builtins/arithm.c""monarith", "Xa.""src/core/stuff.c""equal",
"xa.""src/builtins/arithd.c""dyarith", "Xa.""src/builtins/cmp.c""cmp",
"Xa.""src/builtins/squeeze.c""squeeze", "Xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits",
"xag""src/builtins/transpose.c""transpose",
"X..""src/builtins/select.c""select", "X..""src/builtins/scan.c""scan",
"X..""src/builtins/fold.c""fold", "X..""src/builtins/slash.c""slash",
"X..""src/builtins/slash.c""constrep",
"X..""src/builtins/scan.c""neq",
objs
@ -612,7 +617,7 @@ cachedBin‿linkerCache ← {
singeliObjs @
singeliCache {
¬po.singeli? @;
singeliCache GetCache "singeli", "Singeli generated code location", po.native, po.arch, po.singeliFlags, po.singeliDir
singeliCache GetCache "singeli", "Singeli generated code location", po.native, po.arch, po.avx2, po.has, po.singeliFlags, po.singeliDir
# genArithTables
ga "src/singeli/src/genArithTables.bqn"
@ -627,7 +632,7 @@ cachedBin‿linkerCache ← {
singeliArgs po.singeliFlags"-l", "gen="AtRoot singeliCache.folder{
po.native? ;
"-a" {"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch # TODO expand based on po.has (and then include in cache key)
"-a" ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ','¨ Uppercase "avx2""bmi2"/po.has
}
{𝕊: "Singeli args: "•Repr singeliArgs} _verboseLog @
singeliObjs {MakeSingeliInv singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩".singeli", (𝕩"dyarith")/gaRule}¨ 1¨singeliMap

View File

@ -21,7 +21,7 @@ B atan2_c2(B, B, B);
typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
#if SINGELI_SIMD
#if SINGELI_SIMD || SINGELI_X86_64
#include "../singeli/c/arithdDispatch.c"
static AndBytesFn andBytes_fn = simd_andBytes;
#else
@ -217,7 +217,7 @@ static B modint_AS(B w, B xv) { return modint_AA(w, C2(shape, C1(fne, incG(w))
#undef GC2f
#if SINGELI_SIMD
#if SINGELI_SIMD || SINGELI_X86_64
#define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); }
AA_DISPATCH(add) AA_DISPATCH(or)
AA_DISPATCH(sub)

View File

@ -15,7 +15,7 @@
#include "../builtins.h"
#include "../utils/mut.h"
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SINGELI_FILE fold
#include "../utils/includeSingeli.h"
#endif
@ -92,7 +92,7 @@ B sum_c1(B t, B x) {
}
r += s;
} else {
#if SINGELI_X86_64
#if SINGELI_AVX2
r = avx2_sum_f64(xv, ia);
#else
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
@ -133,7 +133,7 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
#if SINGELI_X86_64
#if SINGELI_AVX2
static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); }
static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); }
#else

View File

@ -61,7 +61,7 @@
rp[j] = xi; \
}
#if SINGELI_X86_64
#if SINGELI_AVX2
extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
@ -227,7 +227,7 @@ B SORT_C1(B t, B x) {
#undef SORT_C1
#undef INSERTION_SORT
#undef COUNTING_SORT
#if SINGELI_X86_64
#if SINGELI_AVX2
#undef WRITE_SPARSE_i8
#undef WRITE_SPARSE_i16
#endif

View File

@ -11,7 +11,7 @@
#define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); }
#define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); }
#if SINGELI_X86_64
#if SINGELI_AVX2
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
#define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0);
@ -29,7 +29,7 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v
#define RADIX_SUM_4_u32 RDX_SUM_4(u32)
#endif
#if SINGELI_X86_64 && !USZ_64
#if SINGELI_AVX2 && !USZ_64
#define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0);
#define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0);
#define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0);

View File

@ -9,7 +9,7 @@
static u64 vg_rand(u64 x) { return x; }
#endif
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SINGELI_FILE scan
#include "../utils/includeSingeli.h"
#if __PCLMUL__
@ -22,7 +22,7 @@ static u64 vg_rand(u64 x) { return x; }
B scan_ne(B x, u64 p, u64 ia) { // consumes x
u64* xp = bitarr_ptr(x);
u64* rp; B r=m_bitarrv(&rp,ia);
#if SINGELI_X86_64 && __PCLMUL__
#if SINGELI_AVX2 && __PCLMUL__
clmul_scan_ne(p, xp, rp, BIT_N(ia));
#if USE_VALGRIND
if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]);
@ -78,7 +78,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
} else {
void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re));
#define SUM_BITWISE(T) { T c=0; for (usz i=0; i<ia; i++) { c+= bitp_get(xp,i); ((T*)rp)[i]=c; } }
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SUM(W,T) avx2_bcs##W(xp, rp, ia);
#else
#define SUM(W,T) SUM_BITWISE(T)
@ -96,7 +96,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
}
// min/max-scan
#if SINGELI_X86_64
#if SINGELI_AVX2
#define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I);
#else
#define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i<ia; i++) { if (xp[i] C c)c=xp[i]; rp[i]=c; }

View File

@ -41,7 +41,7 @@
#include "../utils/mut.h"
#include "../builtins.h"
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SINGELI_FILE select
#include "../utils/includeSingeli.h"
#endif
@ -175,7 +175,7 @@ B select_c2(B t, B w, B x) {
u8 we = TI(w,elType);
#if SINGELI_X86_64
#if SINGELI_AVX2
#define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \
if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds");
bool bool_use_simd = we==el_i8 && xl==0 && xia<=128;

View File

@ -75,13 +75,13 @@
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SINGELI_FILE slash
#include "../utils/includeSingeli.h"
#endif
#endif
#if SINGELI_X86_64
#if SINGELI_AVX2
#define SINGELI_FILE constrep
#include "../utils/includeSingeli.h"
@ -154,7 +154,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) {
static void where_block_u16(u64* src, u16* dst, usz len, usz sum) {
assert(len <= bsp_max);
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len);
#else
if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0);
@ -227,7 +227,7 @@ static B where(B x, usz xia, u64 s) {
u64* xp = bitarr_ptr(x);
usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1;
if (xia <= 128) {
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8);
bmipopc_1slash8(xp, rp, xia);
FINISH_OVERALLOC_A(r, s, 8);
@ -235,7 +235,7 @@ static B where(B x, usz xia, u64 s) {
i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,);
#endif
} else if (xia <= 32768) {
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
if (s >= xia/8) {
i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16);
bmipopc_1slash16(xp, rp, xia);
@ -257,7 +257,7 @@ static B where(B x, usz xia, u64 s) {
}
}
} else if (xia <= (usz)I32_MAX+1) {
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
i32* rp; r = m_i32arrv(&rp, s);
#else
i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4);
@ -272,7 +272,7 @@ static B where(B x, usz xia, u64 s) {
} else {
bs = bit_sum(xp,b);
}
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
if (bs >= b/8+b/16) {
bmipopc_1slash16(xp, buf, b);
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
@ -358,7 +358,7 @@ B grade_bool(B x, usz xia, bool up) {
u64* xp = bitarr_ptr(x);
u64 sum = bit_sum(xp, xia);
u64 l0 = up? xia-sum : sum; // Length of first set of indices
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
if (xia < 16) { BRANCHLESS_GRADE(i8) }
else if (xia <= 1<<15) {
B notx = bit_negate(inc(x));
@ -478,7 +478,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { DENSE; } \
break; }
#if SINGELI_X86_64 && FAST_PDEP
#if SINGELI_AVX2 && FAST_PDEP
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
#else
@ -535,7 +535,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
#define SUM_CORE(T, WV, PREP, INC) \
SCAN_CORE(WV, PREP; rp[ij]+=INC, , PLUS_SCAN(T))
#if SINGELI_X86_64
#if SINGELI_AVX2
#define IND_BY_SCAN \
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, avx2_scan_max_i32(rp+k,rp+k,e-k))
#else
@ -829,7 +829,7 @@ B slash_c2(B t, B w, B x) {
u8 xk = xl-3;
void* rv = m_tyarrv(&r, 1<<xk, s, xt);
void* xv = tyany_ptr(x);
#if SINGELI_X86_64
#if SINGELI_AVX2
#define CASE(L,T) case L: constrep_##T(wv, xv, rv, xlen); break;
#else
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }

View File

@ -70,6 +70,7 @@ def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b}
def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b}
def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a}
def rsqrt{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcp{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}}
def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a}
def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a}
@ -82,7 +83,6 @@ def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b}
def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b}
def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b}
def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a}
def rsqrt{a:[4]f64} = emit{[4]f64, '_mm256_rsqrt_pd', a}
def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}}
def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
@ -113,8 +113,8 @@ def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}

View File

@ -57,7 +57,6 @@ def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
# super questionable pack - assumes high halves are zero
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}}
@ -78,35 +77,19 @@ def absu{a:[32]i8 } = emit{[32]u8 , '_mm256_abs_epi8', a}
def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a}
def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a}
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}
def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
# saturating add/sub
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}
def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}
def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}
def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}
def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
# structural operations
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
# maskstore with all cases defined, at the cost of not being a single instruction
def maskstoreF{p, m, x:T} = store{p, 0, homBlend{load{p}, x, m}}
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
@ -115,9 +98,9 @@ def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm2
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M,16}} = topBlend{f, t, [32]i8~~m}
def topBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def homBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w256{T, 16} & w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
@ -129,6 +112,14 @@ def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
# masked store; F variants may not be a single instruction
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:P, m:M, v:T & w256i{M} & w256{T,elwidth{M}} & eltype{P}==T} = topMaskStore{p, m, v}
def topMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
# mask stuff
def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}

View File

@ -129,15 +129,15 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
# base cases
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def andnz{...x} = assert{'andnz not supported', show{...x}}
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
def topMask{...x} = assert{'topMask not supported', show{...x}}
def packLo{...x} = assert{'packLo not supported', show{...x}}
def packHi{...x} = assert{'packHi not supported', show{...x}}
def mul12{...x} = assert{'mul12 not supported', show{...x}}
def unpack{...x} = assert{'unpack not supported', show{...x}}
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
def packQ{...x} = assert{'packQ not supported', show{...x}}
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
@ -148,8 +148,9 @@ def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
def pair{...x} = assert{'pair not supported', show{...x}}
def cvt{...x} = assert{'cvt not supported', show{...x}}
def cvt{T, x:X & T==eltype{X}} = x
def shuf{...x} = assert{'shuf not supported', show{...x}}
def shuf16Lo{...x} = assert{'shuf16Lo not supported', show{...x}}
def shuf16Hi{...x} = assert{'shuf16Hi not supported', show{...x}}
def homAll{...x} = assert{'homAll not supported', show{...x}}
def homAny{...x} = assert{'homAny not supported', show{...x}}
def homMask{...x} = assert{'homMask not supported', show{...x}}
@ -159,7 +160,16 @@ def storeLow{...x} = assert{'storeLow not supported', show{...x}}
def sel{...x} = assert{'sel not supported', show{...x}}
def make{...x} = assert{'make not supported', show{...x}}
def half{...x} = assert{'half not supported', show{...x}}
def maskstoreF{...x} = assert{'maskstoreF not supported', show{...x}}
def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}}
def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}}
def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}}
def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}}
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
def cvt{T, x:X & T==eltype{X}} = x
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
@ -171,7 +181,7 @@ def broadcast{T, v & isprim{T}} = v
def iota{n & knum{n}} = @collect(i to n) i
def broadcast{n, v & knum{n}} = @collect(n) v
def arch_defvw = tern{hasarch{'X86_64'}, 256, 128}
def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}

View File

@ -63,10 +63,15 @@ def spreadBits{T==[32]u8, a:u32} = {
e == (d&e)
}
def spreadBits{T==[16]u8, a:u16} = {
def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
}
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
b:= [16]u8~~[8]u16**a
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
}
def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
b:= make{T, 1<<iota{vcount{T}}}

View File

@ -1,10 +1,12 @@
include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'X86_64'}) {
include './sse3'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}

View File

@ -2,7 +2,7 @@ include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'X86_64'}) {
include './sse3'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {

View File

@ -1,5 +1,5 @@
include './base'
include './sse3'
include './sse'
include './avx'
include './avx2'
include './mask'
@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
}}
setlabel{end}
q := nr & (step-1)
if (q) maskstoreF{rv+e, maskOf{V, q}, s}
if (q) homMaskStoreF{rv+e, maskOf{V, q}, s}
}
}
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
@ -122,7 +122,7 @@ fn rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void =
}
setlabel{end}
q := (re+step) - r
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
if (q) homMaskStoreF{*V~~r, maskOf{V, q}, s}
}
fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {

View File

@ -1,6 +1,6 @@
include './base'
if (hasarch{'X86_64'}) {
include './sse3'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {

View File

@ -1,10 +1,12 @@
include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'X86_64'}) {
include './sse3'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
@ -44,7 +46,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasar
def wp = unpackQ{w, T ~~ (T**0 > w)}
def xp = unpackQ{x, T ~~ (T**0 > x)}
def rp = each{__mul, wp, xp}
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp}
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
if (M{0}) { # masked check
tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}}
} else { # unmasked check; can do check in a simpler way
@ -57,8 +59,8 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasa
tup{rl, tup{'anyne', rh, rl>>15}}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
max:= [8]f32 ~~ [8]u32**0x4efffffe
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
max:= to_el{f32, (ty_u{T})**0x4efffffe}
def cf32{x:X} = emit{to_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
f32mul:= cf32{w} * cf32{x}
tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}}
# TODO fallback to the below if the above fails

View File

@ -2,7 +2,7 @@ include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'X86_64'}) {
include './sse3'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {

View File

@ -1,5 +1,5 @@
include './base'
include './sse3'
include './sse'
include './avx'
include './avx2'
include './mask'

View File

@ -9,7 +9,7 @@ mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u
mask256:*i64 = merge{4 ** -1, 4 ** 0}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
# get mask of first n items; 0 ≤ n ≤ vcount{T}
# get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T}
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64}
@ -31,6 +31,7 @@ def maskAfter{n} = {
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}}
def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{X, mode=='to homogeneous bits'} = maskOf{X,n}
def mask{mode=='count'} = n
def mask{x & istup{x} & tuplen{x}==1} = tup{mask{tupsel{0,x}}}
def mask{x==0} = 1
@ -47,7 +48,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
def TF = to_el{E0, T}
xu:= narrow{E0, x}
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, undefPromote{TF, xu}}
if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}}
else storeLow{rpos, vcount{T}*width{E0}, xu}
}

View File

@ -2,7 +2,7 @@ include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'X86_64'}) {
include './sse3'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {

View File

@ -158,5 +158,4 @@ def homMask{x:T & nvecu{T} & T==[16]u8} = {
}
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def maskstore{p:P, m:M, x:T & eltype{P}==T & nvec{T}} = store{p, 0, bitBlend{load{p}, x, m}}
def maskstoreF{p:P, m:M, x:T & nvec{T}} = maskstore{p, m, x}
def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}

View File

@ -1,5 +1,5 @@
include './base'
include './sse3'
include './sse'
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}

View File

@ -1,5 +1,5 @@
include './base'
include './sse3'
include './sse'
include './avx'
include './avx2'
include './mask'
@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
e:= len/step
@for (xv, rv over e) rv = scan{xv,p}
q:= len & (step-1)
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
if (q) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
}
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
def last{v, p} = op{pre{v}, p}
@ -141,7 +141,7 @@ fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
if (jv+vl <= l) {
store{p, j, v}
} else {
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v}
if (jv < l) homMaskStoreF{rv+j, maskOf{V, l - jv}, v}
return{}
}
}

View File

@ -1,6 +1,6 @@
include './base'
include './cbqnDefs'
include './sse3'
include './sse'
include './avx'
include './avx2'
include './mask'
@ -68,7 +68,7 @@ def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
}
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
def bblend {m}{ft} = topBlend{tupsel{0,ft}, tupsel{1,ft}, m}
def bblend {m}{ft} = {def {f,t} = ft; homBlend{f, t, type{f} ~~ m}}
def bblendn{m}{tf} = bblend{m}{reverse{tf}}
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})

View File

@ -1,6 +1,6 @@
include './base'
if (hasarch{'X86_64'}) {
include './sse3'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {

View File

@ -0,0 +1,55 @@
include './sse2'
### SSSE3 ###
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
### SSE4.1 ###
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
# arith
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b}
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
# conversion
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
# mask
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
### SSE4.2 ###
def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b}

View File

@ -28,6 +28,7 @@ def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', p
def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
# broadcast
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
@ -36,6 +37,7 @@ def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T}
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
# make from elements
local def makeGen{T,s,x} = emit{T, s, ...each{{c}=>promote{eltype{T},c}, x}}
def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs}
def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs}
@ -47,34 +49,47 @@ def make{T, x & w128{T} & istup{x}} = make{T, ...x}
def iota{T & w128{T}} = make{T, ...iota{vcount{T}}}
# integer comparison
def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b}
def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) }
# signed comparison
def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{a^s} }
def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{a^s} }
def __le{a:T,b:T & w128u{T}} = ~(a>b)
def __ge{a:T,b:T & w128u{T}} = ~(a<b)
def __le{a:T,b:T & w128u{T, 8}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T, 8}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T, 8}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b)
# rest of integer comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
# float comparison
def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b}
def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b}
def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b}
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
# bit arith
def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
# signed comparison
def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __gt{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __eq{a:T,b:T & T==[16]u8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]u16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]u32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T}} = ~(a<=b)
# rest of comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
# shift
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b}
@ -89,19 +104,7 @@ def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
# shuffle stuff
def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
# arith
# integer arith
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b}
def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b}
@ -112,38 +115,48 @@ def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b}
def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b}
def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
def __mul {a:T,b:T & [8]i16==T} = emit{T, '_mm_mullo_epi16', a, b}
def __mulhi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def __mulhi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
def __mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
def __mul{a:T,b:T & [4]i32==T} = {
def mu{x, y} = [4]i32 ~~ __mul32{[2]u64~~x, [2]u64~~y}
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
lo:= mu{a, b}
hi:= mu{...sw{4b2301, a, b}}
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
}
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
def max{a:T,b:T & w128i{T}} = homBlend{a, b, a<b}
def max{a:T,b:T & w128u{T,16}} = __subs{a,b}+b
def min{a:T,b:T & w128u{T,16}} = a-__subs{a,b}
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
# f32 arith
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}
# saturating
def __adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def __adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b}
def __subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def __subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b}
def __adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def __adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b}
def __subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def __subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b}
# float arith
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
def rsqrt{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[2]u32 ** 0x7FFFFFFF}}
# f64 arith
def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
def rsqrt{a:[2]f64} = emit{[2]f64, '_mm_rsqrt_pd', a}
def rcp{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
# mask stuff
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T & w128{T, 16}} = topMask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}
def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T & w128{T}} = topMask{x}
@ -156,56 +169,29 @@ def topAll{x:T & w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
# the lone SSE2 extract
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
### SSE4.1; TODO either move to own file or rename this file
# rest of extract
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
# float
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}
def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}
def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
# conversion
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X}
def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24}
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def __eq{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
# SSE4.2
def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}
def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m))
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}

View File

@ -1,8 +1,8 @@
include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'X86_64'}) {
include './sse3'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {
@ -32,7 +32,7 @@ def unpack_to{f, l, x} = {
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
# Square kernel where width is a full vector
def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x}
def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x}
def load2{a:T, b:T} = pair{load{a}, load{b}}
def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = {
@ -164,7 +164,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : void
rp:*T = *T~~r0
xp:*T = *T~~x0
if (hasarch{'X86_64'} and w>=k and h>=k) {
if (hasarch{'AVX2'} and w>=k and h>=k) {
transpose_with_kernel{T, k, kh, call_base, rp, xp, w, h, ws, hs}
} else {
if (h==2 and h==hs) @for (x0 in xp, x1 in xp+ws over i to w) { store{rp, i*2, x0}; store{rp, i*2+1, x1} }

View File

@ -2,7 +2,7 @@
#include "mut.h"
#if SINGELI_SIMD
#if SINGELI_SIMD || SINGELI_X86_64
#define SINGELI_FILE bits
#include "../utils/includeSingeli.h"
#define bitselFns simd_bitsel
@ -58,7 +58,7 @@ NOINLINE B bit_sel(B b, B e0, B e1) {
sel:;
void* rp = m_tyarrlc(&r, width, b, type);
#if SINGELI_SIMD
#if SINGELI_SIMD || SINGELI_X86_64
bitselFns[width](rp, bp, e0i, e1i, ia);
#else
switch(width) {