separate SSE2, implement SSE2 dyarith & bitsel

also separate maskstore to homogeneous & top bit mask variants, and make blend require equal element widths
This commit is contained in:
dzaima 2023-04-01 18:52:23 +03:00
parent 8c76561883
commit 3a45eac1aa
30 changed files with 273 additions and 217 deletions

View File

@ -73,6 +73,7 @@ _assert_ ← { 𝔾𝕩?𝕩; Log 𝕨𝔽𝕩 ⋄ OnExit@ ⋄ •Exit 1}
SepArgs {' ' ((¬-˜×·+`»>)) 𝕩} SepArgs {' ' ((¬-˜×·+`»>)) 𝕩}
Lowercase {𝕩 - (-´"Aa")×(𝕩'A')𝕩'Z'} Lowercase {𝕩 - (-´"Aa")×(𝕩'A')𝕩'Z'}
Uppercase {𝕩 - (-´"aA")×(𝕩'a')𝕩'z'}
getOpt { getOpt {
args ר/ •args args ר/ •args
@ -146,7 +147,7 @@ AtRoot ← rootDir⊸•file.At
po { # parsed options po { # parsed options
verbose, J, rebuildAll, rebuildSingeli, tui, output, clangd, versionName, verbose, J, rebuildAll, rebuildSingeli, tui, output, clangd, versionName,
os, arch, native, has, os, arch, native, has, avx2
emcc, wasm, replxx, singeli, emcc, wasm, replxx, singeli,
ffi, o3, debug, rtverify, heapverify, ffi, o3, debug, rtverify, heapverify,
bytecodeDir, replxxDir, singeliDir, bytecodeDir, replxxDir, singeliDir,
@ -200,15 +201,16 @@ po ← { # parsed options
{𝕊: "Error: Unsupported 'has' options; options:"1", "¨𝕩}_assert_(´has) "avx2""bmi2""slow-pdep" {𝕊: "Error: Unsupported 'has' options; options:"1", "¨𝕩}_assert_(´has) "avx2""bmi2""slow-pdep"
{𝕊: "Error: Cannot have 'has' options on architecture '"arch"'; add an argument of "compat"arch=x86-64""target_arch=x86-64"}_assert_¬ (arch"x86-64") 0has {𝕊: "Error: Cannot have 'has' options on architecture '"arch"'; add an argument of "compat"arch=x86-64""target_arch=x86-64"}_assert_¬ (arch"x86-64") 0has
avx2 (arch"x86-64") singeli native "avx2"<has
{𝕊: "Target: "os" "arch 1 ", "¨ hasnative/"native"} _verboseLog@ {𝕊: "Target: "os" "arch 1 ", "¨ hasnative/"native"} _verboseLog@
{𝕊: {𝕊:
arch"generic"
Log((0=has) @GetOpt"native") Log((0=has) @GetOpt"native")
"Note: on x86-64, a generic build doesn't include many optimizations; ", "Note: on x86-64, a generic build doesn't include many optimizations; ",
compat"add an argument of 'native' or 'has=avx2' if possible; add 'native=0' to hide""use 'make o3n-singeli' (targets the current CPU, assumes AVX2) or 'make o3-singeli has=avx2' if possible" compat"add an argument of 'native' or 'has=avx2' if possible; add 'native=0' to hide""use 'make o3n-singeli' (targets the current CPU, assumes AVX2) or 'make o3-singeli has=avx2' if possible"
} (arch"x86-64") singeli (¬native) ¬"avx2"<has } (arch"x86-64") singeli ¬avx2
pie {(¬shared)linuxarch"x86-64"? 0; 1} DOpt "pie" pie {(¬shared)linuxarch"x86-64"? 0; 1} DOpt "pie"
@ -262,9 +264,9 @@ po ← { # parsed options
args GetOpt "f" args GetOpt "f"
args GetOpt "CCFLAGS" args GetOpt "CCFLAGS"
args (singeliarch "x86-64") / "-DSINGELI_X86_64" args (singeli arch "x86-64") / "-DSINGELI_X86_64" avx2 / "-DSINGELI_AVX2"
args (singeliarch"aarch64") / "-DSINGELI_NEON" args (singeli arch"aarch64") / "-DSINGELI_NEON"
args (singeliarch"generic") / "-DSINGELI_SIMD" args (singeli avx2arch"aarch64") / "-DSINGELI_SIMD"
args ( wasm) / "-DWASM" args ( wasm) / "-DWASM"
args ( wasi) / "-DWASI", "-DNO_MMAP", "-DCATCH_ERRORS=0", "-D_WASI_EMULATED_MMAN", "--target=wasm32-wasi" args ( wasi) / "-DWASI", "-DNO_MMAP", "-DCATCH_ERRORS=0", "-D_WASI_EMULATED_MMAN", "--target=wasm32-wasi"
args ( emcc) / "-DEMCC", "-O3" args ( emcc) / "-DEMCC", "-O3"
@ -587,13 +589,16 @@ cachedBin‿linkerCache ← {
"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c" "src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"
cbqnSrc cbqnSrc clangd.Files "src" cbqnSrc cbqnSrc clangd.Files "src"
singeliMap 1¨ (´¨ (po.arch) = ¨)/ singeliMap 1¨ ({´ ((po.avx2/"X")po.arch) 𝕩}¨ ¨)/
"xa.""src/builtins/arithm.c""monarith", "Xa.""src/builtins/arithm.c""monarith", "Xa.""src/core/stuff.c""equal",
"xa.""src/core/stuff.c""equal", "xa.""src/utils/mut.c""copy", "xa.""src/utils/bits.c""bits" "xa.""src/builtins/arithd.c""dyarith", "Xa.""src/builtins/cmp.c""cmp",
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp", "xa.""src/builtins/squeeze.c""squeeze" "Xa.""src/builtins/squeeze.c""squeeze", "Xa.""src/utils/mut.c""copy",
"x..""src/builtins/select.c""select", "x..""src/builtins/fold.c""fold", "x..""src/builtins/scan.c""scan" "xa.""src/utils/bits.c""bits",
"x..""src/builtins/scan.c""neq", "x..""src/builtins/slash.c""slash", "x..""src/builtins/slash.c""constrep" "xag""src/builtins/transpose.c""transpose",
"xag""src/builtins/transpose.c""transpose" "X..""src/builtins/select.c""select", "X..""src/builtins/scan.c""scan",
"X..""src/builtins/fold.c""fold", "X..""src/builtins/slash.c""slash",
"X..""src/builtins/slash.c""constrep",
"X..""src/builtins/scan.c""neq",
objs objs
@ -612,7 +617,7 @@ cachedBin‿linkerCache ← {
singeliObjs @ singeliObjs @
singeliCache { singeliCache {
¬po.singeli? @; ¬po.singeli? @;
singeliCache GetCache "singeli", "Singeli generated code location", po.native, po.arch, po.singeliFlags, po.singeliDir singeliCache GetCache "singeli", "Singeli generated code location", po.native, po.arch, po.avx2, po.has, po.singeliFlags, po.singeliDir
# genArithTables # genArithTables
ga "src/singeli/src/genArithTables.bqn" ga "src/singeli/src/genArithTables.bqn"
@ -627,7 +632,7 @@ cachedBin‿linkerCache ← {
singeliArgs po.singeliFlags"-l", "gen="AtRoot singeliCache.folder{ singeliArgs po.singeliFlags"-l", "gen="AtRoot singeliCache.folder{
po.native? ; po.native? ;
"-a" {"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch # TODO expand based on po.has (and then include in cache key) "-a" ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ','¨ Uppercase "avx2""bmi2"/po.has
} }
{𝕊: "Singeli args: "•Repr singeliArgs} _verboseLog @ {𝕊: "Singeli args: "•Repr singeliArgs} _verboseLog @
singeliObjs {MakeSingeliInv singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩".singeli", (𝕩"dyarith")/gaRule}¨ 1¨singeliMap singeliObjs {MakeSingeliInv singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩".singeli", (𝕩"dyarith")/gaRule}¨ 1¨singeliMap

View File

@ -21,7 +21,7 @@ B atan2_c2(B, B, B);
typedef void (*AndBytesFn)(u8*, u8*, u64, u64); typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
#include "../singeli/c/arithdDispatch.c" #include "../singeli/c/arithdDispatch.c"
static AndBytesFn andBytes_fn = simd_andBytes; static AndBytesFn andBytes_fn = simd_andBytes;
#else #else
@ -217,7 +217,7 @@ static B modint_AS(B w, B xv) { return modint_AA(w, C2(shape, C1(fne, incG(w))
#undef GC2f #undef GC2f
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
#define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); } #define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); }
AA_DISPATCH(add) AA_DISPATCH(or) AA_DISPATCH(add) AA_DISPATCH(or)
AA_DISPATCH(sub) AA_DISPATCH(sub)

View File

@ -15,7 +15,7 @@
#include "../builtins.h" #include "../builtins.h"
#include "../utils/mut.h" #include "../utils/mut.h"
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SINGELI_FILE fold #define SINGELI_FILE fold
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#endif #endif
@ -92,7 +92,7 @@ B sum_c1(B t, B x) {
} }
r += s; r += s;
} else { } else {
#if SINGELI_X86_64 #if SINGELI_AVX2
r = avx2_sum_f64(xv, ia); r = avx2_sum_f64(xv, ia);
#else #else
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i]; r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
@ -133,7 +133,7 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \ static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) } static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32) DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
#if SINGELI_X86_64 #if SINGELI_AVX2
static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); } static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); }
static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); } static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); }
#else #else

View File

@ -61,7 +61,7 @@
rp[j] = xi; \ rp[j] = xi; \
} }
#if SINGELI_X86_64 #if SINGELI_AVX2
extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2); extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2); extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2); extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
@ -227,7 +227,7 @@ B SORT_C1(B t, B x) {
#undef SORT_C1 #undef SORT_C1
#undef INSERTION_SORT #undef INSERTION_SORT
#undef COUNTING_SORT #undef COUNTING_SORT
#if SINGELI_X86_64 #if SINGELI_AVX2
#undef WRITE_SPARSE_i8 #undef WRITE_SPARSE_i8
#undef WRITE_SPARSE_i16 #undef WRITE_SPARSE_i16
#endif #endif

View File

@ -11,7 +11,7 @@
#define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); } #define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); }
#define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); } #define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); }
#if SINGELI_X86_64 #if SINGELI_AVX2
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
#define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0); #define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0);
@ -29,7 +29,7 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v
#define RADIX_SUM_4_u32 RDX_SUM_4(u32) #define RADIX_SUM_4_u32 RDX_SUM_4(u32)
#endif #endif
#if SINGELI_X86_64 && !USZ_64 #if SINGELI_AVX2 && !USZ_64
#define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0); #define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0);
#define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0); #define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0);
#define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0); #define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0);

View File

@ -9,7 +9,7 @@
static u64 vg_rand(u64 x) { return x; } static u64 vg_rand(u64 x) { return x; }
#endif #endif
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SINGELI_FILE scan #define SINGELI_FILE scan
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#if __PCLMUL__ #if __PCLMUL__
@ -22,7 +22,7 @@ static u64 vg_rand(u64 x) { return x; }
B scan_ne(B x, u64 p, u64 ia) { // consumes x B scan_ne(B x, u64 p, u64 ia) { // consumes x
u64* xp = bitarr_ptr(x); u64* xp = bitarr_ptr(x);
u64* rp; B r=m_bitarrv(&rp,ia); u64* rp; B r=m_bitarrv(&rp,ia);
#if SINGELI_X86_64 && __PCLMUL__ #if SINGELI_AVX2 && __PCLMUL__
clmul_scan_ne(p, xp, rp, BIT_N(ia)); clmul_scan_ne(p, xp, rp, BIT_N(ia));
#if USE_VALGRIND #if USE_VALGRIND
if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]); if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]);
@ -78,7 +78,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
} else { } else {
void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re));
#define SUM_BITWISE(T) { T c=0; for (usz i=0; i<ia; i++) { c+= bitp_get(xp,i); ((T*)rp)[i]=c; } } #define SUM_BITWISE(T) { T c=0; for (usz i=0; i<ia; i++) { c+= bitp_get(xp,i); ((T*)rp)[i]=c; } }
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SUM(W,T) avx2_bcs##W(xp, rp, ia); #define SUM(W,T) avx2_bcs##W(xp, rp, ia);
#else #else
#define SUM(W,T) SUM_BITWISE(T) #define SUM(W,T) SUM_BITWISE(T)
@ -96,7 +96,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
} }
// min/max-scan // min/max-scan
#if SINGELI_X86_64 #if SINGELI_AVX2
#define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I); #define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I);
#else #else
#define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i<ia; i++) { if (xp[i] C c)c=xp[i]; rp[i]=c; } #define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i<ia; i++) { if (xp[i] C c)c=xp[i]; rp[i]=c; }

View File

@ -41,7 +41,7 @@
#include "../utils/mut.h" #include "../utils/mut.h"
#include "../builtins.h" #include "../builtins.h"
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SINGELI_FILE select #define SINGELI_FILE select
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#endif #endif
@ -175,7 +175,7 @@ B select_c2(B t, B w, B x) {
u8 we = TI(w,elType); u8 we = TI(w,elType);
#if SINGELI_X86_64 #if SINGELI_AVX2
#define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \ #define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \
if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds"); if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds");
bool bool_use_simd = we==el_i8 && xl==0 && xia<=128; bool bool_use_simd = we==el_i8 && xl==0 && xia<=128;

View File

@ -75,13 +75,13 @@
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SINGELI_FILE slash #define SINGELI_FILE slash
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#endif #endif
#endif #endif
#if SINGELI_X86_64 #if SINGELI_AVX2
#define SINGELI_FILE constrep #define SINGELI_FILE constrep
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
@ -154,7 +154,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) {
static void where_block_u16(u64* src, u16* dst, usz len, usz sum) { static void where_block_u16(u64* src, u16* dst, usz len, usz sum) {
assert(len <= bsp_max); assert(len <= bsp_max);
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len); if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len);
#else #else
if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0);
@ -227,7 +227,7 @@ static B where(B x, usz xia, u64 s) {
u64* xp = bitarr_ptr(x); u64* xp = bitarr_ptr(x);
usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1; usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1;
if (xia <= 128) { if (xia <= 128) {
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8); i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8);
bmipopc_1slash8(xp, rp, xia); bmipopc_1slash8(xp, rp, xia);
FINISH_OVERALLOC_A(r, s, 8); FINISH_OVERALLOC_A(r, s, 8);
@ -235,7 +235,7 @@ static B where(B x, usz xia, u64 s) {
i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,); i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,);
#endif #endif
} else if (xia <= 32768) { } else if (xia <= 32768) {
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
if (s >= xia/8) { if (s >= xia/8) {
i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16);
bmipopc_1slash16(xp, rp, xia); bmipopc_1slash16(xp, rp, xia);
@ -257,7 +257,7 @@ static B where(B x, usz xia, u64 s) {
} }
} }
} else if (xia <= (usz)I32_MAX+1) { } else if (xia <= (usz)I32_MAX+1) {
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
i32* rp; r = m_i32arrv(&rp, s); i32* rp; r = m_i32arrv(&rp, s);
#else #else
i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4); i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4);
@ -272,7 +272,7 @@ static B where(B x, usz xia, u64 s) {
} else { } else {
bs = bit_sum(xp,b); bs = bit_sum(xp,b);
} }
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
if (bs >= b/8+b/16) { if (bs >= b/8+b/16) {
bmipopc_1slash16(xp, buf, b); bmipopc_1slash16(xp, buf, b);
for (usz j=0; j<bs; j++) rq[j] = i+buf[j]; for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
@ -358,7 +358,7 @@ B grade_bool(B x, usz xia, bool up) {
u64* xp = bitarr_ptr(x); u64* xp = bitarr_ptr(x);
u64 sum = bit_sum(xp, xia); u64 sum = bit_sum(xp, xia);
u64 l0 = up? xia-sum : sum; // Length of first set of indices u64 l0 = up? xia-sum : sum; // Length of first set of indices
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
if (xia < 16) { BRANCHLESS_GRADE(i8) } if (xia < 16) { BRANCHLESS_GRADE(i8) }
else if (xia <= 1<<15) { else if (xia <= 1<<15) {
B notx = bit_negate(inc(x)); B notx = bit_negate(inc(x));
@ -478,7 +478,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \ else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { DENSE; } \ else { DENSE; } \
break; } break; }
#if SINGELI_X86_64 && FAST_PDEP #if SINGELI_AVX2 && FAST_PDEP
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
#else #else
@ -535,7 +535,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
#define SUM_CORE(T, WV, PREP, INC) \ #define SUM_CORE(T, WV, PREP, INC) \
SCAN_CORE(WV, PREP; rp[ij]+=INC, , PLUS_SCAN(T)) SCAN_CORE(WV, PREP; rp[ij]+=INC, , PLUS_SCAN(T))
#if SINGELI_X86_64 #if SINGELI_AVX2
#define IND_BY_SCAN \ #define IND_BY_SCAN \
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, avx2_scan_max_i32(rp+k,rp+k,e-k)) SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, avx2_scan_max_i32(rp+k,rp+k,e-k))
#else #else
@ -829,7 +829,7 @@ B slash_c2(B t, B w, B x) {
u8 xk = xl-3; u8 xk = xl-3;
void* rv = m_tyarrv(&r, 1<<xk, s, xt); void* rv = m_tyarrv(&r, 1<<xk, s, xt);
void* xv = tyany_ptr(x); void* xv = tyany_ptr(x);
#if SINGELI_X86_64 #if SINGELI_AVX2
#define CASE(L,T) case L: constrep_##T(wv, xv, rv, xlen); break; #define CASE(L,T) case L: constrep_##T(wv, xv, rv, xlen); break;
#else #else
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; } #define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }

View File

@ -70,6 +70,7 @@ def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b}
def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b} def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b}
def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a} def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a}
def rsqrt{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a} def rsqrt{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcp{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}} def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}}
def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a} def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a}
def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a} def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a}
@ -82,7 +83,6 @@ def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b}
def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b} def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b}
def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b} def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b}
def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a} def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a}
def rsqrt{a:[4]f64} = emit{[4]f64, '_mm256_rsqrt_pd', a}
def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}} def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}}
def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a} def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
@ -113,8 +113,8 @@ def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i} def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i} def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}} def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m} def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}

View File

@ -57,7 +57,6 @@ def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b} def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b} def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b} def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
# super questionable pack - assumes high halves are zero # super questionable pack - assumes high halves are zero
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011} def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}} def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}}
@ -78,35 +77,19 @@ def absu{a:[32]i8 } = emit{[32]u8 , '_mm256_abs_epi8', a}
def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a} def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a}
def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a} def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a}
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b} def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b} def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b} def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b} def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
# saturating add/sub # saturating add/sub
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b} def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b} def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b} def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}
def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}
def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
# structural operations # structural operations
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
# maskstore with all cases defined, at the cost of not being a single instruction
def maskstoreF{p, m, x:T} = store{p, 0, homBlend{load{p}, x, m}}
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
@ -115,9 +98,9 @@ def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm2
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m} def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} def topBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = topBlend{f, t, m} def homBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M,16}} = topBlend{f, t, [32]i8~~m} def homBlend{f:T, t:T, m:M & w256{T, 16} & w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n} def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n} def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
@ -129,6 +112,14 @@ def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
# masked store; F variants may not be a single instruction
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:P, m:M, v:T & w256i{M} & w256{T,elwidth{M}} & eltype{P}==T} = topMaskStore{p, m, v}
def topMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
# mask stuff # mask stuff
def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}

View File

@ -129,15 +129,15 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1 def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
# base cases # base cases
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def andnz{...x} = assert{'andnz not supported', show{...x}} def andnz{...x} = assert{'andnz not supported', show{...x}}
def topBlend{...x} = assert{'topBlend not supported', show{...x}} def topBlend{...x} = assert{'topBlend not supported', show{...x}}
def topMask{...x} = assert{'topMask not supported', show{...x}} def topMask{...x} = assert{'topMask not supported', show{...x}}
def packLo{...x} = assert{'packLo not supported', show{...x}} def packLo{...x} = assert{'packLo not supported', show{...x}}
def packHi{...x} = assert{'packHi not supported', show{...x}} def packHi{...x} = assert{'packHi not supported', show{...x}}
def mul12{...x} = assert{'mul12 not supported', show{...x}} def mul12{...x} = assert{'mul12 not supported', show{...x}}
def unpack{...x} = assert{'unpack not supported', show{...x}}
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
def packQ{...x} = assert{'packQ not supported', show{...x}} def packQ{...x} = assert{'packQ not supported', show{...x}}
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}} def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
@ -148,8 +148,9 @@ def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}} def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
def pair{...x} = assert{'pair not supported', show{...x}} def pair{...x} = assert{'pair not supported', show{...x}}
def cvt{...x} = assert{'cvt not supported', show{...x}} def cvt{...x} = assert{'cvt not supported', show{...x}}
def cvt{T, x:X & T==eltype{X}} = x
def shuf{...x} = assert{'shuf not supported', show{...x}} def shuf{...x} = assert{'shuf not supported', show{...x}}
def shuf16Lo{...x} = assert{'shuf16Lo not supported', show{...x}}
def shuf16Hi{...x} = assert{'shuf16Hi not supported', show{...x}}
def homAll{...x} = assert{'homAll not supported', show{...x}} def homAll{...x} = assert{'homAll not supported', show{...x}}
def homAny{...x} = assert{'homAny not supported', show{...x}} def homAny{...x} = assert{'homAny not supported', show{...x}}
def homMask{...x} = assert{'homMask not supported', show{...x}} def homMask{...x} = assert{'homMask not supported', show{...x}}
@ -159,7 +160,16 @@ def storeLow{...x} = assert{'storeLow not supported', show{...x}}
def sel{...x} = assert{'sel not supported', show{...x}} def sel{...x} = assert{'sel not supported', show{...x}}
def make{...x} = assert{'make not supported', show{...x}} def make{...x} = assert{'make not supported', show{...x}}
def half{...x} = assert{'half not supported', show{...x}} def half{...x} = assert{'half not supported', show{...x}}
def maskstoreF{...x} = assert{'maskstoreF not supported', show{...x}} def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}}
def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}}
def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}}
def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}}
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
def cvt{T, x:X & T==eltype{X}} = x
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b} def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b} def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
@ -171,7 +181,7 @@ def broadcast{T, v & isprim{T}} = v
def iota{n & knum{n}} = @collect(i to n) i def iota{n & knum{n}} = @collect(i to n) i
def broadcast{n, v & knum{n}} = @collect(n) v def broadcast{n, v & knum{n}} = @collect(n) v
def arch_defvw = tern{hasarch{'X86_64'}, 256, 128} def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}

View File

@ -63,10 +63,15 @@ def spreadBits{T==[32]u8, a:u32} = {
e == (d&e) e == (d&e)
} }
def spreadBits{T==[16]u8, a:u16} = { def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}} b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{[16]u8, 1<<(iota{16}&7)}} andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
} }
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
b:= [16]u8~~[8]u16**a
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
}
def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
b:= make{T, 1<<iota{vcount{T}}} b:= make{T, 1<<iota{vcount{T}}}

View File

@ -1,10 +1,12 @@
include './base' include './base'
include './f64' include './f64'
include './cbqnDefs' include './cbqnDefs'
if (hasarch{'X86_64'}) { if (hasarch{'AVX2'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {
include './neon' include './neon'
} }

View File

@ -2,7 +2,7 @@ include './base'
include './cbqnDefs' include './cbqnDefs'
include './f64' include './f64'
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {

View File

@ -1,5 +1,5 @@
include './base' include './base'
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
include './mask' include './mask'
@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
}} }}
setlabel{end} setlabel{end}
q := nr & (step-1) q := nr & (step-1)
if (q) maskstoreF{rv+e, maskOf{V, q}, s} if (q) homMaskStoreF{rv+e, maskOf{V, q}, s}
} }
} }
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
@ -122,7 +122,7 @@ fn rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void =
} }
setlabel{end} setlabel{end}
q := (re+step) - r q := (re+step) - r
if (q) maskstoreF{*V~~r, maskOf{V, q}, s} if (q) homMaskStoreF{*V~~r, maskOf{V, q}, s}
} }
fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {

View File

@ -1,6 +1,6 @@
include './base' include './base'
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {

View File

@ -1,10 +1,12 @@
include './base' include './base'
include './f64' include './f64'
include './cbqnDefs' include './cbqnDefs'
if (hasarch{'X86_64'}) { if (hasarch{'AVX2'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {
include './neon' include './neon'
} }
@ -44,7 +46,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasar
def wp = unpackQ{w, T ~~ (T**0 > w)} def wp = unpackQ{w, T ~~ (T**0 > w)}
def xp = unpackQ{x, T ~~ (T**0 > x)} def xp = unpackQ{x, T ~~ (T**0 > x)}
def rp = each{__mul, wp, xp} def rp = each{__mul, wp, xp}
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp} def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
if (M{0}) { # masked check if (M{0}) { # masked check
tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}} tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}}
} else { # unmasked check; can do check in a simpler way } else { # unmasked check; can do check in a simpler way
@ -57,8 +59,8 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasa
tup{rl, tup{'anyne', rh, rl>>15}} tup{rl, tup{'anyne', rh, rl>>15}}
} }
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = { def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
max:= [8]f32 ~~ [8]u32**0x4efffffe max:= to_el{f32, (ty_u{T})**0x4efffffe}
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} def cf32{x:X} = emit{to_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
f32mul:= cf32{w} * cf32{x} f32mul:= cf32{w} * cf32{x}
tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}} tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}}
# TODO fallback to the below if the above fails # TODO fallback to the below if the above fails

View File

@ -2,7 +2,7 @@ include './base'
include './cbqnDefs' include './cbqnDefs'
include './f64' include './f64'
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {

View File

@ -1,5 +1,5 @@
include './base' include './base'
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
include './mask' include './mask'

View File

@ -9,7 +9,7 @@ mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u
mask256:*i64 = merge{4 ** -1, 4 ** 0} mask256:*i64 = merge{4 ** -1, 4 ** 0}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
# get mask of first n items; 0 ≤ n ≤ vcount{T} # get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T}
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256} def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128} def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64} def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64}
@ -31,6 +31,7 @@ def maskAfter{n} = {
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1) def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}} def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}}
def mask{X, mode=='to sign bits'} = maskOf{X,n} def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{X, mode=='to homogeneous bits'} = maskOf{X,n}
def mask{mode=='count'} = n def mask{mode=='count'} = n
def mask{x & istup{x} & tuplen{x}==1} = tup{mask{tupsel{0,x}}} def mask{x & istup{x} & tuplen{x}==1} = tup{mask{tupsel{0,x}}}
def mask{x==0} = 1 def mask{x==0} = 1
@ -47,7 +48,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
def TF = to_el{E0, T} def TF = to_el{E0, T}
xu:= narrow{E0, x} xu:= narrow{E0, x}
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, undefPromote{TF, xu}} if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}}
else storeLow{rpos, vcount{T}*width{E0}, xu} else storeLow{rpos, vcount{T}*width{E0}, xu}
} }

View File

@ -2,7 +2,7 @@ include './base'
include './f64' include './f64'
include './cbqnDefs' include './cbqnDefs'
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {

View File

@ -158,5 +158,4 @@ def homMask{x:T & nvecu{T} & T==[16]u8} = {
} }
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y} def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def maskstore{p:P, m:M, x:T & eltype{P}==T & nvec{T}} = store{p, 0, bitBlend{load{p}, x, m}} def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def maskstoreF{p:P, m:M, x:T & nvec{T}} = maskstore{p, m, x}

View File

@ -1,5 +1,5 @@
include './base' include './base'
include './sse3' include './sse'
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b} def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}

View File

@ -1,5 +1,5 @@
include './base' include './base'
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
include './mask' include './mask'
@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
e:= len/step e:= len/step
@for (xv, rv over e) rv = scan{xv,p} @for (xv, rv over e) rv = scan{xv,p}
q:= len & (step-1) q:= len & (step-1)
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}} if (q) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
} }
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
def last{v, p} = op{pre{v}, p} def last{v, p} = op{pre{v}, p}
@ -141,7 +141,7 @@ fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
if (jv+vl <= l) { if (jv+vl <= l) {
store{p, j, v} store{p, j, v}
} else { } else {
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v} if (jv < l) homMaskStoreF{rv+j, maskOf{V, l - jv}, v}
return{} return{}
} }
} }

View File

@ -1,6 +1,6 @@
include './base' include './base'
include './cbqnDefs' include './cbqnDefs'
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
include './mask' include './mask'
@ -68,7 +68,7 @@ def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
} }
def makeselx{VI, VD, nsel, xd, logv, cshuf} = { def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
def bblend {m}{ft} = topBlend{tupsel{0,ft}, tupsel{1,ft}, m} def bblend {m}{ft} = {def {f,t} = ft; homBlend{f, t, type{f} ~~ m}}
def bblendn{m}{tf} = bblend{m}{reverse{tf}} def bblendn{m}{tf} = bblend{m}{reverse{tf}}
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v}) def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})

View File

@ -1,6 +1,6 @@
include './base' include './base'
if (hasarch{'X86_64'}) { if (hasarch{'X86_64'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {

View File

@ -0,0 +1,55 @@
include './sse2'
### SSSE3 ###
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
### SSE4.1 ###
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
# arith
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b}
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
# conversion
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
# mask
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
### SSE4.2 ###
def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b}

View File

@ -28,6 +28,7 @@ def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', p
def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x} def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
# broadcast # broadcast
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}} def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}} def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
@ -36,6 +37,7 @@ def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T}
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v} def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v} def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
# make from elements
local def makeGen{T,s,x} = emit{T, s, ...each{{c}=>promote{eltype{T},c}, x}} local def makeGen{T,s,x} = emit{T, s, ...each{{c}=>promote{eltype{T},c}, x}}
def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs} def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs}
def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs} def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs}
@ -47,34 +49,47 @@ def make{T, x & w128{T} & istup{x}} = make{T, ...x}
def iota{T & w128{T}} = make{T, ...iota{vcount{T}}} def iota{T & w128{T}} = make{T, ...iota{vcount{T}}}
# integer comparison
def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b}
def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) }
# signed comparison
def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{a^s} }
def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{a^s} }
def __le{a:T,b:T & w128u{T}} = ~(a>b)
def __ge{a:T,b:T & w128u{T}} = ~(a<b)
def __le{a:T,b:T & w128u{T, 8}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T, 8}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T, 8}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b)
# rest of integer comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
# float comparison
def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b}
def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b}
def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b}
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
# bit arith # bit arith
def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}} def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
# signed comparison
def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __gt{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __eq{a:T,b:T & T==[16]u8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[ 8]u16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 4]u32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T}} = ~(a<=b)
# rest of comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
# shift # shift
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b} def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b}
@ -89,19 +104,7 @@ def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} # integer arith
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
# shuffle stuff
def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
# arith
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b} def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b} def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b}
def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b} def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b}
@ -112,38 +115,48 @@ def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b}
def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b} def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b}
def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
def __mul {a:T,b:T & [8]i16==T} = emit{T, '_mm_mullo_epi16', a, b}
def __mulhi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def __mulhi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
def __mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
def __mul{a:T,b:T & [4]i32==T} = {
def mu{x, y} = [4]i32 ~~ __mul32{[2]u64~~x, [2]u64~~y}
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
lo:= mu{a, b}
hi:= mu{...sw{4b2301, a, b}}
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
}
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
def max{a:T,b:T & w128i{T}} = homBlend{a, b, a<b}
def max{a:T,b:T & w128u{T,16}} = __subs{a,b}+b
def min{a:T,b:T & w128u{T,16}} = a-__subs{a,b}
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
# f32 arith # saturating
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b} def __adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def __adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b}
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b} def __subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def __subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b}
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b} def __adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def __adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b}
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b} def __subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def __subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b}
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b} # float arith
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a} def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
def rsqrt{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a} def rsqrt{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[2]u32 ** 0x7FFFFFFF}} def rcp{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
# f64 arith
def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
def rsqrt{a:[2]f64} = emit{[2]f64, '_mm_rsqrt_pd', a}
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}} def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
# mask stuff # mask stuff
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T & w128{T, 16}} = topMask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}} def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T & w128{T}} = topMask{x} def homMask{x:T & w128{T}} = topMask{x}
@ -156,56 +169,29 @@ def topAll{x:T & w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0} def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0} def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
# the lone SSE2 extract
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
### SSE4.1; TODO either move to own file or rename this file
# rest of extract
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
# float
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}
def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}
def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
# conversion
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X}
def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24}
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m))
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
def __eq{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
# SSE4.2
def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}

View File

@ -1,8 +1,8 @@
include './base' include './base'
include './cbqnDefs' include './cbqnDefs'
include './f64' include './f64'
if (hasarch{'X86_64'}) { if (hasarch{'AVX2'}) {
include './sse3' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {
@ -32,7 +32,7 @@ def unpack_to{f, l, x} = {
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
# Square kernel where width is a full vector # Square kernel where width is a full vector
def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x} def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x}
def load2{a:T, b:T} = pair{load{a}, load{b}} def load2{a:T, b:T} = pair{load{a}, load{b}}
def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = {
@ -164,7 +164,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : void
rp:*T = *T~~r0 rp:*T = *T~~r0
xp:*T = *T~~x0 xp:*T = *T~~x0
if (hasarch{'X86_64'} and w>=k and h>=k) { if (hasarch{'AVX2'} and w>=k and h>=k) {
transpose_with_kernel{T, k, kh, call_base, rp, xp, w, h, ws, hs} transpose_with_kernel{T, k, kh, call_base, rp, xp, w, h, ws, hs}
} else { } else {
if (h==2 and h==hs) @for (x0 in xp, x1 in xp+ws over i to w) { store{rp, i*2, x0}; store{rp, i*2+1, x1} } if (h==2 and h==hs) @for (x0 in xp, x1 in xp+ws over i to w) { store{rp, i*2, x0}; store{rp, i*2+1, x1} }

View File

@ -2,7 +2,7 @@
#include "mut.h" #include "mut.h"
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
#define SINGELI_FILE bits #define SINGELI_FILE bits
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#define bitselFns simd_bitsel #define bitselFns simd_bitsel
@ -58,7 +58,7 @@ NOINLINE B bit_sel(B b, B e0, B e1) {
sel:; sel:;
void* rp = m_tyarrlc(&r, width, b, type); void* rp = m_tyarrlc(&r, width, b, type);
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
bitselFns[width](rp, bp, e0i, e1i, ia); bitselFns[width](rp, bp, e0i, e1i, ia);
#else #else
switch(width) { switch(width) {