separate SSE2, implement SSE2 dyarith & bitsel
also separate maskstore to homogeneous & top bit mask variants, and make blend require equal element widths
This commit is contained in:
parent
8c76561883
commit
3a45eac1aa
@ -73,6 +73,7 @@ _assert_ ← { 𝔾𝕩?𝕩; Log 𝕨𝔽𝕩 ⋄ OnExit@ ⋄ •Exit 1}
|
||||
|
||||
SepArgs ← {' ' ((¬-˜⊢×·+`»⊸>)∘≠⊔⊢) 𝕩}
|
||||
Lowercase ← {𝕩 - (-´"Aa")×(𝕩≥'A')∧𝕩≤'Z'}
|
||||
Uppercase ← {𝕩 - (-´"aA")×(𝕩≥'a')∧𝕩≤'z'}
|
||||
|
||||
getOpt ← {
|
||||
args ← ×∘≠¨⊸/ •args
|
||||
@ -146,7 +147,7 @@ AtRoot ← rootDir⊸•file.At
|
||||
po ← { # parsed options
|
||||
⟨
|
||||
verbose, J, rebuildAll, rebuildSingeli, tui, output, clangd, versionName,
|
||||
os, arch, native, has,
|
||||
os, arch, native, has, avx2
|
||||
emcc, wasm, replxx, singeli,
|
||||
ffi, o3, debug, rtverify, heapverify,
|
||||
bytecodeDir, replxxDir, singeliDir,
|
||||
@ -200,15 +201,16 @@ po ← { # parsed options
|
||||
{𝕊: "Error: Unsupported 'has' options; options:"∾1↓∾", "⊸∾¨𝕩}_assert_(∧´has∊⊢) "avx2"‿"bmi2"‿"slow-pdep"
|
||||
{𝕊: "Error: Cannot have 'has' options on architecture '"∾arch∾"'; add an argument of "∾compat⊑"arch=x86-64"‿"target_arch=x86-64"}_assert_¬ (arch≢"x86-64") ∧ 0≠≠has
|
||||
|
||||
avx2 ⇐ (arch≡"x86-64") ∧ singeli ∧ native ∨ ⊑"avx2"<⊸∊has
|
||||
|
||||
{𝕊: "Target: "∾os∾" "∾arch∾ 1↓ ∾", "⊸∾¨ has∾native/⋈"native"} _verboseLog@
|
||||
|
||||
{𝕊:
|
||||
arch↩"generic"
|
||||
Log⍟((0=≠has) ∧ @≡GetOpt"native") ∾⟨
|
||||
"Note: on x86-64, a generic build doesn't include many optimizations; ",
|
||||
compat⊑"add an argument of 'native' or 'has=avx2' if possible; add 'native=0' to hide"‿"use 'make o3n-singeli' (targets the current CPU, assumes AVX2) or 'make o3-singeli has=avx2' if possible"
|
||||
⟩
|
||||
}⍟⊢ (arch≡"x86-64") ∧ singeli ∧ (¬native) ∧ ¬⊑"avx2"<⊸∊has
|
||||
}⍟⊢ (arch≡"x86-64") ∧ singeli ∧ ¬avx2
|
||||
|
||||
pie ⇐ {(¬shared)∧linux∧arch≡"x86-64"? 0; 1} DOpt "pie"
|
||||
|
||||
@ -262,9 +264,9 @@ po ← { # parsed options
|
||||
|
||||
args∾↩ GetOpt "f"
|
||||
args∾↩ GetOpt "CCFLAGS"
|
||||
args∾↩ (singeli∧arch≡ "x86-64") / ⟨"-DSINGELI_X86_64"⟩
|
||||
args∾↩ (singeli∧arch≡"aarch64") / ⟨"-DSINGELI_NEON"⟩
|
||||
args∾↩ (singeli∧⊑arch≢"generic") / ⟨"-DSINGELI_SIMD"⟩
|
||||
args∾↩ (singeli ∧ arch≡ "x86-64") / ⟨"-DSINGELI_X86_64"⟩ ∾ avx2 / ⋈"-DSINGELI_AVX2"
|
||||
args∾↩ (singeli ∧ arch≡"aarch64") / ⟨"-DSINGELI_NEON"⟩
|
||||
args∾↩ (singeli ∧ avx2∨arch≡"aarch64") / ⟨"-DSINGELI_SIMD"⟩
|
||||
args∾↩ ( wasm) / ⟨"-DWASM"⟩
|
||||
args∾↩ ( wasi) / ⟨"-DWASI", "-DNO_MMAP", "-DCATCH_ERRORS=0", "-D_WASI_EMULATED_MMAN", "--target=wasm32-wasi"⟩
|
||||
args∾↩ ( emcc) / ⟨"-DEMCC", "-O3"⟩
|
||||
@ -587,13 +589,16 @@ cachedBin‿linkerCache ← {
|
||||
⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩
|
||||
⟩
|
||||
cbqnSrc ↩ cbqnSrc clangd.Files "src"
|
||||
singeliMap ← 1↓¨ (∨´¨ (⊑po.arch) = ⊑¨)⊸/ ⟨
|
||||
"xa."‿"src/builtins/arithm.c"‿"monarith",
|
||||
"xa."‿"src/core/stuff.c"‿"equal", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits"
|
||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze"
|
||||
"x.."‿"src/builtins/select.c"‿"select", "x.."‿"src/builtins/fold.c"‿"fold", "x.."‿"src/builtins/scan.c"‿"scan"
|
||||
"x.."‿"src/builtins/scan.c"‿"neq", "x.."‿"src/builtins/slash.c"‿"slash", "x.."‿"src/builtins/slash.c"‿"constrep"
|
||||
"xag"‿"src/builtins/transpose.c"‿"transpose"
|
||||
singeliMap ← 1↓¨ ({∨´ ((po.avx2/"X")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨
|
||||
"Xa."‿"src/builtins/arithm.c"‿"monarith", "Xa."‿"src/core/stuff.c"‿"equal",
|
||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "Xa."‿"src/builtins/cmp.c"‿"cmp",
|
||||
"Xa."‿"src/builtins/squeeze.c"‿"squeeze", "Xa."‿"src/utils/mut.c"‿"copy",
|
||||
"xa."‿"src/utils/bits.c"‿"bits",
|
||||
"xag"‿"src/builtins/transpose.c"‿"transpose",
|
||||
"X.."‿"src/builtins/select.c"‿"select", "X.."‿"src/builtins/scan.c"‿"scan",
|
||||
"X.."‿"src/builtins/fold.c"‿"fold", "X.."‿"src/builtins/slash.c"‿"slash",
|
||||
"X.."‿"src/builtins/slash.c"‿"constrep",
|
||||
"X.."‿"src/builtins/scan.c"‿"neq",
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
@ -612,7 +617,7 @@ cachedBin‿linkerCache ← {
|
||||
singeliObjs ← @
|
||||
singeliCache ← {
|
||||
¬po.singeli? @;
|
||||
singeliCache ← GetCache ⟨"singeli", "Singeli generated code location", ⟨po.native, po.arch, po.singeliFlags, po.singeliDir⟩⟩
|
||||
singeliCache ← GetCache ⟨"singeli", "Singeli generated code location", ⟨po.native, po.arch, po.avx2, po.has, po.singeliFlags, po.singeliDir⟩⟩
|
||||
|
||||
# genArithTables
|
||||
ga ← "src/singeli/src/genArithTables.bqn"
|
||||
@ -627,7 +632,7 @@ cachedBin‿linkerCache ← {
|
||||
|
||||
singeliArgs ← po.singeliFlags∾⟨"-l", "gen="∾AtRoot singeliCache.folder⟩∾{
|
||||
po.native? ⟨⟩;
|
||||
"-a" ⋈ {"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch # TODO expand based on po.has (and then include in cache key)
|
||||
"-a" ⋈ ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ∾ ∾ ','⊸∾¨ Uppercase ∊⟜"avx2"‿"bmi2"⊸/po.has
|
||||
}
|
||||
{𝕊: "Singeli args: "∾•Repr singeliArgs} _verboseLog @
|
||||
singeliObjs ↩ {MakeSingeliInv ⟨singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩∾".singeli", (𝕩≡"dyarith")/⟨gaRule⟩⟩}¨ 1⊑¨singeliMap
|
||||
|
||||
@ -21,7 +21,7 @@ B atan2_c2(B, B, B);
|
||||
|
||||
typedef void (*AndBytesFn)(u8*, u8*, u64, u64);
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#if SINGELI_SIMD || SINGELI_X86_64
|
||||
#include "../singeli/c/arithdDispatch.c"
|
||||
static AndBytesFn andBytes_fn = simd_andBytes;
|
||||
#else
|
||||
@ -217,7 +217,7 @@ static B modint_AS(B w, B xv) { return modint_AA(w, C2(shape, C1(fne, incG(w))
|
||||
#undef GC2f
|
||||
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#if SINGELI_SIMD || SINGELI_X86_64
|
||||
#define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); }
|
||||
AA_DISPATCH(add) AA_DISPATCH(or)
|
||||
AA_DISPATCH(sub)
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
#include "../builtins.h"
|
||||
#include "../utils/mut.h"
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE fold
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
@ -92,7 +92,7 @@ B sum_c1(B t, B x) {
|
||||
}
|
||||
r += s;
|
||||
} else {
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
r = avx2_sum_f64(xv, ia);
|
||||
#else
|
||||
r=0; for (usz i=0; i<ia; i++) r+=((f64*)xv)[i];
|
||||
@ -133,7 +133,7 @@ static f64 (*const prod_fns[])(void*, usz, f64) = { prod_i8, prod_i16, prod_i32,
|
||||
static f64 min_##T(void* xv, usz ia) { MIN_MAX(T,<) } \
|
||||
static f64 max_##T(void* xv, usz ia) { MIN_MAX(T,>) }
|
||||
DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32)
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); }
|
||||
static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); }
|
||||
#else
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
rp[j] = xi; \
|
||||
}
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
@ -227,7 +227,7 @@ B SORT_C1(B t, B x) {
|
||||
#undef SORT_C1
|
||||
#undef INSERTION_SORT
|
||||
#undef COUNTING_SORT
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#undef WRITE_SPARSE_i8
|
||||
#undef WRITE_SPARSE_i16
|
||||
#endif
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
#define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); }
|
||||
#define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); }
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0);
|
||||
@ -29,7 +29,7 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v
|
||||
#define RADIX_SUM_4_u32 RDX_SUM_4(u32)
|
||||
#endif
|
||||
|
||||
#if SINGELI_X86_64 && !USZ_64
|
||||
#if SINGELI_AVX2 && !USZ_64
|
||||
#define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0);
|
||||
#define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0);
|
||||
#define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0);
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
static u64 vg_rand(u64 x) { return x; }
|
||||
#endif
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE scan
|
||||
#include "../utils/includeSingeli.h"
|
||||
#if __PCLMUL__
|
||||
@ -22,7 +22,7 @@ static u64 vg_rand(u64 x) { return x; }
|
||||
B scan_ne(B x, u64 p, u64 ia) { // consumes x
|
||||
u64* xp = bitarr_ptr(x);
|
||||
u64* rp; B r=m_bitarrv(&rp,ia);
|
||||
#if SINGELI_X86_64 && __PCLMUL__
|
||||
#if SINGELI_AVX2 && __PCLMUL__
|
||||
clmul_scan_ne(p, xp, rp, BIT_N(ia));
|
||||
#if USE_VALGRIND
|
||||
if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]);
|
||||
@ -78,7 +78,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
|
||||
} else {
|
||||
void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re));
|
||||
#define SUM_BITWISE(T) { T c=0; for (usz i=0; i<ia; i++) { c+= bitp_get(xp,i); ((T*)rp)[i]=c; } }
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SUM(W,T) avx2_bcs##W(xp, rp, ia);
|
||||
#else
|
||||
#define SUM(W,T) SUM_BITWISE(T)
|
||||
@ -96,7 +96,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x
|
||||
}
|
||||
|
||||
// min/max-scan
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I);
|
||||
#else
|
||||
#define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i<ia; i++) { if (xp[i] C c)c=xp[i]; rp[i]=c; }
|
||||
|
||||
@ -41,7 +41,7 @@
|
||||
#include "../utils/mut.h"
|
||||
#include "../builtins.h"
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE select
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
@ -175,7 +175,7 @@ B select_c2(B t, B w, B x) {
|
||||
u8 we = TI(w,elType);
|
||||
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define CPUSEL(W, NEXT) /*assumes 3≤xl≤6*/ \
|
||||
if (!avx2_select_tab[4*(we-el_i8)+xl-3](wp, xp, rp, wia, xn)) thrM("⊏: Indexing out-of-bounds");
|
||||
bool bool_use_simd = we==el_i8 && xl==0 && xia<=128;
|
||||
|
||||
@ -75,13 +75,13 @@
|
||||
|
||||
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
||||
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE slash
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE constrep
|
||||
#include "../utils/includeSingeli.h"
|
||||
|
||||
@ -154,7 +154,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
|
||||
static void where_block_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
assert(len <= bsp_max);
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len);
|
||||
#else
|
||||
if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0);
|
||||
@ -227,7 +227,7 @@ static B where(B x, usz xia, u64 s) {
|
||||
u64* xp = bitarr_ptr(x);
|
||||
usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1;
|
||||
if (xia <= 128) {
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8);
|
||||
bmipopc_1slash8(xp, rp, xia);
|
||||
FINISH_OVERALLOC_A(r, s, 8);
|
||||
@ -235,7 +235,7 @@ static B where(B x, usz xia, u64 s) {
|
||||
i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,);
|
||||
#endif
|
||||
} else if (xia <= 32768) {
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (s >= xia/8) {
|
||||
i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16);
|
||||
bmipopc_1slash16(xp, rp, xia);
|
||||
@ -257,7 +257,7 @@ static B where(B x, usz xia, u64 s) {
|
||||
}
|
||||
}
|
||||
} else if (xia <= (usz)I32_MAX+1) {
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
i32* rp; r = m_i32arrv(&rp, s);
|
||||
#else
|
||||
i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4);
|
||||
@ -272,7 +272,7 @@ static B where(B x, usz xia, u64 s) {
|
||||
} else {
|
||||
bs = bit_sum(xp,b);
|
||||
}
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (bs >= b/8+b/16) {
|
||||
bmipopc_1slash16(xp, buf, b);
|
||||
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
|
||||
@ -358,7 +358,7 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
u64* xp = bitarr_ptr(x);
|
||||
u64 sum = bit_sum(xp, xia);
|
||||
u64 l0 = up? xia-sum : sum; // Length of first set of indices
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (xia < 16) { BRANCHLESS_GRADE(i8) }
|
||||
else if (xia <= 1<<15) {
|
||||
B notx = bit_negate(inc(x));
|
||||
@ -478,7 +478,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
|
||||
else { DENSE; } \
|
||||
break; }
|
||||
#if SINGELI_X86_64 && FAST_PDEP
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
|
||||
#else
|
||||
@ -535,7 +535,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
#define SUM_CORE(T, WV, PREP, INC) \
|
||||
SCAN_CORE(WV, PREP; rp[ij]+=INC, , PLUS_SCAN(T))
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define IND_BY_SCAN \
|
||||
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, avx2_scan_max_i32(rp+k,rp+k,e-k))
|
||||
#else
|
||||
@ -829,7 +829,7 @@ B slash_c2(B t, B w, B x) {
|
||||
u8 xk = xl-3;
|
||||
void* rv = m_tyarrv(&r, 1<<xk, s, xt);
|
||||
void* xv = tyany_ptr(x);
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI_AVX2
|
||||
#define CASE(L,T) case L: constrep_##T(wv, xv, rv, xlen); break;
|
||||
#else
|
||||
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }
|
||||
|
||||
@ -70,6 +70,7 @@ def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b}
|
||||
def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b}
|
||||
def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a}
|
||||
def rsqrt{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
|
||||
def rcp{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
|
||||
def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}}
|
||||
def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a}
|
||||
def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a}
|
||||
@ -82,7 +83,6 @@ def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b}
|
||||
def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b}
|
||||
def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b}
|
||||
def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a}
|
||||
def rsqrt{a:[4]f64} = emit{[4]f64, '_mm256_rsqrt_pd', a}
|
||||
def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}}
|
||||
def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
|
||||
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
|
||||
@ -113,8 +113,8 @@ def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x
|
||||
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
|
||||
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
|
||||
|
||||
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}
|
||||
|
||||
|
||||
|
||||
@ -57,7 +57,6 @@ def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
||||
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
||||
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
|
||||
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
|
||||
# super questionable pack - assumes high halves are zero
|
||||
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
|
||||
def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}}
|
||||
@ -78,35 +77,19 @@ def absu{a:[32]i8 } = emit{[32]u8 , '_mm256_abs_epi8', a}
|
||||
def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a}
|
||||
def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a}
|
||||
|
||||
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}
|
||||
|
||||
def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
|
||||
def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
|
||||
def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
|
||||
def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
|
||||
|
||||
# saturating add/sub
|
||||
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}
|
||||
def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
|
||||
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}
|
||||
def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
|
||||
|
||||
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}
|
||||
def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
|
||||
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}
|
||||
def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
||||
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
|
||||
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
||||
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
|
||||
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
|
||||
|
||||
|
||||
# structural operations
|
||||
def maskstore{a:T, m:M, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32~~a, m, [8]i32~~v}
|
||||
def maskstore{a:T, m:M, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64~~a, m, [4]f64~~v}
|
||||
|
||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||
def maskstoreF{p, m, x:T} = store{p, 0, homBlend{load{p}, x, m}}
|
||||
def maskstoreF{p, m, x:T & elwidth{T}>=32} = maskstore{p,m,x}
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
@ -115,9 +98,9 @@ def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm2
|
||||
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
|
||||
def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
|
||||
|
||||
def topBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M & w256{T} & w256i{M,16}} = topBlend{f, t, [32]i8~~m}
|
||||
def topBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def homBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M & w256{T, 16} & w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
|
||||
|
||||
def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
|
||||
def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
|
||||
@ -129,6 +112,14 @@ def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '
|
||||
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
|
||||
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
|
||||
|
||||
# masked store; F variants may not be a single instruction
|
||||
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
|
||||
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
|
||||
def homMaskStore{p:P, m:M, v:T & w256i{M} & w256{T,elwidth{M}} & eltype{P}==T} = topMaskStore{p, m, v}
|
||||
|
||||
def topMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
|
||||
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
|
||||
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
# mask stuff
|
||||
def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
||||
|
||||
@ -129,15 +129,15 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
|
||||
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
|
||||
|
||||
# base cases
|
||||
def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def andnz{...x} = assert{'andnz not supported', show{...x}}
|
||||
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
|
||||
def topMask{...x} = assert{'topMask not supported', show{...x}}
|
||||
def packLo{...x} = assert{'packLo not supported', show{...x}}
|
||||
def packHi{...x} = assert{'packHi not supported', show{...x}}
|
||||
def mul12{...x} = assert{'mul12 not supported', show{...x}}
|
||||
def unpack{...x} = assert{'unpack not supported', show{...x}}
|
||||
def unpackLo{...x} = assert{'unpackLo not supported', show{...x}}
|
||||
def unpackHi{...x} = assert{'unpackHi not supported', show{...x}}
|
||||
def unpackQ{...x} = assert{'unpackQ not supported', show{...x}}
|
||||
def packQ{...x} = assert{'packQ not supported', show{...x}}
|
||||
def __mulhi{...x} = assert{'__mulhi not supported', show{...x}}
|
||||
@ -148,8 +148,9 @@ def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
|
||||
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
|
||||
def pair{...x} = assert{'pair not supported', show{...x}}
|
||||
def cvt{...x} = assert{'cvt not supported', show{...x}}
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
def shuf{...x} = assert{'shuf not supported', show{...x}}
|
||||
def shuf16Lo{...x} = assert{'shuf16Lo not supported', show{...x}}
|
||||
def shuf16Hi{...x} = assert{'shuf16Hi not supported', show{...x}}
|
||||
def homAll{...x} = assert{'homAll not supported', show{...x}}
|
||||
def homAny{...x} = assert{'homAny not supported', show{...x}}
|
||||
def homMask{...x} = assert{'homMask not supported', show{...x}}
|
||||
@ -159,7 +160,16 @@ def storeLow{...x} = assert{'storeLow not supported', show{...x}}
|
||||
def sel{...x} = assert{'sel not supported', show{...x}}
|
||||
def make{...x} = assert{'make not supported', show{...x}}
|
||||
def half{...x} = assert{'half not supported', show{...x}}
|
||||
def maskstoreF{...x} = assert{'maskstoreF not supported', show{...x}}
|
||||
def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}}
|
||||
def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}}
|
||||
def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}}
|
||||
def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}}
|
||||
|
||||
def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
|
||||
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
|
||||
@ -171,7 +181,7 @@ def broadcast{T, v & isprim{T}} = v
|
||||
def iota{n & knum{n}} = @collect(i to n) i
|
||||
def broadcast{n, v & knum{n}} = @collect(n) v
|
||||
|
||||
def arch_defvw = tern{hasarch{'X86_64'}, 256, 128}
|
||||
def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -63,10 +63,15 @@ def spreadBits{T==[32]u8, a:u32} = {
|
||||
e == (d&e)
|
||||
}
|
||||
|
||||
def spreadBits{T==[16]u8, a:u16} = {
|
||||
def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
|
||||
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
|
||||
andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
|
||||
}
|
||||
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
|
||||
b:= [16]u8~~[8]u16**a
|
||||
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100}
|
||||
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
|
||||
}
|
||||
|
||||
def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
|
||||
b:= make{T, 1<<iota{vcount{T}}}
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@ include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
include './base'
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
@ -78,7 +78,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
|
||||
}}
|
||||
setlabel{end}
|
||||
q := nr & (step-1)
|
||||
if (q) maskstoreF{rv+e, maskOf{V, q}, s}
|
||||
if (q) homMaskStoreF{rv+e, maskOf{V, q}, s}
|
||||
}
|
||||
}
|
||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
||||
@ -122,7 +122,7 @@ fn rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void =
|
||||
}
|
||||
setlabel{end}
|
||||
q := (re+step) - r
|
||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, s}
|
||||
if (q) homMaskStoreF{*V~~r, maskOf{V, q}, s}
|
||||
}
|
||||
|
||||
fn rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
include './base'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
@ -44,7 +46,7 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T} & hasar
|
||||
def wp = unpackQ{w, T ~~ (T**0 > w)}
|
||||
def xp = unpackQ{x, T ~~ (T**0 > x)}
|
||||
def rp = each{__mul, wp, xp}
|
||||
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp}
|
||||
def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp}
|
||||
if (M{0}) { # masked check
|
||||
tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}}
|
||||
} else { # unmasked check; can do check in a simpler way
|
||||
@ -57,8 +59,8 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasa
|
||||
tup{rl, tup{'anyne', rh, rl>>15}}
|
||||
}
|
||||
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
|
||||
max:= [8]f32 ~~ [8]u32**0x4efffffe
|
||||
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
|
||||
max:= to_el{f32, (ty_u{T})**0x4efffffe}
|
||||
def cf32{x:X} = emit{to_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}}
|
||||
# TODO fallback to the below if the above fails
|
||||
|
||||
@ -2,7 +2,7 @@ include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
include './base'
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
|
||||
@ -9,7 +9,7 @@ mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u
|
||||
mask256:*i64 = merge{4 ** -1, 4 ** 0}
|
||||
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
|
||||
|
||||
# get mask of first n items; 0 ≤ n ≤ vcount{T}
|
||||
# get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T}
|
||||
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
|
||||
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
|
||||
def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64}
|
||||
@ -31,6 +31,7 @@ def maskAfter{n} = {
|
||||
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
|
||||
def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}}
|
||||
def mask{X, mode=='to sign bits'} = maskOf{X,n}
|
||||
def mask{X, mode=='to homogeneous bits'} = maskOf{X,n}
|
||||
def mask{mode=='count'} = n
|
||||
def mask{x & istup{x} & tuplen{x}==1} = tup{mask{tupsel{0,x}}}
|
||||
def mask{x==0} = 1
|
||||
@ -47,7 +48,7 @@ def storeBatch{ptr:P, n, x:T, M} = {
|
||||
def TF = to_el{E0, T}
|
||||
xu:= narrow{E0, x}
|
||||
|
||||
if (M{0}) maskstoreF{*TF~~rpos, M{TF, 'to sign bits'}, undefPromote{TF, xu}}
|
||||
if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}}
|
||||
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
||||
}
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
|
||||
@ -158,5 +158,4 @@ def homMask{x:T & nvecu{T} & T==[16]u8} = {
|
||||
}
|
||||
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
|
||||
|
||||
def maskstore{p:P, m:M, x:T & eltype{P}==T & nvec{T}} = store{p, 0, bitBlend{load{p}, x, m}}
|
||||
def maskstoreF{p:P, m:M, x:T & nvec{T}} = maskstore{p, m, x}
|
||||
def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
include './base'
|
||||
include './sse3'
|
||||
include './sse'
|
||||
|
||||
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
|
||||
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
include './base'
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
@ -25,7 +25,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
|
||||
e:= len/step
|
||||
@for (xv, rv over e) rv = scan{xv,p}
|
||||
q:= len & (step-1)
|
||||
if (q) maskstoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
|
||||
if (q) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
|
||||
}
|
||||
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||
def last{v, p} = op{pre{v}, p}
|
||||
@ -141,7 +141,7 @@ fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
if (jv+vl <= l) {
|
||||
store{p, j, v}
|
||||
} else {
|
||||
if (jv < l) maskstoreF{rv+j, maskOf{V, l - jv}, v}
|
||||
if (jv < l) homMaskStoreF{rv+j, maskOf{V, l - jv}, v}
|
||||
return{}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
@ -68,7 +68,7 @@ def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
|
||||
}
|
||||
|
||||
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
|
||||
def bblend {m}{ft} = topBlend{tupsel{0,ft}, tupsel{1,ft}, m}
|
||||
def bblend {m}{ft} = {def {f,t} = ft; homBlend{f, t, type{f} ~~ m}}
|
||||
def bblendn{m}{tf} = bblend{m}{reverse{tf}}
|
||||
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
include './base'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
|
||||
55
src/singeli/src/sse.singeli
Normal file
55
src/singeli/src/sse.singeli
Normal file
@ -0,0 +1,55 @@
|
||||
include './sse2'
|
||||
|
||||
### SSSE3 ###
|
||||
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
|
||||
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
|
||||
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
|
||||
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
|
||||
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
|
||||
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
|
||||
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
|
||||
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
|
||||
|
||||
# arith
|
||||
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
|
||||
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
|
||||
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
|
||||
|
||||
def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b}
|
||||
|
||||
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
|
||||
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
|
||||
|
||||
# conversion
|
||||
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
|
||||
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
|
||||
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
|
||||
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
|
||||
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
|
||||
|
||||
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
|
||||
|
||||
# mask
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
# assumes all bits are the same in each mask item
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
|
||||
|
||||
|
||||
|
||||
### SSE4.2 ###
|
||||
def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b}
|
||||
@ -28,6 +28,7 @@ def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', p
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
|
||||
|
||||
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
|
||||
@ -36,6 +37,7 @@ def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T}
|
||||
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
|
||||
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
|
||||
|
||||
# make from elements
|
||||
local def makeGen{T,s,x} = emit{T, s, ...each{{c}=>promote{eltype{T},c}, x}}
|
||||
def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs}
|
||||
def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs}
|
||||
@ -47,34 +49,47 @@ def make{T, x & w128{T} & istup{x}} = make{T, ...x}
|
||||
|
||||
def iota{T & w128{T}} = make{T, ...iota{vcount{T}}}
|
||||
|
||||
|
||||
|
||||
# integer comparison
|
||||
def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) }
|
||||
# signed comparison
|
||||
def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b}
|
||||
def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b}
|
||||
def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b}
|
||||
def __lt{a:T,b:T & w128s{T}} = b>a
|
||||
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
|
||||
def __le{a:T,b:T & w128s{T}} = ~(a>b)
|
||||
# unsigned comparison
|
||||
def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{a^s} }
|
||||
def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{a^s} }
|
||||
def __le{a:T,b:T & w128u{T}} = ~(a>b)
|
||||
def __ge{a:T,b:T & w128u{T}} = ~(a<b)
|
||||
def __le{a:T,b:T & w128u{T, 8}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w128u{T, 8}} = a==max{a,b}
|
||||
def __lt{a:T,b:T & w128u{T, 8}} = ~(a>=b)
|
||||
def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b)
|
||||
# rest of integer comparison
|
||||
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
|
||||
# float comparison
|
||||
def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b}
|
||||
def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b}
|
||||
def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b}
|
||||
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
|
||||
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
|
||||
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
|
||||
|
||||
|
||||
|
||||
# bit arith
|
||||
def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
|
||||
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
|
||||
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
|
||||
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
|
||||
|
||||
# signed comparison
|
||||
def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
|
||||
def __gt{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpgt_epi8', a, b}
|
||||
def __gt{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpgt_epi16', a, b}
|
||||
def __gt{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpgt_epi32', a, b}
|
||||
def __lt{a:T,b:T & w128s{T}} = b>a
|
||||
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
|
||||
def __le{a:T,b:T & w128s{T}} = ~(a>b)
|
||||
# unsigned comparison
|
||||
def __eq{a:T,b:T & T==[16]u8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & T==[ 8]u16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & T==[ 4]u32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b}
|
||||
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
|
||||
def __lt{a:T,b:T & w128u{T}} = ~(a>=b)
|
||||
def __gt{a:T,b:T & w128u{T}} = ~(a<=b)
|
||||
# rest of comparison
|
||||
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
|
||||
|
||||
|
||||
# shift
|
||||
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
|
||||
def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b}
|
||||
@ -89,19 +104,7 @@ def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
|
||||
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
|
||||
|
||||
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
||||
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
||||
|
||||
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
|
||||
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
|
||||
|
||||
# shuffle stuff
|
||||
def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
|
||||
# arith
|
||||
# integer arith
|
||||
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
|
||||
def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b}
|
||||
def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b}
|
||||
@ -112,38 +115,48 @@ def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b}
|
||||
def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b}
|
||||
def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
|
||||
|
||||
def __mul {a:T,b:T & [8]i16==T} = emit{T, '_mm_mullo_epi16', a, b}
|
||||
def __mulhi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
|
||||
def __mulhi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
|
||||
def __mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
|
||||
def __mul{a:T,b:T & [4]i32==T} = {
|
||||
def mu{x, y} = [4]i32 ~~ __mul32{[2]u64~~x, [2]u64~~y}
|
||||
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
|
||||
lo:= mu{a, b}
|
||||
hi:= mu{...sw{4b2301, a, b}}
|
||||
[4]i32~~unpackLo{...sw{4b3120, lo, hi}}
|
||||
}
|
||||
|
||||
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
|
||||
def max{a:T,b:T & w128i{T}} = homBlend{a, b, a<b}
|
||||
def max{a:T,b:T & w128u{T,16}} = __subs{a,b}+b
|
||||
def min{a:T,b:T & w128u{T,16}} = a-__subs{a,b}
|
||||
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
||||
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
||||
|
||||
# f32 arith
|
||||
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}
|
||||
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}
|
||||
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}
|
||||
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}
|
||||
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}
|
||||
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}
|
||||
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}
|
||||
# saturating
|
||||
def __adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def __adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b}
|
||||
def __subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def __subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b}
|
||||
def __adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def __adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b}
|
||||
def __subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def __subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b}
|
||||
|
||||
# float arith
|
||||
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
|
||||
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
|
||||
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
|
||||
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
|
||||
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
|
||||
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
|
||||
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
|
||||
def rsqrt{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
|
||||
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[2]u32 ** 0x7FFFFFFF}}
|
||||
|
||||
# f64 arith
|
||||
def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
|
||||
def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
|
||||
def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
|
||||
def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
|
||||
def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
|
||||
def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
|
||||
def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
|
||||
def rsqrt{a:[2]f64} = emit{[2]f64, '_mm_rsqrt_pd', a}
|
||||
def rcp{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
|
||||
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
|
||||
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
|
||||
|
||||
|
||||
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
||||
def topMask{x:T & w128{T, 16}} = topMask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}
|
||||
def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
|
||||
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
||||
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
||||
def homMask{x:T & w128{T}} = topMask{x}
|
||||
@ -156,56 +169,29 @@ def topAll{x:T & w128i{T}} = topMask{x} == (1<<vcount{T})-1
|
||||
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
|
||||
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
|
||||
|
||||
# bits of other things SSE2 has
|
||||
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
|
||||
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
|
||||
|
||||
def unpack{a:[16]i8, b:[16]i8 } = { tup{emit{[ 8]i16, '_mm_unpacklo_epi8', a, b}, emit{[ 8]i16, '_mm_unpackhi_epi8', a, b}}}
|
||||
def unpack{a:[ 8]i16, b:[ 8]i16} = { tup{emit{[ 4]i32, '_mm_unpacklo_epi16', a, b}, emit{[ 4]i32, '_mm_unpackhi_epi16', a, b}}}
|
||||
def unpack{a:[ 4]i32, b:[ 4]i32} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi32', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi32', a, b}}}
|
||||
def unpack{a:[ 2]i64, b:[ 2]i64} = { tup{emit{[ 2]i64, '_mm_unpacklo_epi64', a, b}, emit{[ 2]i64, '_mm_unpackhi_epi64', a, b}}}
|
||||
def unpack{a:T, b:T & w128u{T}} = each{ty_u, unpack{ty_s{a}, ty_s{b}}}
|
||||
def unpackLo{a:T, b:T & w128i{T}} = tupsel{0, unpack{a, b}}
|
||||
def unpackQ{a:T, b:T & w128{T}} = unpack{a, b}
|
||||
|
||||
# the lone SSE2 extract
|
||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
|
||||
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
|
||||
|
||||
|
||||
|
||||
### SSE4.1; TODO either move to own file or rename this file
|
||||
|
||||
# rest of extract
|
||||
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
|
||||
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
|
||||
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
|
||||
|
||||
# float
|
||||
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}
|
||||
def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
|
||||
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}
|
||||
def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
|
||||
|
||||
# conversion
|
||||
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
|
||||
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
|
||||
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
|
||||
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
|
||||
def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
|
||||
def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
|
||||
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
|
||||
def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X}
|
||||
def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24}
|
||||
|
||||
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
|
||||
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
|
||||
# assumes all bits are the same in each mask item
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
|
||||
|
||||
def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b}
|
||||
def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b}
|
||||
def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b}
|
||||
|
||||
def __eq{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
|
||||
def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b}
|
||||
|
||||
|
||||
|
||||
# SSE4.2
|
||||
def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m))
|
||||
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
@ -1,8 +1,8 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
@ -32,7 +32,7 @@ def unpack_to{f, l, x} = {
|
||||
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
|
||||
|
||||
# Square kernel where width is a full vector
|
||||
def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x}
|
||||
def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x}
|
||||
|
||||
def load2{a:T, b:T} = pair{load{a}, load{b}}
|
||||
def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = {
|
||||
@ -164,7 +164,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : void
|
||||
|
||||
rp:*T = *T~~r0
|
||||
xp:*T = *T~~x0
|
||||
if (hasarch{'X86_64'} and w>=k and h>=k) {
|
||||
if (hasarch{'AVX2'} and w>=k and h>=k) {
|
||||
transpose_with_kernel{T, k, kh, call_base, rp, xp, w, h, ws, hs}
|
||||
} else {
|
||||
if (h==2 and h==hs) @for (x0 in xp, x1 in xp+ws over i to w) { store{rp, i*2, x0}; store{rp, i*2+1, x1} }
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#include "mut.h"
|
||||
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#if SINGELI_SIMD || SINGELI_X86_64
|
||||
#define SINGELI_FILE bits
|
||||
#include "../utils/includeSingeli.h"
|
||||
#define bitselFns simd_bitsel
|
||||
@ -58,7 +58,7 @@ NOINLINE B bit_sel(B b, B e0, B e1) {
|
||||
|
||||
sel:;
|
||||
void* rp = m_tyarrlc(&r, width, b, type);
|
||||
#if SINGELI_SIMD
|
||||
#if SINGELI_SIMD || SINGELI_X86_64
|
||||
bitselFns[width](rp, bp, e0i, e1i, ia);
|
||||
#else
|
||||
switch(width) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user