diff --git a/build/src/build.bqn b/build/src/build.bqn index feb72a3b..ac2fc9b4 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -73,6 +73,7 @@ _assert_ ← { 𝔾𝕩?𝕩; Log 𝕨𝔽𝕩 ⋄ OnExit@ ⋄ •Exit 1} SepArgs ← {' ' ((¬-˜⊢×·+`»⊸>)∘≠⊔⊢) 𝕩} Lowercase ← {𝕩 - (-´"Aa")×(𝕩≥'A')∧𝕩≤'Z'} +Uppercase ← {𝕩 - (-´"aA")×(𝕩≥'a')∧𝕩≤'z'} getOpt ← { args ← ×∘≠¨⊸/ •args @@ -146,7 +147,7 @@ AtRoot ← rootDir⊸•file.At po ← { # parsed options ⟨ verbose, J, rebuildAll, rebuildSingeli, tui, output, clangd, versionName, - os, arch, native, has, + os, arch, native, has, avx2 emcc, wasm, replxx, singeli, ffi, o3, debug, rtverify, heapverify, bytecodeDir, replxxDir, singeliDir, @@ -200,15 +201,16 @@ po ← { # parsed options {𝕊: "Error: Unsupported 'has' options; options:"∾1↓∾", "⊸∾¨𝕩}_assert_(∧´has∊⊢) "avx2"‿"bmi2"‿"slow-pdep" {𝕊: "Error: Cannot have 'has' options on architecture '"∾arch∾"'; add an argument of "∾compat⊑"arch=x86-64"‿"target_arch=x86-64"}_assert_¬ (arch≢"x86-64") ∧ 0≠≠has + avx2 ⇐ (arch≡"x86-64") ∧ singeli ∧ native ∨ ⊑"avx2"<⊸∊has + {𝕊: "Target: "∾os∾" "∾arch∾ 1↓ ∾", "⊸∾¨ has∾native/⋈"native"} _verboseLog@ {𝕊: - arch↩"generic" Log⍟((0=≠has) ∧ @≡GetOpt"native") ∾⟨ "Note: on x86-64, a generic build doesn't include many optimizations; ", compat⊑"add an argument of 'native' or 'has=avx2' if possible; add 'native=0' to hide"‿"use 'make o3n-singeli' (targets the current CPU, assumes AVX2) or 'make o3-singeli has=avx2' if possible" ⟩ - }⍟⊢ (arch≡"x86-64") ∧ singeli ∧ (¬native) ∧ ¬⊑"avx2"<⊸∊has + }⍟⊢ (arch≡"x86-64") ∧ singeli ∧ ¬avx2 pie ⇐ {(¬shared)∧linux∧arch≡"x86-64"? 0; 1} DOpt "pie" @@ -262,9 +264,9 @@ po ← { # parsed options args∾↩ GetOpt "f" args∾↩ GetOpt "CCFLAGS" - args∾↩ (singeli∧arch≡ "x86-64") / ⟨"-DSINGELI_X86_64"⟩ - args∾↩ (singeli∧arch≡"aarch64") / ⟨"-DSINGELI_NEON"⟩ - args∾↩ (singeli∧⊑arch≢"generic") / ⟨"-DSINGELI_SIMD"⟩ + args∾↩ (singeli ∧ arch≡ "x86-64") / ⟨"-DSINGELI_X86_64"⟩ ∾ avx2 / ⋈"-DSINGELI_AVX2" + args∾↩ (singeli ∧ arch≡"aarch64") / ⟨"-DSINGELI_NEON"⟩ + args∾↩ (singeli ∧ avx2∨arch≡"aarch64") / ⟨"-DSINGELI_SIMD"⟩ args∾↩ ( wasm) / ⟨"-DWASM"⟩ args∾↩ ( wasi) / ⟨"-DWASI", "-DNO_MMAP", "-DCATCH_ERRORS=0", "-D_WASI_EMULATED_MMAN", "--target=wasm32-wasi"⟩ args∾↩ ( emcc) / ⟨"-DEMCC", "-O3"⟩ @@ -587,13 +589,16 @@ cachedBin‿linkerCache ← { ⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩ ⟩ cbqnSrc ↩ cbqnSrc clangd.Files "src" - singeliMap ← 1↓¨ (∨´¨ (⊑po.arch) = ⊑¨)⊸/ ⟨ - "xa."‿"src/builtins/arithm.c"‿"monarith", - "xa."‿"src/core/stuff.c"‿"equal", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits" - "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze" - "x.."‿"src/builtins/select.c"‿"select", "x.."‿"src/builtins/fold.c"‿"fold", "x.."‿"src/builtins/scan.c"‿"scan" - "x.."‿"src/builtins/scan.c"‿"neq", "x.."‿"src/builtins/slash.c"‿"slash", "x.."‿"src/builtins/slash.c"‿"constrep" - "xag"‿"src/builtins/transpose.c"‿"transpose" + singeliMap ← 1↓¨ ({∨´ ((po.avx2/"X")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨ + "Xa."‿"src/builtins/arithm.c"‿"monarith", "Xa."‿"src/core/stuff.c"‿"equal", + "xa."‿"src/builtins/arithd.c"‿"dyarith", "Xa."‿"src/builtins/cmp.c"‿"cmp", + "Xa."‿"src/builtins/squeeze.c"‿"squeeze", "Xa."‿"src/utils/mut.c"‿"copy", + "xa."‿"src/utils/bits.c"‿"bits", + "xag"‿"src/builtins/transpose.c"‿"transpose", + "X.."‿"src/builtins/select.c"‿"select", "X.."‿"src/builtins/scan.c"‿"scan", + "X.."‿"src/builtins/fold.c"‿"fold", "X.."‿"src/builtins/slash.c"‿"slash", + "X.."‿"src/builtins/slash.c"‿"constrep", + "X.."‿"src/builtins/scan.c"‿"neq", ⟩ objs ← ⟨⟩ @@ -612,7 +617,7 @@ cachedBin‿linkerCache ← { singeliObjs ← @ singeliCache ← { ¬po.singeli? @; - singeliCache ← GetCache ⟨"singeli", "Singeli generated code location", ⟨po.native, po.arch, po.singeliFlags, po.singeliDir⟩⟩ + singeliCache ← GetCache ⟨"singeli", "Singeli generated code location", ⟨po.native, po.arch, po.avx2, po.has, po.singeliFlags, po.singeliDir⟩⟩ # genArithTables ga ← "src/singeli/src/genArithTables.bqn" @@ -627,7 +632,7 @@ cachedBin‿linkerCache ← { singeliArgs ← po.singeliFlags∾⟨"-l", "gen="∾AtRoot singeliCache.folder⟩∾{ po.native? ⟨⟩; - "-a" ⋈ {"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch # TODO expand based on po.has (and then include in cache key) + "-a" ⋈ ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ∾ ∾ ','⊸∾¨ Uppercase ∊⟜"avx2"‿"bmi2"⊸/po.has } {𝕊: "Singeli args: "∾•Repr singeliArgs} _verboseLog @ singeliObjs ↩ {MakeSingeliInv ⟨singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩∾".singeli", (𝕩≡"dyarith")/⟨gaRule⟩⟩}¨ 1⊑¨singeliMap diff --git a/src/builtins/arithd.c b/src/builtins/arithd.c index 1e85f670..1722d28b 100644 --- a/src/builtins/arithd.c +++ b/src/builtins/arithd.c @@ -21,7 +21,7 @@ B atan2_c2(B, B, B); typedef void (*AndBytesFn)(u8*, u8*, u64, u64); -#if SINGELI_SIMD +#if SINGELI_SIMD || SINGELI_X86_64 #include "../singeli/c/arithdDispatch.c" static AndBytesFn andBytes_fn = simd_andBytes; #else @@ -217,7 +217,7 @@ static B modint_AS(B w, B xv) { return modint_AA(w, C2(shape, C1(fne, incG(w)) #undef GC2f - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 #define AA_DISPATCH(NAME) FORCE_INLINE B NAME##_AA(B t, B w, B x) { return dyArith_AA(&NAME##DyTableAA, w, x); } AA_DISPATCH(add) AA_DISPATCH(or) AA_DISPATCH(sub) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 75fe441a..8c1fe1a8 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -15,7 +15,7 @@ #include "../builtins.h" #include "../utils/mut.h" -#if SINGELI_X86_64 +#if SINGELI_AVX2 #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif @@ -92,7 +92,7 @@ B sum_c1(B t, B x) { } r += s; } else { - #if SINGELI_X86_64 + #if SINGELI_AVX2 r = avx2_sum_f64(xv, ia); #else r=0; for (usz i=0; i) } DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32) -#if SINGELI_X86_64 +#if SINGELI_AVX2 static f64 min_f64(void* xv, usz ia) { return avx2_fold_min_f64(xv,ia); } static f64 max_f64(void* xv, usz ia) { return avx2_fold_max_f64(xv,ia); } #else diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 9f8e8ceb..bf270303 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -61,7 +61,7 @@ rp[j] = xi; \ } -#if SINGELI_X86_64 +#if SINGELI_AVX2 extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2); extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2); extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2); @@ -227,7 +227,7 @@ B SORT_C1(B t, B x) { #undef SORT_C1 #undef INSERTION_SORT #undef COUNTING_SORT -#if SINGELI_X86_64 +#if SINGELI_AVX2 #undef WRITE_SPARSE_i8 #undef WRITE_SPARSE_i16 #endif diff --git a/src/builtins/radix.h b/src/builtins/radix.h index 2a8e8900..6b100447 100644 --- a/src/builtins/radix.h +++ b/src/builtins/radix.h @@ -11,7 +11,7 @@ #define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); } #define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); } -#if SINGELI_X86_64 +#if SINGELI_AVX2 extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); #define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0); @@ -29,7 +29,7 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v #define RADIX_SUM_4_u32 RDX_SUM_4(u32) #endif -#if SINGELI_X86_64 && !USZ_64 +#if SINGELI_AVX2 && !USZ_64 #define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0); #define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0); #define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0); diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 640cc75f..bca121af 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -9,7 +9,7 @@ static u64 vg_rand(u64 x) { return x; } #endif -#if SINGELI_X86_64 +#if SINGELI_AVX2 #define SINGELI_FILE scan #include "../utils/includeSingeli.h" #if __PCLMUL__ @@ -22,7 +22,7 @@ static u64 vg_rand(u64 x) { return x; } B scan_ne(B x, u64 p, u64 ia) { // consumes x u64* xp = bitarr_ptr(x); u64* rp; B r=m_bitarrv(&rp,ia); -#if SINGELI_X86_64 && __PCLMUL__ +#if SINGELI_AVX2 && __PCLMUL__ clmul_scan_ne(p, xp, rp, BIT_N(ia)); #if USE_VALGRIND if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]); @@ -78,7 +78,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x } else { void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); #define SUM_BITWISE(T) { T c=0; for (usz i=0; i= len/8) bmipopc_1slash16(src, (i16*)dst, len); #else if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); @@ -227,7 +227,7 @@ static B where(B x, usz xia, u64 s) { u64* xp = bitarr_ptr(x); usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/8) { i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); bmipopc_1slash16(xp, rp, xia); @@ -257,7 +257,7 @@ static B where(B x, usz xia, u64 s) { } } } else if (xia <= (usz)I32_MAX+1) { - #if SINGELI_X86_64 && FAST_PDEP + #if SINGELI_AVX2 && FAST_PDEP i32* rp; r = m_i32arrv(&rp, s); #else i32* rp = m_tyarrvO(&r, 4, s, t_i32arr, 4); @@ -272,7 +272,7 @@ static B where(B x, usz xia, u64 s) { } else { bs = bit_sum(xp,b); } - #if SINGELI_X86_64 && FAST_PDEP + #if SINGELI_AVX2 && FAST_PDEP if (bs >= b/8+b/16) { bmipopc_1slash16(xp, buf, b); for (usz j=0; j=32} = maskstore{p,m,x} def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} @@ -115,9 +98,9 @@ def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm2 def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m} -def topBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} -def homBlend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = topBlend{f, t, m} -def homBlend{f:T, t:T, m:M & w256{T} & w256i{M,16}} = topBlend{f, t, [32]i8~~m} +def topBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} +def homBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = topBlend{f, t, m} +def homBlend{f:T, t:T, m:M & w256{T, 16} & w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m} def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n} def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n} @@ -129,6 +112,14 @@ def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, ' def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} +# masked store; F variants may not be a single instruction +def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v} +def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v} +def homMaskStore{p:P, m:M, v:T & w256i{M} & w256{T,elwidth{M}} & eltype{P}==T} = topMaskStore{p, m, v} + +def topMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} # mask stuff def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index d10db695..f6cda783 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -129,15 +129,15 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1)) def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1 # base cases -def widen{T, x:X & T==X} = x -def narrow{T, x:X & T==eltype{X}} = x -def undefPromote{T, x:X & T==X} = T~~x def andnz{...x} = assert{'andnz not supported', show{...x}} def topBlend{...x} = assert{'topBlend not supported', show{...x}} def topMask{...x} = assert{'topMask not supported', show{...x}} def packLo{...x} = assert{'packLo not supported', show{...x}} def packHi{...x} = assert{'packHi not supported', show{...x}} def mul12{...x} = assert{'mul12 not supported', show{...x}} +def unpack{...x} = assert{'unpack not supported', show{...x}} +def unpackLo{...x} = assert{'unpackLo not supported', show{...x}} +def unpackHi{...x} = assert{'unpackHi not supported', show{...x}} def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} def packQ{...x} = assert{'packQ not supported', show{...x}} def __mulhi{...x} = assert{'__mulhi not supported', show{...x}} @@ -148,8 +148,9 @@ def fold_addw{...x} = assert{'fold_addw not supported', show{...x}} def narrowPair{...x} = assert{'narrowPair not supported', show{...x}} def pair{...x} = assert{'pair not supported', show{...x}} def cvt{...x} = assert{'cvt not supported', show{...x}} -def cvt{T, x:X & T==eltype{X}} = x def shuf{...x} = assert{'shuf not supported', show{...x}} +def shuf16Lo{...x} = assert{'shuf16Lo not supported', show{...x}} +def shuf16Hi{...x} = assert{'shuf16Hi not supported', show{...x}} def homAll{...x} = assert{'homAll not supported', show{...x}} def homAny{...x} = assert{'homAny not supported', show{...x}} def homMask{...x} = assert{'homMask not supported', show{...x}} @@ -159,7 +160,16 @@ def storeLow{...x} = assert{'storeLow not supported', show{...x}} def sel{...x} = assert{'sel not supported', show{...x}} def make{...x} = assert{'make not supported', show{...x}} def half{...x} = assert{'half not supported', show{...x}} -def maskstoreF{...x} = assert{'maskstoreF not supported', show{...x}} +def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}} +def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}} +def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}} +def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}} + +def widen{T, x:X & T==X} = x +def narrow{T, x:X & T==eltype{X}} = x +def undefPromote{T, x:X & T==X} = T~~x +def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}} +def cvt{T, x:X & T==eltype{X}} = x def min{a, b & knum{a} & knum{b}} = tern{ab, a, b} @@ -171,7 +181,7 @@ def broadcast{T, v & isprim{T}} = v def iota{n & knum{n}} = @collect(i to n) i def broadcast{n, v & knum{n}} = @collect(n) v -def arch_defvw = tern{hasarch{'X86_64'}, 256, 128} +def arch_defvw = tern{hasarch{'AVX2'}, 256, 128} diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index cfc02616..bc9d18d8 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -63,10 +63,15 @@ def spreadBits{T==[32]u8, a:u32} = { e == (d&e) } -def spreadBits{T==[16]u8, a:u16} = { +def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = { b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}} andnz{b, make{[16]u8, 1<<(iota{16}&7)}} } +def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = { + b:= [16]u8~~[8]u16**a + exp:= [16]u8~~shuf{[4]i32, shuf16Lo{unpackLo{b, b}, 4b1100}, 4b1100} + (exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0 +} def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { b:= make{T, 1< w)} def xp = unpackQ{x, T ~~ (T**0 > x)} def rp = each{__mul, wp, xp} - def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp} + def bad = each{{v} => ty_s{(v<<8)>>8 != v}, rp} if (M{0}) { # masked check tup{packQ{rp}, tup{'homAny', M{packQ{bad}}}} } else { # unmasked check; can do check in a simpler way @@ -57,8 +59,8 @@ def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T} & hasa tup{rl, tup{'anyne', rh, rl>>15}} } def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = { - max:= [8]f32 ~~ [8]u32**0x4efffffe - def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} + max:= to_el{f32, (ty_u{T})**0x4efffffe} + def cf32{x:X} = emit{to_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} f32mul:= cf32{w} * cf32{x} tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}} # TODO fallback to the below if the above fails diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 0df47a83..fddb0457 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -2,7 +2,7 @@ include './base' include './cbqnDefs' include './f64' if (hasarch{'X86_64'}) { - include './sse3' + include './sse' include './avx' include './avx2' } else if (hasarch{'AARCH64'}) { diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 587d766a..8de34d67 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,5 +1,5 @@ include './base' -include './sse3' +include './sse' include './avx' include './avx2' include './mask' diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 71cae6cb..d4c03b83 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -9,7 +9,7 @@ mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u mask256:*i64 = merge{4 ** -1, 4 ** 0} local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} -# get mask of first n items; 0 ≤ n ≤ vcount{T} +# get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T} def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256} def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128} def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64} @@ -31,6 +31,7 @@ def maskAfter{n} = { def mask{x:X & anyInt{x}} = x & ((1<promote{eltype{T},c}, x}} def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs} def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs} @@ -47,34 +49,47 @@ def make{T, x & w128{T} & istup{x}} = make{T, ...x} def iota{T & w128{T}} = make{T, ...iota{vcount{T}}} + + +# integer comparison +def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b} +def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b} +def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b} +def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) } +# signed comparison +def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b} +def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b} +def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b} +def __lt{a:T,b:T & w128s{T}} = b>a +def __ge{a:T,b:T & w128s{T}} = ~(b>a) +def __le{a:T,b:T & w128s{T}} = ~(a>b) +# unsigned comparison +def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{a^s} } +def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{a^s} } +def __le{a:T,b:T & w128u{T}} = ~(a>b) +def __ge{a:T,b:T & w128u{T}} = ~(a=b) +def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b) +# rest of integer comparison +def __ne{a:T,b:T & w128i{T}} = ~(b==a) +# float comparison +def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b} +def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b} +def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b} +def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b} +def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b} +def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b} + + + # bit arith def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}} def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) -# signed comparison -def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b} -def __gt{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpgt_epi8', a, b} -def __gt{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpgt_epi16', a, b} -def __gt{a:T,b:T & T==[ 4]i32} = emit{[ 4]u32, '_mm_cmpgt_epi32', a, b} -def __lt{a:T,b:T & w128s{T}} = b>a -def __ge{a:T,b:T & w128s{T}} = ~(b>a) -def __le{a:T,b:T & w128s{T}} = ~(a>b) -# unsigned comparison -def __eq{a:T,b:T & T==[16]u8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[ 8]u16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 4]u32} = emit{[ 4]u32, '_mm_cmpeq_epi32', a, b} -def __le{a:T,b:T & w128u{T}} = a==min{a,b} -def __ge{a:T,b:T & w128u{T}} = a==max{a,b} -def __lt{a:T,b:T & w128u{T}} = ~(a>=b) -def __gt{a:T,b:T & w128u{T}} = ~(a<=b) -# rest of comparison -def __ne{a:T,b:T & w128i{T}} = ~(b==a) - - # shift def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b} @@ -89,19 +104,7 @@ def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} -def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} -def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b} - -def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b} -def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b} -def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b} -def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} - -# shuffle stuff -def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} -def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} - -# arith +# integer arith def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b} def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b} def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b} @@ -112,38 +115,48 @@ def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b} def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b} def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} +def __mul {a:T,b:T & [8]i16==T} = emit{T, '_mm_mullo_epi16', a, b} +def __mulhi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} +def __mulhi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} +def __mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments +def __mul{a:T,b:T & [4]i32==T} = { + def mu{x, y} = [4]i32 ~~ __mul32{[2]u64~~x, [2]u64~~y} + def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs} + lo:= mu{a, b} + hi:= mu{...sw{4b2301, a, b}} + [4]i32~~unpackLo{...sw{4b3120, lo, hi}} +} +def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b} +def max{a:T,b:T & w128i{T}} = homBlend{a, b, a> elwidth{X} +def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24} -def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} -def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} - -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} - -# assumes all bits are the same in each mask item -def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m} -def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m} - -def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b} -def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b} -def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b} -def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b} - -def __eq{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b} -def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b} - - - -# SSE4.2 -def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b} +def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m)) +def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 72050c9e..cdfef3bf 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -1,8 +1,8 @@ include './base' include './cbqnDefs' include './f64' -if (hasarch{'X86_64'}) { - include './sse3' +if (hasarch{'AVX2'}) { + include './sse' include './avx' include './avx2' } else if (hasarch{'AARCH64'}) { @@ -32,7 +32,7 @@ def unpack_to{f, l, x} = { def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} # Square kernel where width is a full vector -def transpose_square{VT, l, x & hasarch{'X86_64'}} = unpack_to{1, l/2, x} +def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x} def load2{a:T, b:T} = pair{load{a}, load{b}} def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { @@ -164,7 +164,7 @@ fn transpose{T, k, kh}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : void rp:*T = *T~~r0 xp:*T = *T~~x0 - if (hasarch{'X86_64'} and w>=k and h>=k) { + if (hasarch{'AVX2'} and w>=k and h>=k) { transpose_with_kernel{T, k, kh, call_base, rp, xp, w, h, ws, hs} } else { if (h==2 and h==hs) @for (x0 in xp, x1 in xp+ws over i to w) { store{rp, i*2, x0}; store{rp, i*2+1, x1} } diff --git a/src/utils/bits.c b/src/utils/bits.c index f2e6eb5a..66a1aab3 100644 --- a/src/utils/bits.c +++ b/src/utils/bits.c @@ -2,7 +2,7 @@ #include "mut.h" -#if SINGELI_SIMD +#if SINGELI_SIMD || SINGELI_X86_64 #define SINGELI_FILE bits #include "../utils/includeSingeli.h" #define bitselFns simd_bitsel @@ -58,7 +58,7 @@ NOINLINE B bit_sel(B b, B e0, B e1) { sel:; void* rp = m_tyarrlc(&r, width, b, type); - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 bitselFns[width](rp, bp, e0i, e1i, ia); #else switch(width) {