diff --git a/build/src/build.bqn b/build/src/build.bqn index ec32a0eb..556eaa79 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -592,7 +592,7 @@ cachedBin‿linkerCache ← { singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨ "xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal", "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", - "2a."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", + "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", diff --git a/src/builtins/squeeze.c b/src/builtins/squeeze.c index f8c63592..1e69b205 100644 --- a/src/builtins/squeeze.c +++ b/src/builtins/squeeze.c @@ -1,6 +1,6 @@ #include "../core.h" -#if SINGELI_SIMD +#if SINGELI_SIMD || SINGELI_X86_64 #define SINGELI_FILE squeeze #include "../utils/includeSingeli.h" #endif @@ -36,14 +36,14 @@ B num_squeeze(B x) { goto r_bit; } - #if !SINGELI_SIMD + #if !SINGELI_SIMD && !SINGELI_X86_64 usz i = 0; #endif u32 or = 0; // using bitwise or as an approximate ⌈´ switch (xe) { default: UD; case el_bit: goto r_x; - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; } case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; } case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; } @@ -69,7 +69,7 @@ B num_squeeze(B x) { B* xp = arr_bptr(x); if (xp==NULL) goto r_f; - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 or = avx2_squeeze_numB(xp, ia); if (-2==(i32)or) goto r_x; if (-1==(i32)or) goto r_f64; @@ -117,7 +117,7 @@ B chr_squeeze(B x) { i32 or = 0; switch(xe) { default: UD; case el_c8: goto r_x; - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; } case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; } #else @@ -142,7 +142,7 @@ B chr_squeeze(B x) { B* xp = arr_bptr(x); if (xp!=NULL) { - #if SINGELI_SIMD + #if SINGELI_SIMD || SINGELI_X86_64 u32 t = avx2_squeeze_chrB(xp, ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 67a327a4..a282dfd7 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -1,8 +1,10 @@ include './base' -if (hasarch{'X86_64'}) { +if (hasarch{'AVX2'}) { include './sse' include './avx' include './avx2' +} else if (hasarch{'X86_64'}) { + include './sse2' } else if (hasarch{'AARCH64'}) { include './neon' } @@ -24,26 +26,45 @@ def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = { def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} -def isSNaN{x:T & isunsigned{eltype{T}}} = inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2} -def q_chr{x:T & isvec{T} & eltype{T}==u64} = inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48} +# SSE2 versions avoid any 64-bit integer comparsions +def anySNaN{M, x:T & eltype{T}==u64} = { + homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}} +} +def anySNaN{M, x:T & T==[2]u64 & hasarch{'X86_64'} & ~hasarch{'SSE4.2'}} = { + topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}} +} +def anyNonChar{M, x:T & isvec{T} & eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} +def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = { + def H = to_el{u32, T} + def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} + topAny{M{T~~ne}} +} + + def fold{F, x:T} = { - show{'WARNING: using fallback fold'} + show{'WARNING: using fallback fold for ', F, T} def E = eltype{T} r:E = 0 each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} r } -def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { +def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = { c:= x def EW = elwidth{T} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} - if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} - if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} - extract{c, 0} + if (hasarch{'SSSE3'} and 0) { + if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} + if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} + extract{c, 0} + } else { + if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} + if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} } + else extract{c, 0} + } } -def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} +def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} def makeOptBranch{enable, F} = { if (enable) { @@ -67,7 +88,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { def bulk = vw / width{X} def XV = [bulk]X def E = tern{X==f64, u32, ty_u{X}} - def EV = [bulk]E + def EV2 = [bulk*2]E + def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E} # fold with either Max or Bitwise Or, truncating/zero-extending to TE def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}} @@ -89,7 +111,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { c:= EV~~loadBatch{xp, i, XV} if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end if (B) { - if (homAny{M{~q_chr{c}}}) return{3} + if (anyNonChar{M, c}) return{3} } else { if (anynePositive{EV**((1< { def XU = [bulk]u64 v:= XU ~~ loadBatch{xp, i, XV} - if (homAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number + if (anySNaN{M, v}) return{0xffff_fffe} # not even a number }} return{0xffff_ffff} # float }} @@ -126,7 +148,6 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}} }} } else { # f64 - def EV2 = v_dbl{EV} r2:= EV2**0 muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => { def v0 = loadBatch{xp, is, XV} @@ -139,13 +160,16 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { tup{intp, wdn} } else { def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0} - assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it def wdn = each{{v} => cvtWiden{XV, v}, ints} - def intp = { - if (tuplen{ints}==1) tupsel{0, ints} - else pair{ints} + if (vcount{type{tupsel{0,ints}}} == bulk) { + def intp = { + if (tuplen{ints}==1) tupsel{0, ints} + else pair{ints} + } + tup{intp, wdn} + } else if (tuplen{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX'}) { + tup{tupsel{0, ints}, wdn} } - tup{intp, wdn} } } def int = tupsel{0, tmp} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index b6205dac..d18b32ba 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -82,6 +82,8 @@ def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b} def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b} def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b} +def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} +def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} @@ -90,6 +92,7 @@ def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}} def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) +def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}} # shift def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} diff --git a/test/utils.bqn b/test/utils.bqn index 5aba7103..d3ab0f63 100644 --- a/test/utils.bqn +++ b/test/utils.bqn @@ -8,7 +8,7 @@ min ← 0∾(-2⋆7‿15‿31)∾¯0.5∾0‿0‿0 max ← 2∾( 2⋆7‿15‿31)∾ 0.5∾(2⋆8‿16)∾1114112 szs ← max-min chr ← 0‿0‿0‿0‿0‿1‿1‿1 -specF64 ⇐ ∾⟜- π‿2∾ ÷⟜0⊸∾ 1‿0 +specF64 ⇐ (∾⟜- π‿2∾ 1÷0) ∾ (0÷0) ∾ -0÷0 specI32 ⇐ "Ai32" •internal.Variation ∧ (<⟜(2⋆31)∧≥⟜(-2⋆31))⊸/ ∾⟜- ≥⟜0⊸/ ∧⥊(¯4+↕8) +⌜ 2⋆↕32 vars ⇐ "Ab"‿"Ai8"‿"Ai16"‿"Ai32"‿"Af64"‿"Ac8"‿"Ac16"‿"Ac32" casts ⇐ 1‿8‿16‿32‿64‿8‿16‿32⋈¨"uiiifccc"