SSE2 squeeze
This commit is contained in:
parent
e0a265472c
commit
00a5ddcb89
@ -592,7 +592,7 @@ cachedBin‿linkerCache ← {
|
|||||||
singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨
|
singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨
|
||||||
"xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal",
|
"xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal",
|
||||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
||||||
"2a."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy",
|
"xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy",
|
||||||
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
||||||
|
|
||||||
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
#include "../core.h"
|
#include "../core.h"
|
||||||
|
|
||||||
#if SINGELI_SIMD
|
#if SINGELI_SIMD || SINGELI_X86_64
|
||||||
#define SINGELI_FILE squeeze
|
#define SINGELI_FILE squeeze
|
||||||
#include "../utils/includeSingeli.h"
|
#include "../utils/includeSingeli.h"
|
||||||
#endif
|
#endif
|
||||||
@ -36,14 +36,14 @@ B num_squeeze(B x) {
|
|||||||
goto r_bit;
|
goto r_bit;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if !SINGELI_SIMD
|
#if !SINGELI_SIMD && !SINGELI_X86_64
|
||||||
usz i = 0;
|
usz i = 0;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
u32 or = 0; // using bitwise or as an approximate ⌈´
|
u32 or = 0; // using bitwise or as an approximate ⌈´
|
||||||
switch (xe) { default: UD;
|
switch (xe) { default: UD;
|
||||||
case el_bit: goto r_x;
|
case el_bit: goto r_x;
|
||||||
#if SINGELI_SIMD
|
#if SINGELI_SIMD || SINGELI_X86_64
|
||||||
case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; }
|
case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; }
|
||||||
case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; }
|
case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; }
|
||||||
case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; }
|
case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; }
|
||||||
@ -69,7 +69,7 @@ B num_squeeze(B x) {
|
|||||||
B* xp = arr_bptr(x);
|
B* xp = arr_bptr(x);
|
||||||
if (xp==NULL) goto r_f;
|
if (xp==NULL) goto r_f;
|
||||||
|
|
||||||
#if SINGELI_SIMD
|
#if SINGELI_SIMD || SINGELI_X86_64
|
||||||
or = avx2_squeeze_numB(xp, ia);
|
or = avx2_squeeze_numB(xp, ia);
|
||||||
if (-2==(i32)or) goto r_x;
|
if (-2==(i32)or) goto r_x;
|
||||||
if (-1==(i32)or) goto r_f64;
|
if (-1==(i32)or) goto r_f64;
|
||||||
@ -117,7 +117,7 @@ B chr_squeeze(B x) {
|
|||||||
i32 or = 0;
|
i32 or = 0;
|
||||||
switch(xe) { default: UD;
|
switch(xe) { default: UD;
|
||||||
case el_c8: goto r_x;
|
case el_c8: goto r_x;
|
||||||
#if SINGELI_SIMD
|
#if SINGELI_SIMD || SINGELI_X86_64
|
||||||
case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; }
|
case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; }
|
||||||
case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; }
|
case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; }
|
||||||
#else
|
#else
|
||||||
@ -142,7 +142,7 @@ B chr_squeeze(B x) {
|
|||||||
|
|
||||||
B* xp = arr_bptr(x);
|
B* xp = arr_bptr(x);
|
||||||
if (xp!=NULL) {
|
if (xp!=NULL) {
|
||||||
#if SINGELI_SIMD
|
#if SINGELI_SIMD || SINGELI_X86_64
|
||||||
u32 t = avx2_squeeze_chrB(xp, ia);
|
u32 t = avx2_squeeze_chrB(xp, ia);
|
||||||
if (t==0) goto r_c8;
|
if (t==0) goto r_c8;
|
||||||
else if (t==1) goto r_c16;
|
else if (t==1) goto r_c16;
|
||||||
|
|||||||
@ -1,8 +1,10 @@
|
|||||||
include './base'
|
include './base'
|
||||||
if (hasarch{'X86_64'}) {
|
if (hasarch{'AVX2'}) {
|
||||||
include './sse'
|
include './sse'
|
||||||
include './avx'
|
include './avx'
|
||||||
include './avx2'
|
include './avx2'
|
||||||
|
} else if (hasarch{'X86_64'}) {
|
||||||
|
include './sse2'
|
||||||
} else if (hasarch{'AARCH64'}) {
|
} else if (hasarch{'AARCH64'}) {
|
||||||
include './neon'
|
include './neon'
|
||||||
}
|
}
|
||||||
@ -24,26 +26,45 @@ def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = {
|
|||||||
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1}
|
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1}
|
||||||
def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start}
|
def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start}
|
||||||
|
|
||||||
def isSNaN{x:T & isunsigned{eltype{T}}} = inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2}
|
# SSE2 versions avoid any 64-bit integer comparsions
|
||||||
def q_chr{x:T & isvec{T} & eltype{T}==u64} = inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}
|
def anySNaN{M, x:T & eltype{T}==u64} = {
|
||||||
|
homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}}
|
||||||
|
}
|
||||||
|
def anySNaN{M, x:T & T==[2]u64 & hasarch{'X86_64'} & ~hasarch{'SSE4.2'}} = {
|
||||||
|
topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}}
|
||||||
|
}
|
||||||
|
def anyNonChar{M, x:T & isvec{T} & eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
|
||||||
|
def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = {
|
||||||
|
def H = to_el{u32, T}
|
||||||
|
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
|
||||||
|
topAny{M{T~~ne}}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fold{F, x:T} = {
|
def fold{F, x:T} = {
|
||||||
show{'WARNING: using fallback fold'}
|
show{'WARNING: using fallback fold for ', F, T}
|
||||||
def E = eltype{T}
|
def E = eltype{T}
|
||||||
r:E = 0
|
r:E = 0
|
||||||
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
|
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
|
||||||
r
|
r
|
||||||
}
|
}
|
||||||
def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = {
|
def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
|
||||||
c:= x
|
c:= x
|
||||||
def EW = elwidth{T}
|
def EW = elwidth{T}
|
||||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||||
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
||||||
|
if (hasarch{'SSSE3'} and 0) {
|
||||||
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
|
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
|
||||||
if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
|
if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
|
||||||
extract{c, 0}
|
extract{c, 0}
|
||||||
|
} else {
|
||||||
|
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
|
||||||
|
if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} }
|
||||||
|
else extract{c, 0}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
|
def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
|
||||||
|
|
||||||
def makeOptBranch{enable, F} = {
|
def makeOptBranch{enable, F} = {
|
||||||
if (enable) {
|
if (enable) {
|
||||||
@ -67,7 +88,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
|||||||
def bulk = vw / width{X}
|
def bulk = vw / width{X}
|
||||||
def XV = [bulk]X
|
def XV = [bulk]X
|
||||||
def E = tern{X==f64, u32, ty_u{X}}
|
def E = tern{X==f64, u32, ty_u{X}}
|
||||||
def EV = [bulk]E
|
def EV2 = [bulk*2]E
|
||||||
|
def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E}
|
||||||
|
|
||||||
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
|
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
|
||||||
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
|
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
|
||||||
@ -89,7 +111,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
|||||||
c:= EV~~loadBatch{xp, i, XV}
|
c:= EV~~loadBatch{xp, i, XV}
|
||||||
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
|
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
|
||||||
if (B) {
|
if (B) {
|
||||||
if (homAny{M{~q_chr{c}}}) return{3}
|
if (anyNonChar{M, c}) return{3}
|
||||||
} else {
|
} else {
|
||||||
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
|
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
|
||||||
}
|
}
|
||||||
@ -111,7 +133,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
|||||||
maskedLoop{bulk, iCont, len, {i, M} => {
|
maskedLoop{bulk, iCont, len, {i, M} => {
|
||||||
def XU = [bulk]u64
|
def XU = [bulk]u64
|
||||||
v:= XU ~~ loadBatch{xp, i, XV}
|
v:= XU ~~ loadBatch{xp, i, XV}
|
||||||
if (homAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number
|
if (anySNaN{M, v}) return{0xffff_fffe} # not even a number
|
||||||
}}
|
}}
|
||||||
return{0xffff_ffff} # float
|
return{0xffff_ffff} # float
|
||||||
}}
|
}}
|
||||||
@ -126,7 +148,6 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
|||||||
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
|
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
|
||||||
}}
|
}}
|
||||||
} else { # f64
|
} else { # f64
|
||||||
def EV2 = v_dbl{EV}
|
|
||||||
r2:= EV2**0
|
r2:= EV2**0
|
||||||
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
|
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
|
||||||
def v0 = loadBatch{xp, is, XV}
|
def v0 = loadBatch{xp, is, XV}
|
||||||
@ -139,13 +160,16 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
|||||||
tup{intp, wdn}
|
tup{intp, wdn}
|
||||||
} else {
|
} else {
|
||||||
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
|
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
|
||||||
assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it
|
|
||||||
def wdn = each{{v} => cvtWiden{XV, v}, ints}
|
def wdn = each{{v} => cvtWiden{XV, v}, ints}
|
||||||
|
if (vcount{type{tupsel{0,ints}}} == bulk) {
|
||||||
def intp = {
|
def intp = {
|
||||||
if (tuplen{ints}==1) tupsel{0, ints}
|
if (tuplen{ints}==1) tupsel{0, ints}
|
||||||
else pair{ints}
|
else pair{ints}
|
||||||
}
|
}
|
||||||
tup{intp, wdn}
|
tup{intp, wdn}
|
||||||
|
} else if (tuplen{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX'}) {
|
||||||
|
tup{tupsel{0, ints}, wdn}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
def int = tupsel{0, tmp}
|
def int = tupsel{0, tmp}
|
||||||
|
|||||||
@ -82,6 +82,8 @@ def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def
|
|||||||
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
|
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
|
||||||
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
|
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
|
||||||
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
|
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
|
||||||
|
def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
|
||||||
|
def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -90,6 +92,7 @@ def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
|
|||||||
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
|
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
|
||||||
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
|
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
|
||||||
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
|
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
|
||||||
|
def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}}
|
||||||
|
|
||||||
# shift
|
# shift
|
||||||
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
|
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
|
||||||
|
|||||||
@ -8,7 +8,7 @@ min ← 0∾(-2⋆7‿15‿31)∾¯0.5∾0‿0‿0
|
|||||||
max ← 2∾( 2⋆7‿15‿31)∾ 0.5∾(2⋆8‿16)∾1114112
|
max ← 2∾( 2⋆7‿15‿31)∾ 0.5∾(2⋆8‿16)∾1114112
|
||||||
szs ← max-min
|
szs ← max-min
|
||||||
chr ← 0‿0‿0‿0‿0‿1‿1‿1
|
chr ← 0‿0‿0‿0‿0‿1‿1‿1
|
||||||
specF64 ⇐ ∾⟜- π‿2∾ ÷⟜0⊸∾ 1‿0
|
specF64 ⇐ (∾⟜- π‿2∾ 1÷0) ∾ (0÷0) ∾ -0÷0
|
||||||
specI32 ⇐ "Ai32" •internal.Variation ∧ (<⟜(2⋆31)∧≥⟜(-2⋆31))⊸/ ∾⟜- ≥⟜0⊸/ ∧⥊(¯4+↕8) +⌜ 2⋆↕32
|
specI32 ⇐ "Ai32" •internal.Variation ∧ (<⟜(2⋆31)∧≥⟜(-2⋆31))⊸/ ∾⟜- ≥⟜0⊸/ ∧⥊(¯4+↕8) +⌜ 2⋆↕32
|
||||||
vars ⇐ "Ab"‿"Ai8"‿"Ai16"‿"Ai32"‿"Af64"‿"Ac8"‿"Ac16"‿"Ac32"
|
vars ⇐ "Ab"‿"Ai8"‿"Ai16"‿"Ai32"‿"Af64"‿"Ac8"‿"Ac16"‿"Ac32"
|
||||||
casts ⇐ 1‿8‿16‿32‿64‿8‿16‿32⋈¨"uiiifccc"
|
casts ⇐ 1‿8‿16‿32‿64‿8‿16‿32⋈¨"uiiifccc"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user