SSE2 squeeze

This commit is contained in:
dzaima 2023-04-02 14:27:47 +03:00
parent e0a265472c
commit 00a5ddcb89
5 changed files with 53 additions and 26 deletions

View File

@ -592,7 +592,7 @@ cachedBin‿linkerCache ← {
singeliMap 1¨ ({´ ((po.avx2/"2")po.arch) 𝕩}¨ ¨)/ singeliMap 1¨ ({´ ((po.avx2/"2")po.arch) 𝕩}¨ ¨)/
"xa.""src/builtins/arithm.c""monarith", "xa.""src/core/stuff.c""equal", "xa.""src/builtins/arithm.c""monarith", "xa.""src/core/stuff.c""equal",
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp", "xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp",
"2a.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy", "xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose", "xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose",
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan", "2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",

View File

@ -1,6 +1,6 @@
#include "../core.h" #include "../core.h"
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
#define SINGELI_FILE squeeze #define SINGELI_FILE squeeze
#include "../utils/includeSingeli.h" #include "../utils/includeSingeli.h"
#endif #endif
@ -36,14 +36,14 @@ B num_squeeze(B x) {
goto r_bit; goto r_bit;
} }
#if !SINGELI_SIMD #if !SINGELI_SIMD && !SINGELI_X86_64
usz i = 0; usz i = 0;
#endif #endif
u32 or = 0; // using bitwise or as an approximate ⌈´ u32 or = 0; // using bitwise or as an approximate ⌈´
switch (xe) { default: UD; switch (xe) { default: UD;
case el_bit: goto r_x; case el_bit: goto r_x;
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; } case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; }
case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; } case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; }
case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; } case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; }
@ -69,7 +69,7 @@ B num_squeeze(B x) {
B* xp = arr_bptr(x); B* xp = arr_bptr(x);
if (xp==NULL) goto r_f; if (xp==NULL) goto r_f;
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
or = avx2_squeeze_numB(xp, ia); or = avx2_squeeze_numB(xp, ia);
if (-2==(i32)or) goto r_x; if (-2==(i32)or) goto r_x;
if (-1==(i32)or) goto r_f64; if (-1==(i32)or) goto r_f64;
@ -117,7 +117,7 @@ B chr_squeeze(B x) {
i32 or = 0; i32 or = 0;
switch(xe) { default: UD; switch(xe) { default: UD;
case el_c8: goto r_x; case el_c8: goto r_x;
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; } case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; }
case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; } case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; }
#else #else
@ -142,7 +142,7 @@ B chr_squeeze(B x) {
B* xp = arr_bptr(x); B* xp = arr_bptr(x);
if (xp!=NULL) { if (xp!=NULL) {
#if SINGELI_SIMD #if SINGELI_SIMD || SINGELI_X86_64
u32 t = avx2_squeeze_chrB(xp, ia); u32 t = avx2_squeeze_chrB(xp, ia);
if (t==0) goto r_c8; if (t==0) goto r_c8;
else if (t==1) goto r_c16; else if (t==1) goto r_c16;

View File

@ -1,8 +1,10 @@
include './base' include './base'
if (hasarch{'X86_64'}) { if (hasarch{'AVX2'}) {
include './sse' include './sse'
include './avx' include './avx'
include './avx2' include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
} else if (hasarch{'AARCH64'}) { } else if (hasarch{'AARCH64'}) {
include './neon' include './neon'
} }
@ -24,26 +26,45 @@ def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = {
def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1}
def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start}
def isSNaN{x:T & isunsigned{eltype{T}}} = inRangeLen{x<<1, (0xFFE<<52)+2, (1<<52)-2} # SSE2 versions avoid any 64-bit integer comparsions
def q_chr{x:T & isvec{T} & eltype{T}==u64} = inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48} def anySNaN{M, x:T & eltype{T}==u64} = {
homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}}
}
def anySNaN{M, x:T & T==[2]u64 & hasarch{'X86_64'} & ~hasarch{'SSE4.2'}} = {
topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}}
}
def anyNonChar{M, x:T & isvec{T} & eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = {
def H = to_el{u32, T}
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
topAny{M{T~~ne}}
}
def fold{F, x:T} = { def fold{F, x:T} = {
show{'WARNING: using fallback fold'} show{'WARNING: using fallback fold for ', F, T}
def E = eltype{T} def E = eltype{T}
r:E = 0 r:E = 0
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
r r
} }
def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
c:= x c:= x
def EW = elwidth{T} def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} if (hasarch{'SSSE3'} and 0) {
if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
extract{c, 0} if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
extract{c, 0}
} else {
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} }
else extract{c, 0}
}
} }
def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
def makeOptBranch{enable, F} = { def makeOptBranch{enable, F} = {
if (enable) { if (enable) {
@ -67,7 +88,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
def bulk = vw / width{X} def bulk = vw / width{X}
def XV = [bulk]X def XV = [bulk]X
def E = tern{X==f64, u32, ty_u{X}} def E = tern{X==f64, u32, ty_u{X}}
def EV = [bulk]E def EV2 = [bulk*2]E
def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E}
# fold with either Max or Bitwise Or, truncating/zero-extending to TE # fold with either Max or Bitwise Or, truncating/zero-extending to TE
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}} def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
@ -89,7 +111,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
c:= EV~~loadBatch{xp, i, XV} c:= EV~~loadBatch{xp, i, XV}
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
if (B) { if (B) {
if (homAny{M{~q_chr{c}}}) return{3} if (anyNonChar{M, c}) return{3}
} else { } else {
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2} if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
} }
@ -111,7 +133,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
maskedLoop{bulk, iCont, len, {i, M} => { maskedLoop{bulk, iCont, len, {i, M} => {
def XU = [bulk]u64 def XU = [bulk]u64
v:= XU ~~ loadBatch{xp, i, XV} v:= XU ~~ loadBatch{xp, i, XV}
if (homAny{M{isSNaN{v}}}) return{0xffff_fffe} # not even a number if (anySNaN{M, v}) return{0xffff_fffe} # not even a number
}} }}
return{0xffff_ffff} # float return{0xffff_ffff} # float
}} }}
@ -126,7 +148,6 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}} r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
}} }}
} else { # f64 } else { # f64
def EV2 = v_dbl{EV}
r2:= EV2**0 r2:= EV2**0
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => { muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
def v0 = loadBatch{xp, is, XV} def v0 = loadBatch{xp, is, XV}
@ -139,13 +160,16 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
tup{intp, wdn} tup{intp, wdn}
} else { } else {
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0} def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it
def wdn = each{{v} => cvtWiden{XV, v}, ints} def wdn = each{{v} => cvtWiden{XV, v}, ints}
def intp = { if (vcount{type{tupsel{0,ints}}} == bulk) {
if (tuplen{ints}==1) tupsel{0, ints} def intp = {
else pair{ints} if (tuplen{ints}==1) tupsel{0, ints}
else pair{ints}
}
tup{intp, wdn}
} else if (tuplen{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX'}) {
tup{tupsel{0, ints}, wdn}
} }
tup{intp, wdn}
} }
} }
def int = tupsel{0, tmp} def int = tupsel{0, tmp}

View File

@ -82,6 +82,8 @@ def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b} def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b} def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b} def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
@ -90,6 +92,7 @@ def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}}
# shift # shift
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}

View File

@ -8,7 +8,7 @@ min ← 0∾(-2⋆7‿15‿31)∾¯0.5∾0‿0‿0
max 2( 271531) 0.5(2816)1114112 max 2( 271531) 0.5(2816)1114112
szs max-min szs max-min
chr 00000111 chr 00000111
specF64 - π2 ÷0 10 specF64 (- π2 1÷0) (0÷0) -0÷0
specI32 "Ai32" •internal.Variation (<(231)(-231))/ - 0/ (¯4+8) + 232 specI32 "Ai32" •internal.Variation (<(231)(-231))/ - 0/ (¯4+8) + 232
vars "Ab""Ai8""Ai16""Ai32""Af64""Ac8""Ac16""Ac32" vars "Ab""Ai8""Ai16""Ai32""Af64""Ac8""Ac16""Ac32"
casts 1816326481632¨"uiiifccc" casts 1816326481632¨"uiiifccc"