diff --git a/src/builtins/internal.c b/src/builtins/internal.c index ddf04c52..b78f5ceb 100644 --- a/src/builtins/internal.c +++ b/src/builtins/internal.c @@ -280,7 +280,20 @@ B eequal_c2(B t, B w, B x) { extern i32 fullCellFills; extern i32 cellFillErrored; #endif +#if TEST_RANGE + #include "../utils/calls.h" +#endif B internalTemp_c1(B t, B x) { + #if TEST_RANGE + i64 buf[2]; + bool b = getRange_fns[TI(x,elType)](tyany_ptr(x), buf, IA(x)); + f64* rp; + B r = m_f64arrv(&rp, 3); + rp[0] = buf[0]; + rp[1] = buf[1]; + rp[2] = b; + return r; + #endif #if TEST_CELL_FILLS if (isNum(x)) fullCellFills = o2iG(x); B r = m_i32(cellFillErrored); diff --git a/src/builtins/search.c b/src/builtins/search.c index f5d4a6f9..d3de718f 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -22,12 +22,28 @@ #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" +#include "../utils/calls.h" +RangeFn getRange_fns[el_f64+1]; #if SINGELI + extern RangeFn* const simd_getRangeRaw; #define SINGELI_FILE search #include "../utils/includeSingeli.h" +#else + #define GETRANGE(T,X) bool getRange_##T(void* x0, i64* res, u64 ia) { \ + assert(ia>0); T* x=x0; T min=*x,max=min; \ + for (ux i=1; imax)max=c; \ + } \ + res[0]=min; res[1]=max; return 1; \ + } + GETRANGE(i8,) + GETRANGE(i16,) + GETRANGE(i32,) + GETRANGE(f64, if (!q_fi64(c)) return 0) #endif + #define C2i(F, W, X) C2(F, m_i32(W), X) extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); @@ -447,7 +463,23 @@ B asNormalized(B x, usz n, bool nanBad) { return m_f64(0); } +bool getRangeBool(void* xp, i64* res, u64 ia) { + assert(ia>0); + u64 x0 = 1 & ((u64*)xp)[0]; + if (bit_has(xp, ia, !x0)) { res[0]=0; res[1]=1; } + else { res[0]=res[1]=x0; } + return true; +} void search_init(void) { { u64* p; Arr* a=m_bitarrp(&p, 1); arr_shAtm(a); *p= 0; gc_add(enclosed_0=taga(a)); } { u64* p; Arr* a=m_bitarrp(&p, 1); arr_shAtm(a); *p=~0ULL; gc_add(enclosed_1=taga(a)); } + getRange_fns[0] = getRangeBool; + #if SINGELI + for (i32 i=0; i<4; i++) getRange_fns[i+1] = simd_getRangeRaw[i]; + #else + getRange_fns[1] = getRange_i8; + getRange_fns[2] = getRange_i16; + getRange_fns[3] = getRange_i32; + getRange_fns[4] = getRange_f64; + #endif } diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index fb8b8d07..82a5fd82 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -174,6 +174,7 @@ def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}} def widen{...x} = assert{'widen not supported', show{...x}} def narrow{...x} = assert{'narrow not supported', show{...x}} def unord{...x} = assert{'unord not supported', show{...x}} +def floor{...x} = assert{'floor not supported', show{...x}} def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b oper &~ andnot infix none 35 diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index c2e90da3..510a6a35 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -11,6 +11,7 @@ if (hasarch{'AVX2'}) { include './neon' } include './mask' +include './vecfold' def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index @@ -73,7 +74,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { 0 } -if (hasarch{'X86_64'} or hasarch{'AARCH64'}) { +if (hasarch{'X86_64'} | hasarch{'AARCH64'}) { export{'simd_search_u8', searchOne{u64, u8}} export{'simd_search_u16', searchOne{u64, u16}} export{'simd_search_u32', searchOne{u64, u32}} @@ -294,3 +295,54 @@ export{'simd_mark_firsts_u8', simd_mark_firsts_u8} export{'simd_deduplicate_u8', simd_deduplicate_u8} export{'simd_member_u8', simd_member_u8} export{'simd_index_tab_u8', simd_index_tab_u8{usz}} + +def acc{unr, init:T} = { + a0v := init + def a0 = tup{a0v} + def a1 = @collect(unr) { reg:=init } + def op{S=='get'} = a0v + def op{S=='tr', F} = { a0v = tree_fold{F, a1} } + def op{S=='upd', is, F} = { + if (tuplen{is}==1) a0 = F{a0} + else a1 = F{a1} + } +} +def isI64{x:T & eltype{T}==f64 & hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}} +def isI64{x:T & eltype{T}==f64 & hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53)) + +def maskBlend{b:T, x:T, M} = x +def maskBlend{b:T, x:T, M & M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}} + +include './debug' +fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { + assert{n>0} + x:= *E~~x0 + min1:E = *x + max1:E = *x + if (has_simd and not (E==f64 and not (hasarch{'AARCH64'} or hasarch{'SSE4.1'}))) { + def bulk = arch_defvw/width{E} + def VT = [bulk]E + def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2} + def minA = acc{2, VT**min1} + def maxA = acc{2, VT**min1} + muLoop{bulk, unr, n, {is, M} => { + def cx = loadBatch{x, is, VT} + if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} + minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend + maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend + }, {} => { minA{'tr',min}; maxA{'tr',max} }} + min1 = vfold{min, minA{'get'}} + max1 = vfold{max, maxA{'get'}} + } else { + @for (x over i to n) { + if (E==f64 and rare{x != emit{f64, '', emit{i64, '', x}}}) return{0} + min1 = min{min1, x} + max1 = max{max1, x} + } + } + store{res, 0, min1} + store{res, 1, max1} + 1 +} + +exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} diff --git a/src/utils/calls.h b/src/utils/calls.h index 57e9604a..72064802 100644 --- a/src/utils/calls.h +++ b/src/utils/calls.h @@ -34,3 +34,6 @@ extern u8 eqFnData[]; typedef struct { EqFn fn; u8 data; } EqFnObj; #define EQFN_GET(W_ELT, X_ELT) ({ u8 eqfn_i_ = EQFN_INDEX(W_ELT, X_ELT); (EqFnObj){.fn=eqFns[eqfn_i_], .data=eqFnData[eqfn_i_]}; }) #define EQFN_CALL(FN, W, X, L) (FN).fn(W, X, L, (FN).data) + +typedef bool (*RangeFn)(void* xp, i64* res, u64 len); // writes min,max in res; returns 0 and leaves res undefined if either any (floor(x)≠x or abs>2⋆53), or (x≠(i64)x) +extern RangeFn getRange_fns[el_f64+1];