Merge pull request #96 from mlochbaum/search
Fast 32- and 64-bit dyadic search hash tables
This commit is contained in:
commit
e4cef4f46c
@ -648,11 +648,11 @@ cachedBin‿linkerCache ← {
|
|||||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
||||||
"xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy",
|
"xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy",
|
||||||
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
||||||
"xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold",
|
"xag"‿"src/builtins/search.c"‿"search", "xag"‿"src/builtins/selfsearch.c"‿"selfsearch"
|
||||||
|
"xag"‿"src/builtins/scan.c"‿"scan", "xa."‿"src/builtins/fold.c"‿"fold",
|
||||||
"xag"‿"src/builtins/sort.c"‿"bins"
|
"xag"‿"src/builtins/sort.c"‿"bins"
|
||||||
|
|
||||||
"2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan",
|
"2.."‿"src/builtins/select.c"‿"select", "2a."‿"src/builtins/slash.c"‿"constrep",
|
||||||
"2a."‿"src/builtins/slash.c"‿"constrep",
|
|
||||||
"xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
"xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
||||||
⟩
|
⟩
|
||||||
objs ← ⟨⟩
|
objs ← ⟨⟩
|
||||||
|
|||||||
@ -1,29 +1,49 @@
|
|||||||
// Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒)
|
// Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒)
|
||||||
|
|
||||||
// 𝕨⊐unit or unit∊𝕩: scalar loop with early-exit
|
// 𝕨⊐unit or unit∊𝕩: SIMD shortcutting search
|
||||||
// SHOULD use simd
|
// 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩
|
||||||
// SHOULD unify implementations
|
|
||||||
// 𝕩⊒unit or 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩
|
|
||||||
|
|
||||||
// Both arguments with rank≥1:
|
|
||||||
// High-rank inputs:
|
// High-rank inputs:
|
||||||
// Convert to a (lower-rank) typed integer array if cells are ≤62 bits
|
// Convert to a typed numeric list if cells are ≤62 bits
|
||||||
// COULD have special hashing for equal type >64 bit cells, skipping squeezing
|
// COULD have hashing for equal-type >64 bit cells, to skip squeezing
|
||||||
// COULD try conditionally squeezing ahead-of-time, and not squeezing in bqn_hash
|
// COULD try squeezing ahead-of-time to skip it in bqn_hash
|
||||||
// p⊐n & n∊p with short p & long n: n⊸=¨ p
|
// SHOULD have fast path when cell sizes don't match
|
||||||
// bitarr⊐𝕩: more special arithmetic
|
// One input empty: fast not-found
|
||||||
// SHOULD have impls for long p & short n
|
// Character elements:
|
||||||
|
// Character versus number array is fast not-found for ∊ and ⊐
|
||||||
|
// SHOULD have fast character-number path for ⊒
|
||||||
|
// Reinterpret as integer elements
|
||||||
|
// COULD try p=⌜n when all arguments are short (may not be faster?)
|
||||||
|
// p⊐n & n∊p with short n: p⊸⊐¨n
|
||||||
|
// p⊐n & n∊p with boolean p: based on ⊑p and p⊐¬⊑p
|
||||||
|
// p⊐n & n∊p with short p:
|
||||||
|
// AVX2 binary search when applicable
|
||||||
|
// SHOULD apply vector binary search to characters (sort as ints)
|
||||||
|
// n⊸=¨p otherwise
|
||||||
// ≤16-bit elements: lookup tables
|
// ≤16-bit elements: lookup tables
|
||||||
// Character elements: reinterpret as integer elements
|
// 8-bit ∊ and ⊐: SSSE3 table
|
||||||
|
// SHOULD make 8-bit NEON table
|
||||||
|
// SHOULD have branchy reverse 16-bit table search
|
||||||
|
// 32- or 64-bit elements: hash tables
|
||||||
|
// Store hash in table and not element; Robin Hood ordering; resizing
|
||||||
|
// Reverse hash if searched-for is shorter
|
||||||
|
// Shortcutting for reverse hashes and non-reversed ⊒
|
||||||
|
// SIMD lookup for 32-bit ∊ if chain length is small enough
|
||||||
|
// SHOULD partition if hash table size gets large
|
||||||
|
// SHOULD handle unequal search types better
|
||||||
// Otherwise, generic hashtable
|
// Otherwise, generic hashtable
|
||||||
// SHOULD handle up to 64 bit cells via proper typed hash tables
|
|
||||||
// SHOULD have fast path when cell sizes or element types doesn't match
|
|
||||||
|
|
||||||
#include "../core.h"
|
#include "../core.h"
|
||||||
#include "../utils/hash.h"
|
#include "../utils/hash.h"
|
||||||
#include "../utils/talloc.h"
|
#include "../utils/talloc.h"
|
||||||
#include "../utils/calls.h"
|
#include "../utils/calls.h"
|
||||||
|
|
||||||
|
extern NOINLINE void memset16(u16* p, u16 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
|
||||||
|
extern NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
|
||||||
|
extern NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
|
||||||
|
#define memset_i8 memset
|
||||||
|
#define memset_i16(P,V,L) memset16((u16*)(P), V, L)
|
||||||
|
#define memset_i32(P,V,L) memset32((u32*)(P), V, L)
|
||||||
|
|
||||||
RangeFn getRange_fns[el_f64+1];
|
RangeFn getRange_fns[el_f64+1];
|
||||||
#if SINGELI
|
#if SINGELI
|
||||||
extern RangeFn* const simd_getRangeRaw;
|
extern RangeFn* const simd_getRangeRaw;
|
||||||
@ -42,15 +62,31 @@ RangeFn getRange_fns[el_f64+1];
|
|||||||
GETRANGE(i32,)
|
GETRANGE(i32,)
|
||||||
GETRANGE(f64, if (!q_fi64(c)) return 0)
|
GETRANGE(f64, if (!q_fi64(c)) return 0)
|
||||||
#endif
|
#endif
|
||||||
|
#if SINGELI_AVX2
|
||||||
|
extern void (**const avx2_member_sort)(uint64_t*,void*,uint64_t,void*,uint64_t);
|
||||||
|
extern void (**const avx2_indexOf_sort)(int8_t*,void*,uint64_t,void*,uint64_t);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
#define C2i(F, W, X) C2(F, m_i32(W), X)
|
#define C2i(F, W, X) C2(F, m_i32(W), X)
|
||||||
|
extern B and_c1(B,B);
|
||||||
|
extern B gradeDown_c1(B,B);
|
||||||
|
extern B reverse_c1(B,B);
|
||||||
extern B eq_c2(B,B,B);
|
extern B eq_c2(B,B,B);
|
||||||
extern B ne_c2(B,B,B);
|
extern B ne_c2(B,B,B);
|
||||||
extern B or_c2(B,B,B);
|
extern B or_c2(B,B,B);
|
||||||
extern B add_c2(B,B,B);
|
extern B add_c2(B,B,B);
|
||||||
extern B sub_c2(B,B,B);
|
extern B sub_c2(B,B,B);
|
||||||
extern B mul_c2(B,B,B);
|
extern B mul_c2(B,B,B);
|
||||||
|
extern B join_c2(B,B,B);
|
||||||
|
extern B select_c2(B,B,B);
|
||||||
|
|
||||||
|
B asNormalized(B x, usz n, bool nanBad);
|
||||||
|
SHOULD_INLINE bool canCompare64_norm2(B* w, usz wia, B* x, usz xia) {
|
||||||
|
B wn=asNormalized(*w,wia,true); if (wn.u == m_f64(0).u) return 0; *w=wn;
|
||||||
|
B xn=asNormalized(*x,xia,true); if (xn.u == m_f64(0).u) return 0; *x=xn;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); }
|
static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); }
|
||||||
|
|
||||||
@ -66,7 +102,7 @@ static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); }
|
|||||||
if (IN.u != FOR.u) { \
|
if (IN.u != FOR.u) { \
|
||||||
if (FOR##e==el_i16 && n<ft/(64/sizeof(TY))) \
|
if (FOR##e==el_i16 && n<ft/(64/sizeof(TY))) \
|
||||||
{ for (usz i=0; i<n; i++) tab[((i16*)fp)[i]]=INIT; } \
|
{ for (usz i=0; i<n; i++) tab[((i16*)fp)[i]]=INIT; } \
|
||||||
else { TY* to=tab-(ft/2-(ft==2)); for (i64 i=0; i<ft; i++) to[i]=INIT; } \
|
else { memset_##TY(tab-(ft/2-(ft==2)), INIT, ft); } \
|
||||||
} \
|
} \
|
||||||
/* Set */ \
|
/* Set */ \
|
||||||
if (IN##e==el_i8) { for (usz i=m; i--; ) tab[((i8 *)ip)[i]]=SET; } \
|
if (IN##e==el_i8) { for (usz i=m; i--; ) tab[((i8 *)ip)[i]]=SET; } \
|
||||||
@ -191,7 +227,10 @@ static NOINLINE usz indexOfOne(B l, B e) {
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
#define CHR_TO_INT if (elChr(we) && elChr(xe)) { \
|
#define CHECK_CHRS_ELSE \
|
||||||
|
if (!elNum(we)) { \
|
||||||
|
if (elChr(we)) { \
|
||||||
|
if (elNum(xe)) goto none_found; \
|
||||||
if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \
|
if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \
|
||||||
if (we>xe) x=taga(cpyC16Arr(x)); \
|
if (we>xe) x=taga(cpyC16Arr(x)); \
|
||||||
else w=taga(cpyC16Arr(w)); \
|
else w=taga(cpyC16Arr(w)); \
|
||||||
@ -199,10 +238,15 @@ static NOINLINE usz indexOfOne(B l, B e) {
|
|||||||
goto tyEls; \
|
goto tyEls; \
|
||||||
} \
|
} \
|
||||||
we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \
|
we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \
|
||||||
}
|
} \
|
||||||
|
} else if (!elNum(xe)) { \
|
||||||
|
if (elChr(xe)) goto none_found; \
|
||||||
|
} else
|
||||||
|
|
||||||
B indexOf_c2(B t, B w, B x) {
|
B indexOf_c2(B t, B w, B x) {
|
||||||
|
bool split = 0; (void) split;
|
||||||
if (RARE(!isArr(w) || RNK(w)!=1)) {
|
if (RARE(!isArr(w) || RNK(w)!=1)) {
|
||||||
|
split = 1;
|
||||||
B2 t = splitCells(x, w, 1);
|
B2 t = splitCells(x, w, 1);
|
||||||
w = t.p;
|
w = t.p;
|
||||||
x = t.n;
|
x = t.n;
|
||||||
@ -218,10 +262,23 @@ B indexOf_c2(B t, B w, B x) {
|
|||||||
u8 we = TI(w,elType); usz wia = IA(w);
|
u8 we = TI(w,elType); usz wia = IA(w);
|
||||||
u8 xe = TI(x,elType); usz xia = IA(x);
|
u8 xe = TI(x,elType); usz xia = IA(x);
|
||||||
if (wia==0 || xia==0) {
|
if (wia==0 || xia==0) {
|
||||||
decG(w); return i64EachDec(0, x);
|
none_found:
|
||||||
|
decG(w); return i64EachDec(wia, x);
|
||||||
|
}
|
||||||
|
|
||||||
|
CHECK_CHRS_ELSE { tyEls: // Both numbers
|
||||||
|
if (wia>32 && xia<=(we<=el_i8?1:3)) {
|
||||||
|
SGetU(x);
|
||||||
|
B r;
|
||||||
|
#define IND(T) \
|
||||||
|
T* rp; r = m_##T##arrc(&rp, x); \
|
||||||
|
for (usz i=0; i<xia; i++) rp[i] = indexOfOne(w, GetU(x,i))
|
||||||
|
if (xia<I32_MAX) { IND(i32); r=reduceI32Width(r, wia); }
|
||||||
|
else { IND(f64); }
|
||||||
|
#undef IND
|
||||||
|
decG(w); decG(x); return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (elNum(we) && elNum(xe)) { tyEls:
|
|
||||||
if (we==el_bit) {
|
if (we==el_bit) {
|
||||||
u64* wp = bitarr_ptr(w);
|
u64* wp = bitarr_ptr(w);
|
||||||
u64 w0 = 1 & wp[0];
|
u64 w0 = 1 & wp[0];
|
||||||
@ -231,7 +288,18 @@ B indexOf_c2(B t, B w, B x) {
|
|||||||
return i==wia? r : C2(sub, r, C2i(mul, wia-i, C2i(eq, !w0, x)));
|
return i==wia? r : C2(sub, r, C2i(mul, wia-i, C2i(eq, !w0, x)));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (wia<=(we<=el_i16?4:16) && xia>16) {
|
#if SINGELI_AVX2
|
||||||
|
if (xia>=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<(xe==el_i8?64:16) && we<=xe && !elChr(TI(x,elType))) {
|
||||||
|
B g = C1(reverse, C1(gradeDown, incG(w)));
|
||||||
|
w = C2(select, incG(g), w);
|
||||||
|
switch (xe) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; }
|
||||||
|
i8* rp; B r = m_i8arrc(&rp, x);
|
||||||
|
avx2_indexOf_sort[xe-el_i8](rp, tyany_ptr(w), wia, tyany_ptr(x), xia);
|
||||||
|
r = C2(select, r, C2(join, g, m_i8(wia)));
|
||||||
|
decG(w); decG(x); return r;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
if (wia<=4 && xia>16) {
|
||||||
SGetU(w);
|
SGetU(w);
|
||||||
#define XEQ(I) C2(ne, GetU(w,I), incG(x))
|
#define XEQ(I) C2(ne, GetU(w,I), incG(x))
|
||||||
B r = XEQ(wia-1);
|
B r = XEQ(wia-1);
|
||||||
@ -241,23 +309,37 @@ B indexOf_c2(B t, B w, B x) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
|
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
|
||||||
|
B r;
|
||||||
#if SINGELI
|
#if SINGELI
|
||||||
if (wia>256 && we==el_i8 && xe==el_i8) {
|
if (wia>256 && we==el_i8 && xe==el_i8) {
|
||||||
TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256);
|
TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256);
|
||||||
void* fp = tyany_ptr(x);
|
void* fp = tyany_ptr(x);
|
||||||
simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind);
|
simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind);
|
||||||
decG(w);
|
decG(w);
|
||||||
i32* rp; B r = m_i32arrc(&rp, x);
|
#define LOOKUP(TY) \
|
||||||
|
TY* rp; r = m_##TY##arrc(&rp, x); \
|
||||||
for (usz i=0; i<xia; i++) rp[i]=ind[((u8*)fp)[i]];
|
for (usz i=0; i<xia; i++) rp[i]=ind[((u8*)fp)[i]];
|
||||||
|
if (wia<=INT16_MAX) { LOOKUP(i16) } else { LOOKUP(i32) }
|
||||||
|
#undef LOOKUP
|
||||||
TFREE(tab); decG(x);
|
TFREE(tab); decG(x);
|
||||||
return reduceI32Width(r, wia);
|
return r;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
if (wia<= INT8_MAX) { TABLE(w, x, i8, wia, i) }
|
||||||
|
else if (wia<=INT16_MAX) { TABLE(w, x, i16, wia, i) }
|
||||||
|
else { TABLE(w, x, i32, wia, i) }
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
#if SINGELI
|
||||||
|
if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) {
|
||||||
|
i32* rp; B r = m_i32arrc(&rp, x);
|
||||||
|
if (si_indexOf_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia)) {
|
||||||
|
decG(w); decG(x); return reduceI32Width(r, wia);
|
||||||
|
}
|
||||||
|
decG(r);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
B r;
|
|
||||||
TABLE(w, x, i32, wia, i)
|
|
||||||
return reduceI32Width(r, wia);
|
|
||||||
}
|
}
|
||||||
} else { CHR_TO_INT; }
|
|
||||||
|
|
||||||
i32* rp; B r = m_i32arrc(&rp, x);
|
i32* rp; B r = m_i32arrc(&rp, x);
|
||||||
H_b2i* map = m_b2i(64);
|
H_b2i* map = m_b2i(64);
|
||||||
@ -275,7 +357,9 @@ B indexOf_c2(B t, B w, B x) {
|
|||||||
|
|
||||||
B enclosed_0, enclosed_1;
|
B enclosed_0, enclosed_1;
|
||||||
B memberOf_c2(B t, B w, B x) {
|
B memberOf_c2(B t, B w, B x) {
|
||||||
|
bool split = 0; (void) split;
|
||||||
if (isAtm(x) || RNK(x)!=1) {
|
if (isAtm(x) || RNK(x)!=1) {
|
||||||
|
split = 1;
|
||||||
B2 t = splitCells(w, x, false);
|
B2 t = splitCells(w, x, false);
|
||||||
w = t.n;
|
w = t.n;
|
||||||
x = t.p;
|
x = t.p;
|
||||||
@ -302,10 +386,18 @@ B memberOf_c2(B t, B w, B x) {
|
|||||||
u8 we = TI(w,elType); usz wia = IA(w);
|
u8 we = TI(w,elType); usz wia = IA(w);
|
||||||
u8 xe = TI(x,elType); usz xia = IA(x);
|
u8 xe = TI(x,elType); usz xia = IA(x);
|
||||||
if (wia==0 || xia==0) {
|
if (wia==0 || xia==0) {
|
||||||
|
none_found:
|
||||||
decG(x); return i64EachDec(0, w);
|
decG(x); return i64EachDec(0, w);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (elNum(we) && elNum(xe)) { tyEls:
|
CHECK_CHRS_ELSE { tyEls: // Both numbers
|
||||||
|
if (xia>32 && wia<=(xe<=el_i8?1:xe==el_i32?4:6)) {
|
||||||
|
SGetU(w);
|
||||||
|
i8* rp; r = m_i8arrc(&rp, w);
|
||||||
|
for (usz i=0; i<wia; i++) rp[i] = indexOfOne(x, GetU(w,i)) < xia;
|
||||||
|
decG(w); decG(x); return taga(cpyBitArr(r));
|
||||||
|
}
|
||||||
|
|
||||||
#define WEQ(V) C2(eq, incG(w), V)
|
#define WEQ(V) C2(eq, incG(w), V)
|
||||||
if (xe==el_bit) {
|
if (xe==el_bit) {
|
||||||
u64* xp = bitarr_ptr(x);
|
u64* xp = bitarr_ptr(x);
|
||||||
@ -315,8 +407,16 @@ B memberOf_c2(B t, B w, B x) {
|
|||||||
decG(w); goto dec_x;
|
decG(w); goto dec_x;
|
||||||
}
|
}
|
||||||
|
|
||||||
u8 me = we>xe?we:xe;
|
#if SINGELI_AVX2
|
||||||
if (xia<=(me==el_i8?1:me==el_i16?4:16) && wia>16) {
|
if (wia>=32>>(we-el_i8) && xia>1 && ((we==el_i16 && xia<32) || (we==el_i32 && xia<16)) && xe<=we && !elChr(TI(x,elType))) {
|
||||||
|
x = C1(and, x); // sort
|
||||||
|
if (xe<we) switch (we) { default:UD; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; }
|
||||||
|
u64* rp; r = m_bitarrc(&rp, w);
|
||||||
|
avx2_member_sort[we-el_i16](rp, tyany_ptr(x), xia, tyany_ptr(w), wia);
|
||||||
|
decG(w); goto dec_x;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
if (xia<=(we==el_i8?1:we==el_i16?4:8) && wia>16) {
|
||||||
SGetU(x);
|
SGetU(x);
|
||||||
r = WEQ(GetU(x,0));
|
r = WEQ(GetU(x,0));
|
||||||
for (usz i=1; i<xia; i++) r = C2(or, r, WEQ(GetU(x,i)));
|
for (usz i=1; i<xia; i++) r = C2(or, r, WEQ(GetU(x,i)));
|
||||||
@ -337,7 +437,16 @@ B memberOf_c2(B t, B w, B x) {
|
|||||||
TABLE(x, w, i8, 0, 1)
|
TABLE(x, w, i8, 0, 1)
|
||||||
return taga(cpyBitArr(r));
|
return taga(cpyBitArr(r));
|
||||||
}
|
}
|
||||||
} else { CHR_TO_INT; }
|
#if SINGELI
|
||||||
|
if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) {
|
||||||
|
i8* rp; B r = m_i8arrc(&rp, w);
|
||||||
|
if (si_memberOf_c2_hash[we-el_i32](rp, tyany_ptr(x), xia, tyany_ptr(w), wia)) {
|
||||||
|
decG(w); decG(x); return taga(cpyBitArr(r));
|
||||||
|
}
|
||||||
|
decG(r);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
H_Sb* set = m_Sb(64);
|
H_Sb* set = m_Sb(64);
|
||||||
SGetU(x) SGetU(w)
|
SGetU(x) SGetU(w)
|
||||||
@ -353,10 +462,12 @@ B memberOf_c2(B t, B w, B x) {
|
|||||||
decG(x);
|
decG(x);
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
#undef CHR_TO_INT
|
#undef CHECK_CHRS_ELSE
|
||||||
|
|
||||||
B count_c2(B t, B w, B x) {
|
B count_c2(B t, B w, B x) {
|
||||||
|
bool split = 0; (void) split;
|
||||||
if (RARE(!isArr(w) || RNK(w)!=1)) {
|
if (RARE(!isArr(w) || RNK(w)!=1)) {
|
||||||
|
split = 1;
|
||||||
B2 t = splitCells(x, w, 2);
|
B2 t = splitCells(x, w, 2);
|
||||||
w = t.p;
|
w = t.p;
|
||||||
x = t.n;
|
x = t.n;
|
||||||
@ -396,6 +507,12 @@ B count_c2(B t, B w, B x) {
|
|||||||
we-= el_c8-el_i8; xe-= el_c8-el_i8;
|
we-= el_c8-el_i8; xe-= el_c8-el_i8;
|
||||||
goto el8or16;
|
goto el8or16;
|
||||||
} else {
|
} else {
|
||||||
|
#if SINGELI
|
||||||
|
if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia)))) &&
|
||||||
|
si_count_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia, (u32*)wnext)) {
|
||||||
|
goto dec_nwx;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
H_b2i* map = m_b2i(64);
|
H_b2i* map = m_b2i(64);
|
||||||
SGetU(x)
|
SGetU(x)
|
||||||
SGetU(w)
|
SGetU(w)
|
||||||
@ -412,6 +529,9 @@ B count_c2(B t, B w, B x) {
|
|||||||
}
|
}
|
||||||
free_b2i(map);
|
free_b2i(map);
|
||||||
}
|
}
|
||||||
|
#if SINGELI
|
||||||
|
dec_nwx:;
|
||||||
|
#endif
|
||||||
TFREE(wnext); decG(w); decG(x);
|
TFREE(wnext); decG(w); decG(x);
|
||||||
return reduceI32Width(r, wia);
|
return reduceI32Width(r, wia);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -43,25 +43,19 @@ extern B mul_c2(B, B, B);
|
|||||||
extern B scan_add_bool(B x, u64 ia);
|
extern B scan_add_bool(B x, u64 ia);
|
||||||
extern B scan_max_num(B x, u8 xe, u64 ia);
|
extern B scan_max_num(B x, u8 xe, u64 ia);
|
||||||
|
|
||||||
// These hashes are stored in tables and must be invertible!
|
// From search.c
|
||||||
#if defined(__SSE4_2__)
|
extern NOINLINE void memset32(u32* p, u32 v, usz l);
|
||||||
#include <x86intrin.h>
|
extern NOINLINE void memset64(u64* p, u64 v, usz l);
|
||||||
static inline u32 hash32(u32 x) { return _mm_crc32_u32(0x973afb51, x); }
|
|
||||||
|
#if SINGELI
|
||||||
|
#define SINGELI_FILE selfsearch
|
||||||
|
#include "../utils/includeSingeli.h"
|
||||||
|
|
||||||
|
#define TRY_HASHTAB_RET(NAME, W, RET) \
|
||||||
|
if (NAME##_c1_hash##W(rp, (u##W*)xv, n)) { decG(x); return RET; }
|
||||||
#else
|
#else
|
||||||
// Murmur3
|
#define TRY_HASHTAB_RET(NAME, W, RET) (void)0;
|
||||||
static inline u32 hash32(u32 x) {
|
|
||||||
x ^= x >> 16; x *= 0x85ebca6b;
|
|
||||||
x ^= x >> 13; x *= 0xc2b2ae35;
|
|
||||||
x ^= x >> 16;
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
#endif
|
#endif
|
||||||
static inline u64 hash64(u64 x) {
|
|
||||||
x ^= x >> 33; x *= 0xff51afd7ed558ccd;
|
|
||||||
x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53;
|
|
||||||
x ^= x >> 33;
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline bool use_sorted(B x, u8 logw) {
|
static inline bool use_sorted(B x, u8 logw) {
|
||||||
if (!FL_HAS(x, fl_asc|fl_dsc)) return 0;
|
if (!FL_HAS(x, fl_asc|fl_dsc)) return 0;
|
||||||
@ -146,73 +140,6 @@ u8 radix_offsets_2_usz(usz* c0, u32* v0, usz n) {
|
|||||||
} \
|
} \
|
||||||
decG(x); TFREE(alloc);
|
decG(x); TFREE(alloc);
|
||||||
|
|
||||||
static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
|
|
||||||
static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
|
|
||||||
|
|
||||||
// Resizing hash table, with fallback
|
|
||||||
#define SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, AUXSIZE, AUXINIT, AUXEXTEND, AUXMOVE) \
|
|
||||||
usz log = 64 - CLZ(n); \
|
|
||||||
usz msl = (64 - CLZ(n+n/2)) + 1; if (RAD && msl>20) msl=20; \
|
|
||||||
usz sh = W - (msl<14? msl : 12+(msl&1)); /* Shift to fit to table */ \
|
|
||||||
usz sz = 1 << (W - sh); /* Initial size */ \
|
|
||||||
usz msz = 1ull << msl; /* Max sz */ \
|
|
||||||
usz b = 64; /* Block size */ \
|
|
||||||
/* Resize or abort if more than 1/2^thresh collisions/element */ \
|
|
||||||
usz thresh = THRESH; \
|
|
||||||
/* Filling e slots past the end requires e*(e+1)/2 collisions, so */ \
|
|
||||||
/* n entries with <2 each can fill <sqrt(4*n) */ \
|
|
||||||
usz ext = n<=b? n : b + (1ull << (log/2 + 1)); \
|
|
||||||
TALLOC(u8, halloc, (msz+ext)*(sizeof(T)+AUXSIZE)); \
|
|
||||||
T* hash = (T*)halloc + msz-sz; \
|
|
||||||
T x0 = hash##W(xp[0]); rp[0] = RES0; \
|
|
||||||
memset##W(hash, x0, sz+ext); \
|
|
||||||
AUXINIT \
|
|
||||||
usz cc = 0; /* Collision counter */ \
|
|
||||||
usz i=1; while (1) { \
|
|
||||||
usz e = n-i>b? i+b : n; \
|
|
||||||
for (; i < e; i++) { \
|
|
||||||
T h = hash##W(xp[i]); T j0 = h>>sh, j = j0; T k; \
|
|
||||||
while (k=hash[j], k!=h & k!=x0) j++; \
|
|
||||||
cc += j-j0; \
|
|
||||||
RESWRITE \
|
|
||||||
} \
|
|
||||||
if (i == n) break; \
|
|
||||||
i64 dc = (i64)cc - ((THRESHMUL*i)>>thresh); \
|
|
||||||
if (dc >= 0) { \
|
|
||||||
if (sz == msz || (RAD && i < n/2 && sz >= 1<<18)) break; /*Abort*/ \
|
|
||||||
/* Avoid resizing if close to the end */ \
|
|
||||||
if (cc<STOP && (n-i)*dc < (i*((i64)n+i))>>(5+log-(W-sh))) continue;\
|
|
||||||
/* Resize hash, factor of 4 */ \
|
|
||||||
usz m = 2; \
|
|
||||||
usz dif = sz*((1<<m)-1); \
|
|
||||||
sh -= m; sz <<= m; \
|
|
||||||
hash -= dif; \
|
|
||||||
usz j = 0; \
|
|
||||||
cc = 0; \
|
|
||||||
memset##W(hash, x0, dif); \
|
|
||||||
AUXEXTEND \
|
|
||||||
for (j = dif; j < sz + ext; j++) { \
|
|
||||||
T h = hash[j]; if (h==x0) continue; hash[j] = x0; \
|
|
||||||
T k0 = h>>sh, k = k0; while (hash[k]!=x0) k++; \
|
|
||||||
cc += k-k0; \
|
|
||||||
hash[k] = h; AUXMOVE \
|
|
||||||
} \
|
|
||||||
if (cc >= STOP) break; \
|
|
||||||
thresh = THRESH; \
|
|
||||||
} \
|
|
||||||
} \
|
|
||||||
TFREE(halloc); \
|
|
||||||
if (i==n) { decG(x); return RESULT; }
|
|
||||||
#define SELFHASHTAB_VAL(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, INIT) \
|
|
||||||
SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, \
|
|
||||||
/*AUXSIZE*/sizeof(u32), \
|
|
||||||
/* AUXINIT */ \
|
|
||||||
u32* val = (u32*)(hash+sz+ext) + msz-sz; \
|
|
||||||
memset32(val, 0, sz+ext); \
|
|
||||||
INIT , \
|
|
||||||
/*AUXEXTEND*/val -= dif; memset32(val, 0, dif); , \
|
|
||||||
/*AUXMOVE*/u32 v = val[j]; val[j] = 0; val[k] = v;)
|
|
||||||
|
|
||||||
extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*);
|
extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*);
|
||||||
extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*);
|
extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*);
|
||||||
|
|
||||||
@ -268,14 +195,11 @@ B memberOf_c1(B t, B x) {
|
|||||||
} }
|
} }
|
||||||
if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } }
|
if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } }
|
||||||
#undef LOOKUP
|
#undef LOOKUP
|
||||||
#define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB( \
|
#define TRY_HASHTAB(W) TRY_HASHTAB_RET(memberOf, W, taga(cpyBitArr(r)))
|
||||||
T, W, RAD, STOP, \
|
|
||||||
1, taga(cpyBitArr(r)), hash[j]=h; rp[i]=k!=h;, \
|
|
||||||
1, THRESH, 0,,,)
|
|
||||||
if (lw==5) {
|
if (lw==5) {
|
||||||
if (n<12) { BRUTE(32); }
|
if (n<12) { BRUTE(32); }
|
||||||
i8* rp; B r = m_i8arrv(&rp, n);
|
i8* rp; B r = m_i8arrv(&rp, n);
|
||||||
HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<15)? 3 : 5)
|
TRY_HASHTAB(32)
|
||||||
|
|
||||||
// Radix-assisted lookup when hash table gives up
|
// Radix-assisted lookup when hash table gives up
|
||||||
usz rx = 256, tn = 1<<16; // Radix; table length
|
usz rx = 256, tn = 1<<16; // Radix; table length
|
||||||
@ -301,10 +225,10 @@ B memberOf_c1(B t, B x) {
|
|||||||
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
||||||
if (n<20) { BRUTE(64); }
|
if (n<20) { BRUTE(64); }
|
||||||
i8* rp; B r = m_i8arrv(&rp, n);
|
i8* rp; B r = m_i8arrv(&rp, n);
|
||||||
HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5)
|
TRY_HASHTAB(64)
|
||||||
decG(r); // Fall through
|
decG(r); // Fall through
|
||||||
}
|
}
|
||||||
#undef HASHTAB
|
#undef TRY_HASHTAB
|
||||||
#undef BRUTE
|
#undef BRUTE
|
||||||
|
|
||||||
if (RNK(x)>1) {
|
if (RNK(x)>1) {
|
||||||
@ -377,18 +301,11 @@ B count_c1(B t, B x) {
|
|||||||
if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } }
|
if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } }
|
||||||
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
|
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
|
||||||
#undef LOOKUP
|
#undef LOOKUP
|
||||||
#define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB_VAL( \
|
#define TRY_HASHTAB(W) TRY_HASHTAB_RET(count, W, num_squeeze(r))
|
||||||
T, W, RAD, STOP, \
|
|
||||||
/*RES0*/0, /*RETURN*/num_squeeze(r), \
|
|
||||||
/* RESWRITE */ \
|
|
||||||
bool e0=h==x0; rp[i]=val[j]+(ctr0&-(u32)e0); \
|
|
||||||
hash[j]=h; val[j]+=!e0; ctr0+=e0; , \
|
|
||||||
/*THRESHMUL*/1, THRESH, \
|
|
||||||
/*INIT*/u32 ctr0 = 1;)
|
|
||||||
if (lw==5) {
|
if (lw==5) {
|
||||||
if (n<20) { BRUTE(32); }
|
if (n<20) { BRUTE(32); }
|
||||||
i32* rp; B r = m_i32arrv(&rp, n);
|
i32* rp; B r = m_i32arrv(&rp, n);
|
||||||
HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<14)? 3 : 5)
|
TRY_HASHTAB(32)
|
||||||
// Radix-assisted lookup
|
// Radix-assisted lookup
|
||||||
usz rx = 256, tn = 1<<16; // Radix; table length
|
usz rx = 256, tn = 1<<16; // Radix; table length
|
||||||
u32* v0 = (u32*)xv;
|
u32* v0 = (u32*)xv;
|
||||||
@ -413,10 +330,10 @@ B count_c1(B t, B x) {
|
|||||||
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
||||||
if (n<20) { BRUTE(64); }
|
if (n<20) { BRUTE(64); }
|
||||||
i32* rp; B r = m_i32arrv(&rp, n);
|
i32* rp; B r = m_i32arrv(&rp, n);
|
||||||
HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5)
|
TRY_HASHTAB(64)
|
||||||
decG(r); // Fall through
|
decG(r); // Fall through
|
||||||
}
|
}
|
||||||
#undef HASHTAB
|
#undef TRY_HASHTAB
|
||||||
#undef BRUTE
|
#undef BRUTE
|
||||||
|
|
||||||
if (RNK(x)>1) {
|
if (RNK(x)>1) {
|
||||||
@ -486,17 +403,18 @@ B indexOf_c1(B t, B x) {
|
|||||||
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
|
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
|
||||||
#undef LOOKUP
|
#undef LOOKUP
|
||||||
|
|
||||||
#define HASHTAB(T, W, THRESH) SELFHASHTAB_VAL(T, W, 0, 2*n, \
|
#if SINGELI
|
||||||
/*RES0*/0, /*RETURN*/reduceI32WidthBelow(r, ctr), \
|
#define TRY_HASHTAB(W) \
|
||||||
/* RESWRITE */ \
|
u32 ctr = indexOf_c1_hash##W(rp, (u##W*)xv, n); \
|
||||||
if (k!=h) { val[j]=ctr++; hash[j]=h; } rp[i]=val[j]; , \
|
if (ctr>0) { decG(x); return reduceI32WidthBelow(r, ctr); }
|
||||||
/*THRESHMUL*/2, THRESH, \
|
#else
|
||||||
/*INIT*/u32 ctr = 1;)
|
#define TRY_HASHTAB(W) (void)0;
|
||||||
|
#endif
|
||||||
if (lw==5) {
|
if (lw==5) {
|
||||||
if (n<12) { BRUTE(32); }
|
if (n<12) { BRUTE(32); }
|
||||||
B r;
|
B r;
|
||||||
i32* rp; r = m_i32arrv(&rp, n);
|
i32* rp; r = m_i32arrv(&rp, n);
|
||||||
i32* xp = tyany_ptr(x);
|
i32* xp = xv;
|
||||||
i32 min=I32_MAX, max=I32_MIN;
|
i32 min=I32_MAX, max=I32_MIN;
|
||||||
for (usz i = 0; i < n; i++) {
|
for (usz i = 0; i < n; i++) {
|
||||||
i32 c = xp[i];
|
i32 c = xp[i];
|
||||||
@ -512,17 +430,16 @@ B indexOf_c1(B t, B x) {
|
|||||||
decG(x);
|
decG(x);
|
||||||
return reduceI32WidthBelow(r, u);
|
return reduceI32WidthBelow(r, u);
|
||||||
}
|
}
|
||||||
HASHTAB(u32, 32, sz==msz? 0 : sz>=(1<<18)? 1 : sz>=(1<<14)? 4 : 6)
|
TRY_HASHTAB(32)
|
||||||
decG(r); // Fall through
|
decG(r); // Fall through
|
||||||
}
|
}
|
||||||
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
if (lw==6 && canCompare64_norm(&x, &xv, n)) {
|
||||||
if (n<16) { BRUTE(64); }
|
if (n<16) { BRUTE(64); }
|
||||||
i32* rp; B r = m_i32arrv(&rp, n);
|
i32* rp; B r = m_i32arrv(&rp, n);
|
||||||
u64* xp = tyany_ptr(x);
|
TRY_HASHTAB(64)
|
||||||
HASHTAB(u64, 64, sz==msz? 0 : sz>=(1<<17)? 1 : sz>=(1<<13)? 4 : 6)
|
|
||||||
decG(r); // Fall through
|
decG(r); // Fall through
|
||||||
}
|
}
|
||||||
#undef HASHTAB
|
#undef TRY_HASHTAB
|
||||||
#undef BRUTE
|
#undef BRUTE
|
||||||
#undef DOTAB
|
#undef DOTAB
|
||||||
|
|
||||||
|
|||||||
@ -200,7 +200,9 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Binary search within vector registers
|
# Binary search within vector registers
|
||||||
def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
||||||
|
def up = prim != '⍒'
|
||||||
|
def search = (prim=='∊') | (prim=='⊐')
|
||||||
assert{wn > 1}; assert{wn < maxwn}
|
assert{wn > 1}; assert{wn < maxwn}
|
||||||
def wd = width{T}
|
def wd = width{T}
|
||||||
def I = if (wd<32) u8 else u32; def wi = width{I}
|
def I = if (wd<32) u8 else u32; def wi = width{I}
|
||||||
@ -214,7 +216,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
|||||||
log := ceil_log2{wn+1}
|
log := ceil_log2{wn+1}
|
||||||
gap := 1<<log - wn
|
gap := 1<<log - wn
|
||||||
# Fill with minimum value at the beginning
|
# Fill with minimum value at the beginning
|
||||||
def pre = (if (up) minvalue else maxvalue){T}
|
def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T}
|
||||||
wg := *V~~(w-gap)
|
wg := *V~~(w-gap)
|
||||||
wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}}
|
wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}}
|
||||||
# For multiple lanes, interleave like transpose
|
# For multiple lanes, interleave like transpose
|
||||||
@ -240,7 +242,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
|||||||
def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef'
|
def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef'
|
||||||
def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef'
|
def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef'
|
||||||
# Offset at end
|
# Offset at end
|
||||||
off := U~~V**cast_i{i8, gap-1}
|
off := U~~V**cast_i{i8, gap-(1-search)}
|
||||||
# Midpoint bits for each step
|
# Midpoint bits for each step
|
||||||
def lowbits = bb{copy{isub,isub}}
|
def lowbits = bb{copy{isub,isub}}
|
||||||
bits := each{{j} => U**(lowbits << j), iota{lstep}}
|
bits := each{{j} => U**(lowbits << j), iota{lstep}}
|
||||||
@ -252,31 +254,59 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
|
|||||||
if (this) @for_vec_overlap{vl} (j to xn) {
|
if (this) @for_vec_overlap{vl} (j to xn) {
|
||||||
xv:= load{*V~~(x+j), 0}
|
xv:= load{*V~~(x+j), 0}
|
||||||
s := U**bb{iota{isub}} # Select sequential bytes within each U
|
s := U**bb{iota{isub}} # Select sequential bytes within each U
|
||||||
def ltx{se,ind} = lt{xv, V~~se{re_el{I,ind}}}
|
def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}}
|
||||||
|
def ltx = cmpx{lt}; def eqx = cmpx{==}
|
||||||
@unroll (j to klog) {
|
@unroll (j to klog) {
|
||||||
m := s | tupsel{klog-1-j,bits}
|
m := s | tupsel{klog-1-j,bits}
|
||||||
s = homBlend{m, s, ltx{selw, m}}
|
s = homBlend{m, s, ltx{selw, m}}
|
||||||
}
|
}
|
||||||
r := if (isub==1) s else s>>(lb{isub}+wd-wi)
|
r := if (isub==1) s else s>>(lb{isub}+wd-wi)
|
||||||
|
# b records if xv was found; c is added to the index
|
||||||
|
def r_out = prim!='∊'
|
||||||
|
def get_up{var,cmpx,use}{set,...a} = if (use) set{var, cmpx{...a}}
|
||||||
|
b := undefined{U}; def up_b = get_up{b, eqx, search}
|
||||||
|
c := undefined{U}; def up_c = get_up{c, ltx, r_out}
|
||||||
|
up_b{=, selw, s}
|
||||||
# Extra selection lanes
|
# Extra selection lanes
|
||||||
if (last and ex>=1 and log>=klog+1) {
|
if (last and ex>=1 and log>=klog+1) {
|
||||||
r += r
|
r += r
|
||||||
c := ltx{selw1,s}
|
def up_bc{set,se} = { up_b{|=,se,s}; up_c{set,se,s} }
|
||||||
|
up_bc{=,selw1}
|
||||||
if (ex>=2 and log>=klog+2) {
|
if (ex>=2 and log>=klog+2) {
|
||||||
r += r
|
r += r
|
||||||
each{{se} => c += ltx{se,s}, selw2}
|
each{up_bc{+=, .}, selw2}
|
||||||
}
|
}
|
||||||
r += c
|
if (r_out) r += c
|
||||||
}
|
}
|
||||||
|
if (r_out) {
|
||||||
r -= off
|
r -= off
|
||||||
|
if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b}
|
||||||
rn := if (T==i8) r
|
rn := if (T==i8) r
|
||||||
else if (T==i16) half{narrow{u8, r}, 0}
|
else if (T==i16) half{narrow{u8, r}, 0}
|
||||||
else extract{re_el{i64, narrow{u8, r}}, 0}
|
else extract{re_el{i64, narrow{u8, r}}, 0}
|
||||||
rnp := *type{rn}~~(*i8~~rp+j)
|
rnp := *type{rn}~~(*i8~~rp+j)
|
||||||
if (isvec{type{rn}}) store{rnp, 0, rn}
|
if (isvec{type{rn}}) store{rnp, 0, rn}
|
||||||
else storeu{rnp, rn}
|
else storeu{rnp, rn}
|
||||||
|
} else {
|
||||||
|
def B = ty_u{vl}; out := cast_i{B, homMask{b}}
|
||||||
|
store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasarch{'AVX2'}) {
|
||||||
|
fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = {
|
||||||
|
bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn}
|
||||||
|
}
|
||||||
|
exportT{
|
||||||
|
'avx2_member_sort',
|
||||||
|
each{avx2_search_bin{'∊',.,.}, tup{i16,i32}, tup{32,16}}
|
||||||
|
}
|
||||||
|
exportT{
|
||||||
|
'avx2_indexOf_sort',
|
||||||
|
each{avx2_search_bin{'⊐',.,.}, tup{i8,i16,i32}, tup{64,16,16}}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def unroll_sizes = tup{4,1}
|
def unroll_sizes = tup{4,1}
|
||||||
@ -321,7 +351,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = {
|
|||||||
# For >=8 i8 values, vector bit-table is as good as binary search
|
# For >=8 i8 values, vector bit-table is as good as binary search
|
||||||
def wn_vec = if (T==i8) 8 else 2*256/width{T}
|
def wn_vec = if (T==i8) 8 else 2*256/width{T}
|
||||||
if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) {
|
if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) {
|
||||||
bin_search_vec{T, ...param, wn_vec}
|
bin_search_vec{if (up) '⍋' else '⍒', T, ...slice{param,1}, wn_vec}
|
||||||
# Lookup table threshold has to account for cost of
|
# Lookup table threshold has to account for cost of
|
||||||
# populating the table (proportional to wn until it's large), and
|
# populating the table (proportional to wn until it's large), and
|
||||||
# initializing the table (constant, much higher for i16)
|
# initializing the table (constant, much higher for i16)
|
||||||
|
|||||||
108
src/singeli/src/hashtab.singeli
Normal file
108
src/singeli/src/hashtab.singeli
Normal file
@ -0,0 +1,108 @@
|
|||||||
|
local include 'skin/cext'
|
||||||
|
local include './cbqnDefs' # talloc/tfree
|
||||||
|
|
||||||
|
# Search primitives
|
||||||
|
# Function params are passed as names to keep generated code readable
|
||||||
|
def names = tup{'memberOf', 'count', 'indexOf'}
|
||||||
|
def prims = tup{'∊', '⊒', '⊐' }
|
||||||
|
def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v}
|
||||||
|
def to_prim = map{names, prims, .}
|
||||||
|
|
||||||
|
# Defined in C
|
||||||
|
def memset{p:pT, v, l} = {
|
||||||
|
emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l}
|
||||||
|
}
|
||||||
|
|
||||||
|
# These hashes are stored in tables and must be invertible!
|
||||||
|
# Murmur3
|
||||||
|
def hash_val{x0:u32} = {
|
||||||
|
x := x0
|
||||||
|
x ^= x >> 16; x *= 0x85ebca6b
|
||||||
|
x ^= x >> 13; x *= 0xc2b2ae35
|
||||||
|
x ^= x >> 16; x
|
||||||
|
}
|
||||||
|
def hash_val{x0:u64} = {
|
||||||
|
x := x0
|
||||||
|
x ^= x >> 33; x *= 0xff51afd7ed558ccd
|
||||||
|
x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53
|
||||||
|
x ^= x >> 33; x
|
||||||
|
}
|
||||||
|
# CRC32
|
||||||
|
if (hasarch{'SSE4.2'}) require{'x86intrin.h'}
|
||||||
|
def hash_val{x:u32 & hasarch{'SSE4.2'}} = {
|
||||||
|
emit{u32, '_mm_crc32_u32', 0x973afb51, x}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Allocate and initialize resizing hash table
|
||||||
|
# Initial size sz+ext and maximum size msz+ext
|
||||||
|
# One region for each type in Ts initialized with value from v0s
|
||||||
|
# Allocates the maximum at the start, resizes downwards within the existing allocation
|
||||||
|
def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
|
||||||
|
def ws = each{width,Ts}
|
||||||
|
def wt = tupsel{0,ws}
|
||||||
|
each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment
|
||||||
|
# Variables updated on resize
|
||||||
|
sz := usz~~1 << logsz
|
||||||
|
sh := wt - logsz
|
||||||
|
|
||||||
|
def add{}=0; def add{a,...r} = a+add{...r}
|
||||||
|
halloc := talloc{u8, (msz+ext)*add{...ws/8}}
|
||||||
|
szo := msz-sz # Beginning of allocation to initial table
|
||||||
|
sze := sz+ext # Initial table to end of allocation
|
||||||
|
def pe{{}} = halloc; def pl{{}} = tup{}
|
||||||
|
def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far
|
||||||
|
def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} }
|
||||||
|
ptrs := pl{Ts}
|
||||||
|
def memset{_, ('any'), _} = {} # Indicates initialization not needed
|
||||||
|
each{memset{., ., sze}, ptrs, v0s}
|
||||||
|
|
||||||
|
def hash_resize{cc, m} = {
|
||||||
|
dif := sz*((1<<m)-1) # Number of elements of space to add
|
||||||
|
sh -= m; sz <<= m
|
||||||
|
set_thresh{}
|
||||||
|
cc = 0 # Collision counter
|
||||||
|
k:tupsel{0,Ts} = 0; --k # Index where to move an element to
|
||||||
|
each{{p,v} => { p -= dif; memset{p, v, dif} }, ptrs, v0s}
|
||||||
|
def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s
|
||||||
|
s := sz+ext
|
||||||
|
i := dif
|
||||||
|
|
||||||
|
# Iterate over existing elements in blocks of e<=32
|
||||||
|
# Branchlessly skips over h0
|
||||||
|
while (i < s) {
|
||||||
|
e := min{s-i, usz~~32}
|
||||||
|
b:u32 = 0
|
||||||
|
@for_backwards (h in hash+i over e) b = 2*b + promote{u32, h!=h0}
|
||||||
|
while (b!=0) {
|
||||||
|
j := i + cast_i{usz,ctz{b}}; b &= b-1
|
||||||
|
h := hash->j
|
||||||
|
hash <-{j} h0
|
||||||
|
k0 := h>>sh
|
||||||
|
if (ordered) { k = max{k0, k+1} } # k0 can't be less than k
|
||||||
|
else { k = k0; while (hash->k!=h0) ++k }
|
||||||
|
cc += cast_i{ux, k-k0}
|
||||||
|
hash <-{k} h
|
||||||
|
def move{p,v0} = {
|
||||||
|
v := p->j; if (not same{v0,'any'}) p <-{j} v0; p <-{k} v
|
||||||
|
}
|
||||||
|
each{move, vals, v0r}
|
||||||
|
}
|
||||||
|
i += e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Test for resize if more than 1/2^t collisions/element
|
||||||
|
t:usz = 0 # Shift amount
|
||||||
|
def div_thresh{i} = i>>t
|
||||||
|
# Threshold setter, re-applied on resize
|
||||||
|
def set_thresh{} = {
|
||||||
|
if (sz==msz) t = 0
|
||||||
|
else if ((not has_radix) and sz>=(1<<24)/wt) t = 0
|
||||||
|
else if ( sz>=(1<<20)/wt) t = 3
|
||||||
|
else t = 5
|
||||||
|
}
|
||||||
|
tup{set_thresh, div_thresh}
|
||||||
|
set_thresh{}
|
||||||
|
|
||||||
|
tup{ptrs, sz, sh, div_thresh, hash_resize, {}=>tfree{halloc}}
|
||||||
|
}
|
||||||
@ -1,6 +1,7 @@
|
|||||||
include './base'
|
include './base'
|
||||||
include './mask'
|
include './mask'
|
||||||
include './vecfold'
|
include './vecfold'
|
||||||
|
include './hashtab'
|
||||||
|
|
||||||
def findFirst{C, M, F, ...v1} = {
|
def findFirst{C, M, F, ...v1} = {
|
||||||
def exit = makelabel{}
|
def exit = makelabel{}
|
||||||
@ -329,3 +330,322 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}}
|
exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}}
|
||||||
|
|
||||||
|
|
||||||
|
# Hash tables
|
||||||
|
oper &- ({v:T,m} => v & -promote{T,m}) infix left 35
|
||||||
|
|
||||||
|
def rty{name} = if (to_prim{name}=='∊') i8 else i32
|
||||||
|
def ity{name} = (to_prim{name}=='⊒')**(*u32)
|
||||||
|
fn hashtab{T, name}(rp:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = {
|
||||||
|
# iv,mi/ip,m - searched-in; fv,ni/fp,n - searched-for; may get swapped around & back
|
||||||
|
def prim = to_prim{name}
|
||||||
|
def U = if (prim=='∊') usz else u32
|
||||||
|
m := cast_i{U,mi}; n := cast_i{U,ni}
|
||||||
|
def wt = width{T}
|
||||||
|
ip := *T~~iv; fp := *T~~fv
|
||||||
|
def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}}
|
||||||
|
swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4)
|
||||||
|
if (swap) swap_sides{}
|
||||||
|
|
||||||
|
log := clzc{m}
|
||||||
|
# Max size
|
||||||
|
msl := max{clzc{(m+m/2)|4}+1, min{ux~~14,clzc{m+n/4}}}
|
||||||
|
msz := usz~~1 << msl
|
||||||
|
# Starting log-size (try_vec_memb requires size>4)
|
||||||
|
sl := msl; if (msl>=14) sl = 12+(msl&1)
|
||||||
|
b:U = 64 # Block size
|
||||||
|
|
||||||
|
# Filling e slots past the end requires e*(e+1)/2 collisions, so
|
||||||
|
# m entries with <2 each can fill <sqrt(4*m)
|
||||||
|
def cc_stop = 2*cast_i{u64,m}
|
||||||
|
ext := promote{usz, tern{m<=b, max{m,U~~4}, b + (U~~1 << (log/2 + 1))}}
|
||||||
|
maxh := T~~maxvalue{T}
|
||||||
|
def aux = prim!='∊'
|
||||||
|
def {tabs, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{
|
||||||
|
sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**'any'}, 0, 1
|
||||||
|
}
|
||||||
|
def {hash,...vals} = tabs
|
||||||
|
|
||||||
|
def abort = makelabel{}
|
||||||
|
i:U = 0 # Saved to determine if hashing finished
|
||||||
|
def insert_all{set_tab, set_maxh, dup, ...uniq} = {
|
||||||
|
cc:u64 = 0 # Collision counter
|
||||||
|
while (i < m) {
|
||||||
|
e := tern{m-i>b, i+b, m}
|
||||||
|
while (i < e) {
|
||||||
|
def ii = if (prim!='⊒') i else m-i-1
|
||||||
|
h := hash_val{load{ip,ii}}; j := h>>sh
|
||||||
|
set_maxh{h==maxh, i, j}
|
||||||
|
kv := each{load{.,j}, tabs}; def {k,...kr} = kv
|
||||||
|
# Robin Hood insertion
|
||||||
|
j0 := j; je := j # Save value; end of chain (insert at j)
|
||||||
|
if (k != maxh) {
|
||||||
|
if (k == h) goto{dup}
|
||||||
|
do {
|
||||||
|
++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv
|
||||||
|
def c = promote{T, h >= k}
|
||||||
|
j += c
|
||||||
|
if (kn == h) goto{dup}
|
||||||
|
each{store{.,je-c,.}, tabs, kv}
|
||||||
|
each{=, kv, knv}
|
||||||
|
} while (k != maxh)
|
||||||
|
cc += cast_i{u64, je-j0}
|
||||||
|
}
|
||||||
|
each{{u} => { u += promote{usz,h!=maxh} }, uniq}
|
||||||
|
store{hash, j, h}
|
||||||
|
set_tab{j, h}
|
||||||
|
++i
|
||||||
|
}
|
||||||
|
# Check collision counter and possibly resize
|
||||||
|
def p64 = promote{u64,.}
|
||||||
|
dc := p64{cc} - p64{div_thresh{i}}
|
||||||
|
if (tern{i<m, i64~~dc>=0, sz<msz}) {
|
||||||
|
if (sz == msz) goto{abort}
|
||||||
|
rdc := p64{m-i}*dc # 0 if i==m, no need to recompute
|
||||||
|
mm := p64{i}*p64{m+i}
|
||||||
|
def recheck = setlabel{}
|
||||||
|
if (cc>=cc_stop or p64{n/4}*p64{cc} + rdc >= mm>>(5+log-(wt-sh))) {
|
||||||
|
hash_resize{cc, 2} # Factor of 4
|
||||||
|
if (i==m and sz<msz) goto{recheck}
|
||||||
|
if (cc >= cc_stop) { i=0; goto{abort} }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def get_end{} = {
|
||||||
|
end := maxh>>sh
|
||||||
|
while (load{hash,end}!=maxh) ++end
|
||||||
|
end
|
||||||
|
}
|
||||||
|
def sequester_maxh{j, found} = { j += cast_i{T, ext &- found} }
|
||||||
|
def unsequester_maxh{tab} = {
|
||||||
|
store{tab, get_end{}, load{tab, maxh>>sh + cast_i{T,ext}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
def hash_remove{j,h} = {
|
||||||
|
do {
|
||||||
|
jp:=j; ++j
|
||||||
|
h=load{hash,j}
|
||||||
|
if (h>>sh == j) h = maxh
|
||||||
|
store{hash, jp, h}
|
||||||
|
each{{t} => store{t, jp, load{t,j}}, vals}
|
||||||
|
} while (h!=maxh)
|
||||||
|
}
|
||||||
|
def memb_remove{uniq, has_maxh} = {
|
||||||
|
@for (ip over m) {
|
||||||
|
h := hash_val{ip}; j := h>>sh
|
||||||
|
def shortcut = makelabel{}
|
||||||
|
if (h == maxh) {
|
||||||
|
if (uniq==0) goto{shortcut}
|
||||||
|
has_maxh = 0
|
||||||
|
} else {
|
||||||
|
k := load{hash,j}
|
||||||
|
if (k <= h) {
|
||||||
|
while (k < h) { ++j; k = load{hash,j} }
|
||||||
|
if (k == h) {
|
||||||
|
--uniq
|
||||||
|
if (uniq==0 and not has_maxh) {
|
||||||
|
setlabel{shortcut}
|
||||||
|
@for (rp over n) rp = 1
|
||||||
|
goto{abort}
|
||||||
|
}
|
||||||
|
hash_remove{j,h}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def ind_rev_lookup{uniq, has_maxh} = {
|
||||||
|
def shortcut = makelabel{}
|
||||||
|
@for (ip over i to m) {
|
||||||
|
h := hash_val{ip}; j := h>>sh
|
||||||
|
k := load{hash,j}
|
||||||
|
if (k <= h) {
|
||||||
|
while (k < h) { ++j; k = load{hash,j} }
|
||||||
|
def had_maxh{} = { s:=has_maxh; has_maxh=0; s }
|
||||||
|
if (k == h and (h<maxh or had_maxh{})) {
|
||||||
|
def {inds} = vals
|
||||||
|
store{rp, load{inds, j}, i}
|
||||||
|
--uniq; if (uniq==0) goto{shortcut}
|
||||||
|
hash_remove{j,h}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setlabel{shortcut}
|
||||||
|
}
|
||||||
|
def prog_lookup{swap} = { # Progressive Index-of lookup
|
||||||
|
def rev{a,b} = if (swap) tup{b,a} else tup{a,b}
|
||||||
|
memset{*u32~~rp, ...rev{m,n}}
|
||||||
|
c := m; def shortcut = makelabel{}
|
||||||
|
@for (fp over i to n) {
|
||||||
|
h := hash_val{fp}; j := h>>sh
|
||||||
|
k := load{hash,j}
|
||||||
|
if (k <= h) {
|
||||||
|
while (k < h) { ++j; k = load{hash,j} }
|
||||||
|
if (k==h) {
|
||||||
|
def {inds} = vals; def {link} = links
|
||||||
|
ti := load{inds, j}
|
||||||
|
if (ti > 0) {
|
||||||
|
store{rp, ...rev{i, m-ti}}
|
||||||
|
--c; if (c==0) goto{shortcut}
|
||||||
|
ti = load{link, ti}
|
||||||
|
if (ti > 0 or h==maxh) store{inds, j, ti}
|
||||||
|
else hash_remove{j,h}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setlabel{shortcut}
|
||||||
|
}
|
||||||
|
|
||||||
|
def lookup_all{get_res} = {
|
||||||
|
@for (rp, fp over n) {
|
||||||
|
h := hash_val{fp}; j := h>>sh
|
||||||
|
k := undefined{T}; while ((k=load{hash,j}) < h) ++j
|
||||||
|
rp = get_res{k==h, j}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match (prim, ...vals, ...links) {
|
||||||
|
{('∊')} => {
|
||||||
|
has_maxh:u1 = 0
|
||||||
|
uniq:usz = 0 # Uniques inserted
|
||||||
|
def dup = makelabel{}
|
||||||
|
insert_all{
|
||||||
|
{j, h} => setlabel{dup},
|
||||||
|
{found, i, j} => has_maxh |= found,
|
||||||
|
dup, uniq
|
||||||
|
}
|
||||||
|
if (swap) {
|
||||||
|
swap_sides{}; i=m # i==m return value is kind of dumb
|
||||||
|
memb_remove{uniq,has_maxh} # Remove values in ip from hash
|
||||||
|
}
|
||||||
|
try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort}
|
||||||
|
end := get_end{} # Clip trailing maxh if it shouldn't be in the table
|
||||||
|
if (has_maxh) ++end # Don't clip
|
||||||
|
lookup_all{{found, j} => promote{i8, swap ^ (found & (j<end))}}
|
||||||
|
}
|
||||||
|
|
||||||
|
{('⊐'), inds} => if (not swap) {
|
||||||
|
has_maxh:u1 = 0
|
||||||
|
ind_maxh:u32 = 0
|
||||||
|
def dup = makelabel{}
|
||||||
|
def set{j, h} = {
|
||||||
|
store{inds, j, m-i} # So it can be cleared with one &- in get{}
|
||||||
|
setlabel{dup}
|
||||||
|
}
|
||||||
|
def set_maxh{found, i, j} = {
|
||||||
|
ind_maxh |= i &- (found&~has_maxh)
|
||||||
|
has_maxh |= found
|
||||||
|
}
|
||||||
|
insert_all{set, set_maxh, dup}
|
||||||
|
store{inds, get_end{}, (m-ind_maxh) &- has_maxh}
|
||||||
|
lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))}
|
||||||
|
} else { # swap
|
||||||
|
# After insert_all, position i in rp contains:
|
||||||
|
# - ≠𝕨, if i is the first occurrence of its value in 𝕩, or
|
||||||
|
# - j-≠𝕩, where j<i is the index of the first occurrence
|
||||||
|
uniq:usz = 0
|
||||||
|
has_maxh:u1 = 0
|
||||||
|
ri:i32 = 0 # Placed in rp (should be scoped to insert_all loop body)
|
||||||
|
def dup = makelabel{}
|
||||||
|
def set_maxh{found, i, j} = {
|
||||||
|
sequester_maxh{j, found}
|
||||||
|
ri = cast_i{i32, n} # Initialize to not-found
|
||||||
|
if (found & has_maxh) goto{dup}
|
||||||
|
has_maxh |= found
|
||||||
|
}
|
||||||
|
def set{j, h} = {
|
||||||
|
store{inds, j, i}
|
||||||
|
if (u1~~0) {
|
||||||
|
setlabel{dup}
|
||||||
|
ri = i32~~(load{inds, j} - m)
|
||||||
|
}
|
||||||
|
store{rp, i, ri}
|
||||||
|
}
|
||||||
|
insert_all{set, set_maxh, dup, uniq}
|
||||||
|
# Lookup places correct result index at each first occurrence
|
||||||
|
swap_sides{}; i=m
|
||||||
|
if (has_maxh) { ++uniq; unsequester_maxh{inds} }
|
||||||
|
ind_rev_lookup{uniq, has_maxh}
|
||||||
|
# Propagate to later occurrences
|
||||||
|
@for (r in rp over i to n) r = load{rp, min{i, u32~~r+n}}
|
||||||
|
}
|
||||||
|
|
||||||
|
{('⊒'), inds, link} => {
|
||||||
|
store{link,0,0}
|
||||||
|
store{inds, maxh>>sh + cast_i{T,ext}, 0}
|
||||||
|
def dup = makelabel{}
|
||||||
|
def set{j, h} = {
|
||||||
|
store{inds, j, load{inds,j} &- (h==maxh)}
|
||||||
|
setlabel{dup}
|
||||||
|
i1 := i+1
|
||||||
|
store{link, i1, load{inds,j}}
|
||||||
|
store{inds, j, i1}
|
||||||
|
}
|
||||||
|
def set_maxh{found, i, j} = sequester_maxh{j, found}
|
||||||
|
insert_all{set, set_maxh, dup}
|
||||||
|
unsequester_maxh{inds}
|
||||||
|
if (not swap) prog_lookup{0} else prog_lookup{1}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
setlabel{abort}
|
||||||
|
hash_free{}
|
||||||
|
i == m # Whether it finished
|
||||||
|
}
|
||||||
|
|
||||||
|
def try_vec_memb{..._} = {}
|
||||||
|
def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
|
||||||
|
& hasarch{'SSE4.2'} & T==u32} = {
|
||||||
|
# Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints)
|
||||||
|
# i-h>>sh = i+((1<<sh-1)-h)>>sh = (((i+1)<<sh-1)-h)>>sh
|
||||||
|
# We maintain io = (i+1)<<sh-1
|
||||||
|
def vl = 4; def V = [vl]T
|
||||||
|
assert{sz > 4}
|
||||||
|
assert{sz%vl == 0}
|
||||||
|
io := make{V, each{{k} => T~~k<<sh - 1, 1+iota{vl}}}
|
||||||
|
id := V**(T~~vl<<sh)
|
||||||
|
mv := V**0
|
||||||
|
@for (h in *V~~hash over sz/vl) { mv=max{mv, io-min{h,io}}; io+=id }
|
||||||
|
max_off := vfold{max, mv} >> sh
|
||||||
|
# sz==1<<sh, so when i>=sz the above overflows
|
||||||
|
# And we have to handle maxh specially anyway
|
||||||
|
def mx{i,h} = { max_off = max{max_off, i-h>>sh} }
|
||||||
|
i:T=sz; h:T=i; while ((h=load{hash,i})<maxh) { mx{i,h}; ++i }
|
||||||
|
if (has_maxh) mx{i,maxh}
|
||||||
|
|
||||||
|
vswap := base{256,vl**1} * promote{T,swap}
|
||||||
|
def memb{test} = {
|
||||||
|
def R = i8; def rw = width{R}; def u = width{T}/rw
|
||||||
|
l := n/u
|
||||||
|
@for (r in *T~~rp over i to l) {
|
||||||
|
c := V**0 # Will combine u results to avoid folding too much
|
||||||
|
@unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f}
|
||||||
|
r = vswap ^ vfold{|, c}
|
||||||
|
}
|
||||||
|
@for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}}
|
||||||
|
goto{done}
|
||||||
|
}
|
||||||
|
def try{nv} = {
|
||||||
|
if (max_off < nv*vl) {
|
||||||
|
# Avoid matching maxh if it shouldn't be in the table
|
||||||
|
clear := V**(maxh &- ~has_maxh)
|
||||||
|
@for (hv in *V~~(hash + maxh>>sh) over vl) hv = hv &~ (hv == clear)
|
||||||
|
# Test against nv vectors
|
||||||
|
def test{x} = {
|
||||||
|
h := hash_val{x}; vh := V**h
|
||||||
|
def any{{...r,a}} = any{r}|a; def any{{a}}=a
|
||||||
|
any{@collect (k in *V~~(hash+h>>sh) over nv) vh == k}
|
||||||
|
}
|
||||||
|
memb{test}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
each{try, tup{1,2}}
|
||||||
|
}
|
||||||
|
|
||||||
|
def exp_hash{name} = {
|
||||||
|
exportT{merge{'si_',name,'_c2_hash'}, each{hashtab{., name}, tup{u32,u64}}}
|
||||||
|
}
|
||||||
|
each{exp_hash, names}
|
||||||
|
|||||||
86
src/singeli/src/selfsearch.singeli
Normal file
86
src/singeli/src/selfsearch.singeli
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
include './base'
|
||||||
|
local include 'skin/cext'
|
||||||
|
include './hashtab'
|
||||||
|
|
||||||
|
# Resizing hash table, with fallback
|
||||||
|
def rty{name} = if (to_prim{name}=='∊') i8 else i32
|
||||||
|
fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = {
|
||||||
|
def wt = width{T}
|
||||||
|
def prim = to_prim{name}
|
||||||
|
def has_radix = if (prim=='⊐') 0 else wt==32
|
||||||
|
n64 := promote{u64,n}
|
||||||
|
def {res0,cc_stop} = if (prim=='∊') tup{1,n64>>has_radix} else tup{0,2*n64}
|
||||||
|
|
||||||
|
log := clzc{n}
|
||||||
|
# Max size
|
||||||
|
msl := clzc{n+n/2} + 1; if (has_radix and msl>20) msl=20
|
||||||
|
msz := usz~~1 << msl
|
||||||
|
# Starting log-size
|
||||||
|
sl := msl; if (msl>=14) sl = 12+(msl&1)
|
||||||
|
b:usz = 64 # Block size
|
||||||
|
|
||||||
|
# Filling e slots past the end requires e*(e+1)/2 collisions, so
|
||||||
|
# n entries with <2 each can fill <sqrt(4*n)
|
||||||
|
ext := tern{n<=b, n, b + (usz~~1 << (log/2 + 1))}
|
||||||
|
x0 := hash_val{xp->0}
|
||||||
|
rp <-{0} res0
|
||||||
|
def aux = prim!='∊'
|
||||||
|
def {{hash,...vals}, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{
|
||||||
|
sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, has_radix, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
def {output, write_res} = match (prim) {
|
||||||
|
{('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }}
|
||||||
|
{('⊒')} => {
|
||||||
|
ctr0:u32 = 1; def {val} = vals
|
||||||
|
def res{j,h,k,x0} = {
|
||||||
|
e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0
|
||||||
|
hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0
|
||||||
|
vj + (c0 & -e0)
|
||||||
|
}
|
||||||
|
tup{{b}=>b, res}
|
||||||
|
}
|
||||||
|
{('⊐')} => {
|
||||||
|
ctr:u32 = 1; def {val} = vals
|
||||||
|
def res{j,h,k,x0} = {
|
||||||
|
if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h }
|
||||||
|
val->j
|
||||||
|
}
|
||||||
|
tup{{b}=>ctr & -promote{u32,b}, res}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def break=makelabel{}
|
||||||
|
cc:u64 = 0 # Collision counter
|
||||||
|
i:usz=1; while (1) {
|
||||||
|
e := tern{n-i>b, i+b, n}
|
||||||
|
while (i < e) {
|
||||||
|
h := hash_val{xp->i}; j0 := h>>sh; j := j0
|
||||||
|
k:=undefined{T}; while (((k=hash->j)!=h) & (k!=x0)) ++j
|
||||||
|
cc += cast_i{u64, j-j0}
|
||||||
|
rp <-{i} write_res{j,h,k,x0}
|
||||||
|
++i
|
||||||
|
}
|
||||||
|
if (i == n) goto{break}
|
||||||
|
# Check collision counter and possibly resize
|
||||||
|
def p64 = promote{u64,.}
|
||||||
|
dc := cc - p64{div_thresh{i}}
|
||||||
|
if (i64~~dc >= 0) {
|
||||||
|
if (sz == msz) goto{break} # Abort
|
||||||
|
if (has_radix and i < n/2 and sz >= 1<<18) goto{break}
|
||||||
|
# Avoid resizing if close to the end
|
||||||
|
if (cc>=cc_stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) {
|
||||||
|
hash_resize{cc, 2} # Factor of 4
|
||||||
|
if (cc >= cc_stop) goto{break}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setlabel{break}
|
||||||
|
hash_free{}
|
||||||
|
output{i == n} # Whether it finished, and unique count if ⊐
|
||||||
|
}
|
||||||
|
|
||||||
|
def exp_hash{T, name} = {
|
||||||
|
export{merge{name,'_c1_hash',fmtnat{width{T}}}, selfhashtab{T, name}}
|
||||||
|
}
|
||||||
|
each{{n}=>each{exp_hash{.,n},tup{u32,u64}}, names}
|
||||||
@ -1,5 +1,4 @@
|
|||||||
# Fold associative/commutative operation across a register
|
# Fold associative/commutative operation across a register
|
||||||
# Used by squeeze.singeli, count.singeli
|
|
||||||
|
|
||||||
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
|
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
|
||||||
c:= x
|
c:= x
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user