Merge pull request #96 from mlochbaum/search

Fast 32- and 64-bit dyadic search hash tables
This commit is contained in:
dzaima 2023-11-13 18:18:04 +02:00 committed by GitHub
commit e4cef4f46c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 758 additions and 178 deletions

View File

@ -648,11 +648,11 @@ cachedBin‿linkerCache ← {
"xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp", "xa.""src/builtins/arithd.c""dyarith", "xa.""src/builtins/cmp.c""cmp",
"xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy", "xa.""src/builtins/squeeze.c""squeeze", "xa.""src/utils/mut.c""copy",
"xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose", "xa.""src/utils/bits.c""bits", "xag""src/builtins/transpose.c""transpose",
"xag""src/builtins/search.c""search", "xa.""src/builtins/fold.c""fold", "xag""src/builtins/search.c""search", "xag""src/builtins/selfsearch.c""selfsearch"
"xag""src/builtins/scan.c""scan", "xa.""src/builtins/fold.c""fold",
"xag""src/builtins/sort.c""bins" "xag""src/builtins/sort.c""bins"
"2..""src/builtins/select.c""select", "xag""src/builtins/scan.c""scan", "2..""src/builtins/select.c""select", "2a.""src/builtins/slash.c""constrep",
"2a.""src/builtins/slash.c""constrep",
"xag""src/builtins/slash.c""slash", "2..""src/builtins/slash.c""count" "xag""src/builtins/slash.c""slash", "2..""src/builtins/slash.c""count"
objs objs

View File

@ -1,29 +1,49 @@
// Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒) // Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒)
// 𝕨⊐unit or unit∊𝕩: scalar loop with early-exit // 𝕨⊐unit or unit∊𝕩: SIMD shortcutting search
// SHOULD use simd // 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩
// SHOULD unify implementations // High-rank inputs:
// 𝕩⊒unit or 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩 // Convert to a typed numeric list if cells are ≤62 bits
// COULD have hashing for equal-type >64 bit cells, to skip squeezing
// Both arguments with rank≥1: // COULD try squeezing ahead-of-time to skip it in bqn_hash
// High-rank inputs: // SHOULD have fast path when cell sizes don't match
// Convert to a (lower-rank) typed integer array if cells are ≤62 bits // One input empty: fast not-found
// COULD have special hashing for equal type >64 bit cells, skipping squeezing // Character elements:
// COULD try conditionally squeezing ahead-of-time, and not squeezing in bqn_hash // Character versus number array is fast not-found for ∊ and ⊐
// p⊐n & n∊p with short p & long n: n⊸=¨ p // SHOULD have fast character-number path for ⊒
// bitarr⊐𝕩: more special arithmetic // Reinterpret as integer elements
// SHOULD have impls for long p & short n // COULD try p=⌜n when all arguments are short (may not be faster?)
// ≤16-bit elements: lookup tables // p⊐n & n∊p with short n: p⊸⊐¨n
// Character elements: reinterpret as integer elements // p⊐n & n∊p with boolean p: based on ⊑p and p⊐¬⊑p
// Otherwise, generic hashtable // p⊐n & n∊p with short p:
// SHOULD handle up to 64 bit cells via proper typed hash tables // AVX2 binary search when applicable
// SHOULD have fast path when cell sizes or element types doesn't match // SHOULD apply vector binary search to characters (sort as ints)
// n⊸=¨p otherwise
// ≤16-bit elements: lookup tables
// 8-bit ∊ and ⊐: SSSE3 table
// SHOULD make 8-bit NEON table
// SHOULD have branchy reverse 16-bit table search
// 32- or 64-bit elements: hash tables
// Store hash in table and not element; Robin Hood ordering; resizing
// Reverse hash if searched-for is shorter
// Shortcutting for reverse hashes and non-reversed ⊒
// SIMD lookup for 32-bit ∊ if chain length is small enough
// SHOULD partition if hash table size gets large
// SHOULD handle unequal search types better
// Otherwise, generic hashtable
#include "../core.h" #include "../core.h"
#include "../utils/hash.h" #include "../utils/hash.h"
#include "../utils/talloc.h" #include "../utils/talloc.h"
#include "../utils/calls.h" #include "../utils/calls.h"
extern NOINLINE void memset16(u16* p, u16 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
extern NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
extern NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
#define memset_i8 memset
#define memset_i16(P,V,L) memset16((u16*)(P), V, L)
#define memset_i32(P,V,L) memset32((u32*)(P), V, L)
RangeFn getRange_fns[el_f64+1]; RangeFn getRange_fns[el_f64+1];
#if SINGELI #if SINGELI
extern RangeFn* const simd_getRangeRaw; extern RangeFn* const simd_getRangeRaw;
@ -42,15 +62,31 @@ RangeFn getRange_fns[el_f64+1];
GETRANGE(i32,) GETRANGE(i32,)
GETRANGE(f64, if (!q_fi64(c)) return 0) GETRANGE(f64, if (!q_fi64(c)) return 0)
#endif #endif
#if SINGELI_AVX2
extern void (**const avx2_member_sort)(uint64_t*,void*,uint64_t,void*,uint64_t);
extern void (**const avx2_indexOf_sort)(int8_t*,void*,uint64_t,void*,uint64_t);
#endif
#define C2i(F, W, X) C2(F, m_i32(W), X) #define C2i(F, W, X) C2(F, m_i32(W), X)
extern B and_c1(B,B);
extern B gradeDown_c1(B,B);
extern B reverse_c1(B,B);
extern B eq_c2(B,B,B); extern B eq_c2(B,B,B);
extern B ne_c2(B,B,B); extern B ne_c2(B,B,B);
extern B or_c2(B,B,B); extern B or_c2(B,B,B);
extern B add_c2(B,B,B); extern B add_c2(B,B,B);
extern B sub_c2(B,B,B); extern B sub_c2(B,B,B);
extern B mul_c2(B,B,B); extern B mul_c2(B,B,B);
extern B join_c2(B,B,B);
extern B select_c2(B,B,B);
B asNormalized(B x, usz n, bool nanBad);
SHOULD_INLINE bool canCompare64_norm2(B* w, usz wia, B* x, usz xia) {
B wn=asNormalized(*w,wia,true); if (wn.u == m_f64(0).u) return 0; *w=wn;
B xn=asNormalized(*x,xia,true); if (xn.u == m_f64(0).u) return 0; *x=xn;
return 1;
}
static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); } static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); }
@ -66,7 +102,7 @@ static u64 elRange(u8 eltype) { return 1ull<<(1<<elwBitLog(eltype)); }
if (IN.u != FOR.u) { \ if (IN.u != FOR.u) { \
if (FOR##e==el_i16 && n<ft/(64/sizeof(TY))) \ if (FOR##e==el_i16 && n<ft/(64/sizeof(TY))) \
{ for (usz i=0; i<n; i++) tab[((i16*)fp)[i]]=INIT; } \ { for (usz i=0; i<n; i++) tab[((i16*)fp)[i]]=INIT; } \
else { TY* to=tab-(ft/2-(ft==2)); for (i64 i=0; i<ft; i++) to[i]=INIT; } \ else { memset_##TY(tab-(ft/2-(ft==2)), INIT, ft); } \
} \ } \
/* Set */ \ /* Set */ \
if (IN##e==el_i8) { for (usz i=m; i--; ) tab[((i8 *)ip)[i]]=SET; } \ if (IN##e==el_i8) { for (usz i=m; i--; ) tab[((i8 *)ip)[i]]=SET; } \
@ -191,18 +227,26 @@ static NOINLINE usz indexOfOne(B l, B e) {
#endif #endif
} }
#define CHR_TO_INT if (elChr(we) && elChr(xe)) { \ #define CHECK_CHRS_ELSE \
if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \ if (!elNum(we)) { \
if (we>xe) x=taga(cpyC16Arr(x)); \ if (elChr(we)) { \
else w=taga(cpyC16Arr(w)); \ if (elNum(xe)) goto none_found; \
we = xe = el_i16; \ if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \
goto tyEls; \ if (we>xe) x=taga(cpyC16Arr(x)); \
} \ else w=taga(cpyC16Arr(w)); \
we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \ we = xe = el_i16; \
} goto tyEls; \
} \
we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \
} \
} else if (!elNum(xe)) { \
if (elChr(xe)) goto none_found; \
} else
B indexOf_c2(B t, B w, B x) { B indexOf_c2(B t, B w, B x) {
bool split = 0; (void) split;
if (RARE(!isArr(w) || RNK(w)!=1)) { if (RARE(!isArr(w) || RNK(w)!=1)) {
split = 1;
B2 t = splitCells(x, w, 1); B2 t = splitCells(x, w, 1);
w = t.p; w = t.p;
x = t.n; x = t.n;
@ -218,10 +262,23 @@ B indexOf_c2(B t, B w, B x) {
u8 we = TI(w,elType); usz wia = IA(w); u8 we = TI(w,elType); usz wia = IA(w);
u8 xe = TI(x,elType); usz xia = IA(x); u8 xe = TI(x,elType); usz xia = IA(x);
if (wia==0 || xia==0) { if (wia==0 || xia==0) {
decG(w); return i64EachDec(0, x); none_found:
decG(w); return i64EachDec(wia, x);
} }
if (elNum(we) && elNum(xe)) { tyEls: CHECK_CHRS_ELSE { tyEls: // Both numbers
if (wia>32 && xia<=(we<=el_i8?1:3)) {
SGetU(x);
B r;
#define IND(T) \
T* rp; r = m_##T##arrc(&rp, x); \
for (usz i=0; i<xia; i++) rp[i] = indexOfOne(w, GetU(x,i))
if (xia<I32_MAX) { IND(i32); r=reduceI32Width(r, wia); }
else { IND(f64); }
#undef IND
decG(w); decG(x); return r;
}
if (we==el_bit) { if (we==el_bit) {
u64* wp = bitarr_ptr(w); u64* wp = bitarr_ptr(w);
u64 w0 = 1 & wp[0]; u64 w0 = 1 & wp[0];
@ -231,7 +288,18 @@ B indexOf_c2(B t, B w, B x) {
return i==wia? r : C2(sub, r, C2i(mul, wia-i, C2i(eq, !w0, x))); return i==wia? r : C2(sub, r, C2i(mul, wia-i, C2i(eq, !w0, x)));
} }
if (wia<=(we<=el_i16?4:16) && xia>16) { #if SINGELI_AVX2
if (xia>=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<(xe==el_i8?64:16) && we<=xe && !elChr(TI(x,elType))) {
B g = C1(reverse, C1(gradeDown, incG(w)));
w = C2(select, incG(g), w);
switch (xe) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; }
i8* rp; B r = m_i8arrc(&rp, x);
avx2_indexOf_sort[xe-el_i8](rp, tyany_ptr(w), wia, tyany_ptr(x), xia);
r = C2(select, r, C2(join, g, m_i8(wia)));
decG(w); decG(x); return r;
}
#endif
if (wia<=4 && xia>16) {
SGetU(w); SGetU(w);
#define XEQ(I) C2(ne, GetU(w,I), incG(x)) #define XEQ(I) C2(ne, GetU(w,I), incG(x))
B r = XEQ(wia-1); B r = XEQ(wia-1);
@ -241,23 +309,37 @@ B indexOf_c2(B t, B w, B x) {
} }
if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { if (xia+wia>20 && we<=el_i16 && xe<=el_i16) {
B r;
#if SINGELI #if SINGELI
if (wia>256 && we==el_i8 && xe==el_i8) { if (wia>256 && we==el_i8 && xe==el_i8) {
TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256); TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256);
void* fp = tyany_ptr(x); void* fp = tyany_ptr(x);
simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind); simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind);
decG(w); decG(w);
i32* rp; B r = m_i32arrc(&rp, x); #define LOOKUP(TY) \
for (usz i=0; i<xia; i++) rp[i]=ind[((u8*)fp)[i]]; TY* rp; r = m_##TY##arrc(&rp, x); \
for (usz i=0; i<xia; i++) rp[i]=ind[((u8*)fp)[i]];
if (wia<=INT16_MAX) { LOOKUP(i16) } else { LOOKUP(i32) }
#undef LOOKUP
TFREE(tab); decG(x); TFREE(tab); decG(x);
return reduceI32Width(r, wia); return r;
} }
#endif #endif
B r; if (wia<= INT8_MAX) { TABLE(w, x, i8, wia, i) }
TABLE(w, x, i32, wia, i) else if (wia<=INT16_MAX) { TABLE(w, x, i16, wia, i) }
return reduceI32Width(r, wia); else { TABLE(w, x, i32, wia, i) }
return r;
} }
} else { CHR_TO_INT; } #if SINGELI
if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) {
i32* rp; B r = m_i32arrc(&rp, x);
if (si_indexOf_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia)) {
decG(w); decG(x); return reduceI32Width(r, wia);
}
decG(r);
}
#endif
}
i32* rp; B r = m_i32arrc(&rp, x); i32* rp; B r = m_i32arrc(&rp, x);
H_b2i* map = m_b2i(64); H_b2i* map = m_b2i(64);
@ -275,7 +357,9 @@ B indexOf_c2(B t, B w, B x) {
B enclosed_0, enclosed_1; B enclosed_0, enclosed_1;
B memberOf_c2(B t, B w, B x) { B memberOf_c2(B t, B w, B x) {
bool split = 0; (void) split;
if (isAtm(x) || RNK(x)!=1) { if (isAtm(x) || RNK(x)!=1) {
split = 1;
B2 t = splitCells(w, x, false); B2 t = splitCells(w, x, false);
w = t.n; w = t.n;
x = t.p; x = t.p;
@ -302,10 +386,18 @@ B memberOf_c2(B t, B w, B x) {
u8 we = TI(w,elType); usz wia = IA(w); u8 we = TI(w,elType); usz wia = IA(w);
u8 xe = TI(x,elType); usz xia = IA(x); u8 xe = TI(x,elType); usz xia = IA(x);
if (wia==0 || xia==0) { if (wia==0 || xia==0) {
none_found:
decG(x); return i64EachDec(0, w); decG(x); return i64EachDec(0, w);
} }
if (elNum(we) && elNum(xe)) { tyEls: CHECK_CHRS_ELSE { tyEls: // Both numbers
if (xia>32 && wia<=(xe<=el_i8?1:xe==el_i32?4:6)) {
SGetU(w);
i8* rp; r = m_i8arrc(&rp, w);
for (usz i=0; i<wia; i++) rp[i] = indexOfOne(x, GetU(w,i)) < xia;
decG(w); decG(x); return taga(cpyBitArr(r));
}
#define WEQ(V) C2(eq, incG(w), V) #define WEQ(V) C2(eq, incG(w), V)
if (xe==el_bit) { if (xe==el_bit) {
u64* xp = bitarr_ptr(x); u64* xp = bitarr_ptr(x);
@ -315,8 +407,16 @@ B memberOf_c2(B t, B w, B x) {
decG(w); goto dec_x; decG(w); goto dec_x;
} }
u8 me = we>xe?we:xe; #if SINGELI_AVX2
if (xia<=(me==el_i8?1:me==el_i16?4:16) && wia>16) { if (wia>=32>>(we-el_i8) && xia>1 && ((we==el_i16 && xia<32) || (we==el_i32 && xia<16)) && xe<=we && !elChr(TI(x,elType))) {
x = C1(and, x); // sort
if (xe<we) switch (we) { default:UD; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; }
u64* rp; r = m_bitarrc(&rp, w);
avx2_member_sort[we-el_i16](rp, tyany_ptr(x), xia, tyany_ptr(w), wia);
decG(w); goto dec_x;
}
#endif
if (xia<=(we==el_i8?1:we==el_i16?4:8) && wia>16) {
SGetU(x); SGetU(x);
r = WEQ(GetU(x,0)); r = WEQ(GetU(x,0));
for (usz i=1; i<xia; i++) r = C2(or, r, WEQ(GetU(x,i))); for (usz i=1; i<xia; i++) r = C2(or, r, WEQ(GetU(x,i)));
@ -337,7 +437,16 @@ B memberOf_c2(B t, B w, B x) {
TABLE(x, w, i8, 0, 1) TABLE(x, w, i8, 0, 1)
return taga(cpyBitArr(r)); return taga(cpyBitArr(r));
} }
} else { CHR_TO_INT; } #if SINGELI
if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) {
i8* rp; B r = m_i8arrc(&rp, w);
if (si_memberOf_c2_hash[we-el_i32](rp, tyany_ptr(x), xia, tyany_ptr(w), wia)) {
decG(w); decG(x); return taga(cpyBitArr(r));
}
decG(r);
}
#endif
}
H_Sb* set = m_Sb(64); H_Sb* set = m_Sb(64);
SGetU(x) SGetU(w) SGetU(x) SGetU(w)
@ -353,10 +462,12 @@ B memberOf_c2(B t, B w, B x) {
decG(x); decG(x);
return r; return r;
} }
#undef CHR_TO_INT #undef CHECK_CHRS_ELSE
B count_c2(B t, B w, B x) { B count_c2(B t, B w, B x) {
bool split = 0; (void) split;
if (RARE(!isArr(w) || RNK(w)!=1)) { if (RARE(!isArr(w) || RNK(w)!=1)) {
split = 1;
B2 t = splitCells(x, w, 2); B2 t = splitCells(x, w, 2);
w = t.p; w = t.p;
x = t.n; x = t.n;
@ -396,6 +507,12 @@ B count_c2(B t, B w, B x) {
we-= el_c8-el_i8; xe-= el_c8-el_i8; we-= el_c8-el_i8; xe-= el_c8-el_i8;
goto el8or16; goto el8or16;
} else { } else {
#if SINGELI
if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia)))) &&
si_count_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia, (u32*)wnext)) {
goto dec_nwx;
}
#endif
H_b2i* map = m_b2i(64); H_b2i* map = m_b2i(64);
SGetU(x) SGetU(x)
SGetU(w) SGetU(w)
@ -412,6 +529,9 @@ B count_c2(B t, B w, B x) {
} }
free_b2i(map); free_b2i(map);
} }
#if SINGELI
dec_nwx:;
#endif
TFREE(wnext); decG(w); decG(x); TFREE(wnext); decG(w); decG(x);
return reduceI32Width(r, wia); return reduceI32Width(r, wia);
} }

View File

@ -43,25 +43,19 @@ extern B mul_c2(B, B, B);
extern B scan_add_bool(B x, u64 ia); extern B scan_add_bool(B x, u64 ia);
extern B scan_max_num(B x, u8 xe, u64 ia); extern B scan_max_num(B x, u8 xe, u64 ia);
// These hashes are stored in tables and must be invertible! // From search.c
#if defined(__SSE4_2__) extern NOINLINE void memset32(u32* p, u32 v, usz l);
#include <x86intrin.h> extern NOINLINE void memset64(u64* p, u64 v, usz l);
static inline u32 hash32(u32 x) { return _mm_crc32_u32(0x973afb51, x); }
#if SINGELI
#define SINGELI_FILE selfsearch
#include "../utils/includeSingeli.h"
#define TRY_HASHTAB_RET(NAME, W, RET) \
if (NAME##_c1_hash##W(rp, (u##W*)xv, n)) { decG(x); return RET; }
#else #else
// Murmur3 #define TRY_HASHTAB_RET(NAME, W, RET) (void)0;
static inline u32 hash32(u32 x) {
x ^= x >> 16; x *= 0x85ebca6b;
x ^= x >> 13; x *= 0xc2b2ae35;
x ^= x >> 16;
return x;
}
#endif #endif
static inline u64 hash64(u64 x) {
x ^= x >> 33; x *= 0xff51afd7ed558ccd;
x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53;
x ^= x >> 33;
return x;
}
static inline bool use_sorted(B x, u8 logw) { static inline bool use_sorted(B x, u8 logw) {
if (!FL_HAS(x, fl_asc|fl_dsc)) return 0; if (!FL_HAS(x, fl_asc|fl_dsc)) return 0;
@ -146,73 +140,6 @@ u8 radix_offsets_2_usz(usz* c0, u32* v0, usz n) {
} \ } \
decG(x); TFREE(alloc); decG(x); TFREE(alloc);
static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i<l; i++) p[i]=v; }
// Resizing hash table, with fallback
#define SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, AUXSIZE, AUXINIT, AUXEXTEND, AUXMOVE) \
usz log = 64 - CLZ(n); \
usz msl = (64 - CLZ(n+n/2)) + 1; if (RAD && msl>20) msl=20; \
usz sh = W - (msl<14? msl : 12+(msl&1)); /* Shift to fit to table */ \
usz sz = 1 << (W - sh); /* Initial size */ \
usz msz = 1ull << msl; /* Max sz */ \
usz b = 64; /* Block size */ \
/* Resize or abort if more than 1/2^thresh collisions/element */ \
usz thresh = THRESH; \
/* Filling e slots past the end requires e*(e+1)/2 collisions, so */ \
/* n entries with <2 each can fill <sqrt(4*n) */ \
usz ext = n<=b? n : b + (1ull << (log/2 + 1)); \
TALLOC(u8, halloc, (msz+ext)*(sizeof(T)+AUXSIZE)); \
T* hash = (T*)halloc + msz-sz; \
T x0 = hash##W(xp[0]); rp[0] = RES0; \
memset##W(hash, x0, sz+ext); \
AUXINIT \
usz cc = 0; /* Collision counter */ \
usz i=1; while (1) { \
usz e = n-i>b? i+b : n; \
for (; i < e; i++) { \
T h = hash##W(xp[i]); T j0 = h>>sh, j = j0; T k; \
while (k=hash[j], k!=h & k!=x0) j++; \
cc += j-j0; \
RESWRITE \
} \
if (i == n) break; \
i64 dc = (i64)cc - ((THRESHMUL*i)>>thresh); \
if (dc >= 0) { \
if (sz == msz || (RAD && i < n/2 && sz >= 1<<18)) break; /*Abort*/ \
/* Avoid resizing if close to the end */ \
if (cc<STOP && (n-i)*dc < (i*((i64)n+i))>>(5+log-(W-sh))) continue;\
/* Resize hash, factor of 4 */ \
usz m = 2; \
usz dif = sz*((1<<m)-1); \
sh -= m; sz <<= m; \
hash -= dif; \
usz j = 0; \
cc = 0; \
memset##W(hash, x0, dif); \
AUXEXTEND \
for (j = dif; j < sz + ext; j++) { \
T h = hash[j]; if (h==x0) continue; hash[j] = x0; \
T k0 = h>>sh, k = k0; while (hash[k]!=x0) k++; \
cc += k-k0; \
hash[k] = h; AUXMOVE \
} \
if (cc >= STOP) break; \
thresh = THRESH; \
} \
} \
TFREE(halloc); \
if (i==n) { decG(x); return RESULT; }
#define SELFHASHTAB_VAL(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, INIT) \
SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, \
/*AUXSIZE*/sizeof(u32), \
/* AUXINIT */ \
u32* val = (u32*)(hash+sz+ext) + msz-sz; \
memset32(val, 0, sz+ext); \
INIT , \
/*AUXEXTEND*/val -= dif; memset32(val, 0, dif); , \
/*AUXMOVE*/u32 v = val[j]; val[j] = 0; val[k] = v;)
extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*); extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*);
extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*); extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*);
@ -268,14 +195,11 @@ B memberOf_c1(B t, B x) {
} } } }
if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } } if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } }
#undef LOOKUP #undef LOOKUP
#define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB( \ #define TRY_HASHTAB(W) TRY_HASHTAB_RET(memberOf, W, taga(cpyBitArr(r)))
T, W, RAD, STOP, \
1, taga(cpyBitArr(r)), hash[j]=h; rp[i]=k!=h;, \
1, THRESH, 0,,,)
if (lw==5) { if (lw==5) {
if (n<12) { BRUTE(32); } if (n<12) { BRUTE(32); }
i8* rp; B r = m_i8arrv(&rp, n); i8* rp; B r = m_i8arrv(&rp, n);
HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<15)? 3 : 5) TRY_HASHTAB(32)
// Radix-assisted lookup when hash table gives up // Radix-assisted lookup when hash table gives up
usz rx = 256, tn = 1<<16; // Radix; table length usz rx = 256, tn = 1<<16; // Radix; table length
@ -301,10 +225,10 @@ B memberOf_c1(B t, B x) {
if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (lw==6 && canCompare64_norm(&x, &xv, n)) {
if (n<20) { BRUTE(64); } if (n<20) { BRUTE(64); }
i8* rp; B r = m_i8arrv(&rp, n); i8* rp; B r = m_i8arrv(&rp, n);
HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) TRY_HASHTAB(64)
decG(r); // Fall through decG(r); // Fall through
} }
#undef HASHTAB #undef TRY_HASHTAB
#undef BRUTE #undef BRUTE
if (RNK(x)>1) { if (RNK(x)>1) {
@ -377,18 +301,11 @@ B count_c1(B t, B x) {
if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } }
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
#undef LOOKUP #undef LOOKUP
#define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB_VAL( \ #define TRY_HASHTAB(W) TRY_HASHTAB_RET(count, W, num_squeeze(r))
T, W, RAD, STOP, \
/*RES0*/0, /*RETURN*/num_squeeze(r), \
/* RESWRITE */ \
bool e0=h==x0; rp[i]=val[j]+(ctr0&-(u32)e0); \
hash[j]=h; val[j]+=!e0; ctr0+=e0; , \
/*THRESHMUL*/1, THRESH, \
/*INIT*/u32 ctr0 = 1;)
if (lw==5) { if (lw==5) {
if (n<20) { BRUTE(32); } if (n<20) { BRUTE(32); }
i32* rp; B r = m_i32arrv(&rp, n); i32* rp; B r = m_i32arrv(&rp, n);
HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<14)? 3 : 5) TRY_HASHTAB(32)
// Radix-assisted lookup // Radix-assisted lookup
usz rx = 256, tn = 1<<16; // Radix; table length usz rx = 256, tn = 1<<16; // Radix; table length
u32* v0 = (u32*)xv; u32* v0 = (u32*)xv;
@ -413,10 +330,10 @@ B count_c1(B t, B x) {
if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (lw==6 && canCompare64_norm(&x, &xv, n)) {
if (n<20) { BRUTE(64); } if (n<20) { BRUTE(64); }
i32* rp; B r = m_i32arrv(&rp, n); i32* rp; B r = m_i32arrv(&rp, n);
HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) TRY_HASHTAB(64)
decG(r); // Fall through decG(r); // Fall through
} }
#undef HASHTAB #undef TRY_HASHTAB
#undef BRUTE #undef BRUTE
if (RNK(x)>1) { if (RNK(x)>1) {
@ -485,18 +402,19 @@ B indexOf_c1(B t, B x) {
if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } }
if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } }
#undef LOOKUP #undef LOOKUP
#define HASHTAB(T, W, THRESH) SELFHASHTAB_VAL(T, W, 0, 2*n, \ #if SINGELI
/*RES0*/0, /*RETURN*/reduceI32WidthBelow(r, ctr), \ #define TRY_HASHTAB(W) \
/* RESWRITE */ \ u32 ctr = indexOf_c1_hash##W(rp, (u##W*)xv, n); \
if (k!=h) { val[j]=ctr++; hash[j]=h; } rp[i]=val[j]; , \ if (ctr>0) { decG(x); return reduceI32WidthBelow(r, ctr); }
/*THRESHMUL*/2, THRESH, \ #else
/*INIT*/u32 ctr = 1;) #define TRY_HASHTAB(W) (void)0;
#endif
if (lw==5) { if (lw==5) {
if (n<12) { BRUTE(32); } if (n<12) { BRUTE(32); }
B r; B r;
i32* rp; r = m_i32arrv(&rp, n); i32* rp; r = m_i32arrv(&rp, n);
i32* xp = tyany_ptr(x); i32* xp = xv;
i32 min=I32_MAX, max=I32_MIN; i32 min=I32_MAX, max=I32_MIN;
for (usz i = 0; i < n; i++) { for (usz i = 0; i < n; i++) {
i32 c = xp[i]; i32 c = xp[i];
@ -512,17 +430,16 @@ B indexOf_c1(B t, B x) {
decG(x); decG(x);
return reduceI32WidthBelow(r, u); return reduceI32WidthBelow(r, u);
} }
HASHTAB(u32, 32, sz==msz? 0 : sz>=(1<<18)? 1 : sz>=(1<<14)? 4 : 6) TRY_HASHTAB(32)
decG(r); // Fall through decG(r); // Fall through
} }
if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (lw==6 && canCompare64_norm(&x, &xv, n)) {
if (n<16) { BRUTE(64); } if (n<16) { BRUTE(64); }
i32* rp; B r = m_i32arrv(&rp, n); i32* rp; B r = m_i32arrv(&rp, n);
u64* xp = tyany_ptr(x); TRY_HASHTAB(64)
HASHTAB(u64, 64, sz==msz? 0 : sz>=(1<<17)? 1 : sz>=(1<<13)? 4 : 6)
decG(r); // Fall through decG(r); // Fall through
} }
#undef HASHTAB #undef TRY_HASHTAB
#undef BRUTE #undef BRUTE
#undef DOTAB #undef DOTAB

View File

@ -200,7 +200,9 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = {
} }
# Binary search within vector registers # Binary search within vector registers
def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def up = prim != '⍒'
def search = (prim=='∊') | (prim=='⊐')
assert{wn > 1}; assert{wn < maxwn} assert{wn > 1}; assert{wn < maxwn}
def wd = width{T} def wd = width{T}
def I = if (wd<32) u8 else u32; def wi = width{I} def I = if (wd<32) u8 else u32; def wi = width{I}
@ -214,7 +216,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
log := ceil_log2{wn+1} log := ceil_log2{wn+1}
gap := 1<<log - wn gap := 1<<log - wn
# Fill with minimum value at the beginning # Fill with minimum value at the beginning
def pre = (if (up) minvalue else maxvalue){T} def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T}
wg := *V~~(w-gap) wg := *V~~(w-gap)
wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}} wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}}
# For multiple lanes, interleave like transpose # For multiple lanes, interleave like transpose
@ -240,7 +242,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef' def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef'
def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef'
# Offset at end # Offset at end
off := U~~V**cast_i{i8, gap-1} off := U~~V**cast_i{i8, gap-(1-search)}
# Midpoint bits for each step # Midpoint bits for each step
def lowbits = bb{copy{isub,isub}} def lowbits = bb{copy{isub,isub}}
bits := each{{j} => U**(lowbits << j), iota{lstep}} bits := each{{j} => U**(lowbits << j), iota{lstep}}
@ -252,33 +254,61 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
if (this) @for_vec_overlap{vl} (j to xn) { if (this) @for_vec_overlap{vl} (j to xn) {
xv:= load{*V~~(x+j), 0} xv:= load{*V~~(x+j), 0}
s := U**bb{iota{isub}} # Select sequential bytes within each U s := U**bb{iota{isub}} # Select sequential bytes within each U
def ltx{se,ind} = lt{xv, V~~se{re_el{I,ind}}} def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}}
def ltx = cmpx{lt}; def eqx = cmpx{==}
@unroll (j to klog) { @unroll (j to klog) {
m := s | tupsel{klog-1-j,bits} m := s | tupsel{klog-1-j,bits}
s = homBlend{m, s, ltx{selw, m}} s = homBlend{m, s, ltx{selw, m}}
} }
r := if (isub==1) s else s>>(lb{isub}+wd-wi) r := if (isub==1) s else s>>(lb{isub}+wd-wi)
# b records if xv was found; c is added to the index
def r_out = prim!='∊'
def get_up{var,cmpx,use}{set,...a} = if (use) set{var, cmpx{...a}}
b := undefined{U}; def up_b = get_up{b, eqx, search}
c := undefined{U}; def up_c = get_up{c, ltx, r_out}
up_b{=, selw, s}
# Extra selection lanes # Extra selection lanes
if (last and ex>=1 and log>=klog+1) { if (last and ex>=1 and log>=klog+1) {
r += r r += r
c := ltx{selw1,s} def up_bc{set,se} = { up_b{|=,se,s}; up_c{set,se,s} }
up_bc{=,selw1}
if (ex>=2 and log>=klog+2) { if (ex>=2 and log>=klog+2) {
r += r r += r
each{{se} => c += ltx{se,s}, selw2} each{up_bc{+=, .}, selw2}
} }
r += c if (r_out) r += c
}
if (r_out) {
r -= off
if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b}
rn := if (T==i8) r
else if (T==i16) half{narrow{u8, r}, 0}
else extract{re_el{i64, narrow{u8, r}}, 0}
rnp := *type{rn}~~(*i8~~rp+j)
if (isvec{type{rn}}) store{rnp, 0, rn}
else storeu{rnp, rn}
} else {
def B = ty_u{vl}; out := cast_i{B, homMask{b}}
store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)}
} }
r -= off
rn := if (T==i8) r
else if (T==i16) half{narrow{u8, r}, 0}
else extract{re_el{i64, narrow{u8, r}}, 0}
rnp := *type{rn}~~(*i8~~rp+j)
if (isvec{type{rn}}) store{rnp, 0, rn}
else storeu{rnp, rn}
} }
} }
} }
if (hasarch{'AVX2'}) {
fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = {
bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn}
}
exportT{
'avx2_member_sort',
each{avx2_search_bin{'∊',.,.}, tup{i16,i32}, tup{32,16}}
}
exportT{
'avx2_indexOf_sort',
each{avx2_search_bin{'⊐',.,.}, tup{i8,i16,i32}, tup{64,16,16}}
}
}
def unroll_sizes = tup{4,1} def unroll_sizes = tup{4,1}
fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = { fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = {
each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs} each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs}
@ -321,7 +351,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = {
# For >=8 i8 values, vector bit-table is as good as binary search # For >=8 i8 values, vector bit-table is as good as binary search
def wn_vec = if (T==i8) 8 else 2*256/width{T} def wn_vec = if (T==i8) 8 else 2*256/width{T}
if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) { if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) {
bin_search_vec{T, ...param, wn_vec} bin_search_vec{if (up) '⍋' else '⍒', T, ...slice{param,1}, wn_vec}
# Lookup table threshold has to account for cost of # Lookup table threshold has to account for cost of
# populating the table (proportional to wn until it's large), and # populating the table (proportional to wn until it's large), and
# initializing the table (constant, much higher for i16) # initializing the table (constant, much higher for i16)

View File

@ -0,0 +1,108 @@
local include 'skin/cext'
local include './cbqnDefs' # talloc/tfree
# Search primitives
# Function params are passed as names to keep generated code readable
def names = tup{'memberOf', 'count', 'indexOf'}
def prims = tup{'∊', '⊒', '⊐' }
def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v}
def to_prim = map{names, prims, .}
# Defined in C
def memset{p:pT, v, l} = {
emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l}
}
# These hashes are stored in tables and must be invertible!
# Murmur3
def hash_val{x0:u32} = {
x := x0
x ^= x >> 16; x *= 0x85ebca6b
x ^= x >> 13; x *= 0xc2b2ae35
x ^= x >> 16; x
}
def hash_val{x0:u64} = {
x := x0
x ^= x >> 33; x *= 0xff51afd7ed558ccd
x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53
x ^= x >> 33; x
}
# CRC32
if (hasarch{'SSE4.2'}) require{'x86intrin.h'}
def hash_val{x:u32 & hasarch{'SSE4.2'}} = {
emit{u32, '_mm_crc32_u32', 0x973afb51, x}
}
# Allocate and initialize resizing hash table
# Initial size sz+ext and maximum size msz+ext
# One region for each type in Ts initialized with value from v0s
# Allocates the maximum at the start, resizes downwards within the existing allocation
def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
def ws = each{width,Ts}
def wt = tupsel{0,ws}
each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment
# Variables updated on resize
sz := usz~~1 << logsz
sh := wt - logsz
def add{}=0; def add{a,...r} = a+add{...r}
halloc := talloc{u8, (msz+ext)*add{...ws/8}}
szo := msz-sz # Beginning of allocation to initial table
sze := sz+ext # Initial table to end of allocation
def pe{{}} = halloc; def pl{{}} = tup{}
def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far
def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} }
ptrs := pl{Ts}
def memset{_, ('any'), _} = {} # Indicates initialization not needed
each{memset{., ., sze}, ptrs, v0s}
def hash_resize{cc, m} = {
dif := sz*((1<<m)-1) # Number of elements of space to add
sh -= m; sz <<= m
set_thresh{}
cc = 0 # Collision counter
k:tupsel{0,Ts} = 0; --k # Index where to move an element to
each{{p,v} => { p -= dif; memset{p, v, dif} }, ptrs, v0s}
def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s
s := sz+ext
i := dif
# Iterate over existing elements in blocks of e<=32
# Branchlessly skips over h0
while (i < s) {
e := min{s-i, usz~~32}
b:u32 = 0
@for_backwards (h in hash+i over e) b = 2*b + promote{u32, h!=h0}
while (b!=0) {
j := i + cast_i{usz,ctz{b}}; b &= b-1
h := hash->j
hash <-{j} h0
k0 := h>>sh
if (ordered) { k = max{k0, k+1} } # k0 can't be less than k
else { k = k0; while (hash->k!=h0) ++k }
cc += cast_i{ux, k-k0}
hash <-{k} h
def move{p,v0} = {
v := p->j; if (not same{v0,'any'}) p <-{j} v0; p <-{k} v
}
each{move, vals, v0r}
}
i += e
}
}
# Test for resize if more than 1/2^t collisions/element
t:usz = 0 # Shift amount
def div_thresh{i} = i>>t
# Threshold setter, re-applied on resize
def set_thresh{} = {
if (sz==msz) t = 0
else if ((not has_radix) and sz>=(1<<24)/wt) t = 0
else if ( sz>=(1<<20)/wt) t = 3
else t = 5
}
tup{set_thresh, div_thresh}
set_thresh{}
tup{ptrs, sz, sh, div_thresh, hash_resize, {}=>tfree{halloc}}
}

View File

@ -1,6 +1,7 @@
include './base' include './base'
include './mask' include './mask'
include './vecfold' include './vecfold'
include './hashtab'
def findFirst{C, M, F, ...v1} = { def findFirst{C, M, F, ...v1} = {
def exit = makelabel{} def exit = makelabel{}
@ -329,3 +330,322 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
} }
exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}}
# Hash tables
oper &- ({v:T,m} => v & -promote{T,m}) infix left 35
def rty{name} = if (to_prim{name}=='∊') i8 else i32
def ity{name} = (to_prim{name}=='⊒')**(*u32)
fn hashtab{T, name}(rp:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = {
# iv,mi/ip,m - searched-in; fv,ni/fp,n - searched-for; may get swapped around & back
def prim = to_prim{name}
def U = if (prim=='∊') usz else u32
m := cast_i{U,mi}; n := cast_i{U,ni}
def wt = width{T}
ip := *T~~iv; fp := *T~~fv
def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}}
swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4)
if (swap) swap_sides{}
log := clzc{m}
# Max size
msl := max{clzc{(m+m/2)|4}+1, min{ux~~14,clzc{m+n/4}}}
msz := usz~~1 << msl
# Starting log-size (try_vec_memb requires size>4)
sl := msl; if (msl>=14) sl = 12+(msl&1)
b:U = 64 # Block size
# Filling e slots past the end requires e*(e+1)/2 collisions, so
# m entries with <2 each can fill <sqrt(4*m)
def cc_stop = 2*cast_i{u64,m}
ext := promote{usz, tern{m<=b, max{m,U~~4}, b + (U~~1 << (log/2 + 1))}}
maxh := T~~maxvalue{T}
def aux = prim!='∊'
def {tabs, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{
sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**'any'}, 0, 1
}
def {hash,...vals} = tabs
def abort = makelabel{}
i:U = 0 # Saved to determine if hashing finished
def insert_all{set_tab, set_maxh, dup, ...uniq} = {
cc:u64 = 0 # Collision counter
while (i < m) {
e := tern{m-i>b, i+b, m}
while (i < e) {
def ii = if (prim!='⊒') i else m-i-1
h := hash_val{load{ip,ii}}; j := h>>sh
set_maxh{h==maxh, i, j}
kv := each{load{.,j}, tabs}; def {k,...kr} = kv
# Robin Hood insertion
j0 := j; je := j # Save value; end of chain (insert at j)
if (k != maxh) {
if (k == h) goto{dup}
do {
++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv
def c = promote{T, h >= k}
j += c
if (kn == h) goto{dup}
each{store{.,je-c,.}, tabs, kv}
each{=, kv, knv}
} while (k != maxh)
cc += cast_i{u64, je-j0}
}
each{{u} => { u += promote{usz,h!=maxh} }, uniq}
store{hash, j, h}
set_tab{j, h}
++i
}
# Check collision counter and possibly resize
def p64 = promote{u64,.}
dc := p64{cc} - p64{div_thresh{i}}
if (tern{i<m, i64~~dc>=0, sz<msz}) {
if (sz == msz) goto{abort}
rdc := p64{m-i}*dc # 0 if i==m, no need to recompute
mm := p64{i}*p64{m+i}
def recheck = setlabel{}
if (cc>=cc_stop or p64{n/4}*p64{cc} + rdc >= mm>>(5+log-(wt-sh))) {
hash_resize{cc, 2} # Factor of 4
if (i==m and sz<msz) goto{recheck}
if (cc >= cc_stop) { i=0; goto{abort} }
}
}
}
}
def get_end{} = {
end := maxh>>sh
while (load{hash,end}!=maxh) ++end
end
}
def sequester_maxh{j, found} = { j += cast_i{T, ext &- found} }
def unsequester_maxh{tab} = {
store{tab, get_end{}, load{tab, maxh>>sh + cast_i{T,ext}}}
}
def hash_remove{j,h} = {
do {
jp:=j; ++j
h=load{hash,j}
if (h>>sh == j) h = maxh
store{hash, jp, h}
each{{t} => store{t, jp, load{t,j}}, vals}
} while (h!=maxh)
}
def memb_remove{uniq, has_maxh} = {
@for (ip over m) {
h := hash_val{ip}; j := h>>sh
def shortcut = makelabel{}
if (h == maxh) {
if (uniq==0) goto{shortcut}
has_maxh = 0
} else {
k := load{hash,j}
if (k <= h) {
while (k < h) { ++j; k = load{hash,j} }
if (k == h) {
--uniq
if (uniq==0 and not has_maxh) {
setlabel{shortcut}
@for (rp over n) rp = 1
goto{abort}
}
hash_remove{j,h}
}
}
}
}
}
def ind_rev_lookup{uniq, has_maxh} = {
def shortcut = makelabel{}
@for (ip over i to m) {
h := hash_val{ip}; j := h>>sh
k := load{hash,j}
if (k <= h) {
while (k < h) { ++j; k = load{hash,j} }
def had_maxh{} = { s:=has_maxh; has_maxh=0; s }
if (k == h and (h<maxh or had_maxh{})) {
def {inds} = vals
store{rp, load{inds, j}, i}
--uniq; if (uniq==0) goto{shortcut}
hash_remove{j,h}
}
}
}
setlabel{shortcut}
}
def prog_lookup{swap} = { # Progressive Index-of lookup
def rev{a,b} = if (swap) tup{b,a} else tup{a,b}
memset{*u32~~rp, ...rev{m,n}}
c := m; def shortcut = makelabel{}
@for (fp over i to n) {
h := hash_val{fp}; j := h>>sh
k := load{hash,j}
if (k <= h) {
while (k < h) { ++j; k = load{hash,j} }
if (k==h) {
def {inds} = vals; def {link} = links
ti := load{inds, j}
if (ti > 0) {
store{rp, ...rev{i, m-ti}}
--c; if (c==0) goto{shortcut}
ti = load{link, ti}
if (ti > 0 or h==maxh) store{inds, j, ti}
else hash_remove{j,h}
}
}
}
}
setlabel{shortcut}
}
def lookup_all{get_res} = {
@for (rp, fp over n) {
h := hash_val{fp}; j := h>>sh
k := undefined{T}; while ((k=load{hash,j}) < h) ++j
rp = get_res{k==h, j}
}
}
match (prim, ...vals, ...links) {
{('∊')} => {
has_maxh:u1 = 0
uniq:usz = 0 # Uniques inserted
def dup = makelabel{}
insert_all{
{j, h} => setlabel{dup},
{found, i, j} => has_maxh |= found,
dup, uniq
}
if (swap) {
swap_sides{}; i=m # i==m return value is kind of dumb
memb_remove{uniq,has_maxh} # Remove values in ip from hash
}
try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort}
end := get_end{} # Clip trailing maxh if it shouldn't be in the table
if (has_maxh) ++end # Don't clip
lookup_all{{found, j} => promote{i8, swap ^ (found & (j<end))}}
}
{('⊐'), inds} => if (not swap) {
has_maxh:u1 = 0
ind_maxh:u32 = 0
def dup = makelabel{}
def set{j, h} = {
store{inds, j, m-i} # So it can be cleared with one &- in get{}
setlabel{dup}
}
def set_maxh{found, i, j} = {
ind_maxh |= i &- (found&~has_maxh)
has_maxh |= found
}
insert_all{set, set_maxh, dup}
store{inds, get_end{}, (m-ind_maxh) &- has_maxh}
lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))}
} else { # swap
# After insert_all, position i in rp contains:
# - ≠𝕨, if i is the first occurrence of its value in 𝕩, or
# - j-≠𝕩, where j<i is the index of the first occurrence
uniq:usz = 0
has_maxh:u1 = 0
ri:i32 = 0 # Placed in rp (should be scoped to insert_all loop body)
def dup = makelabel{}
def set_maxh{found, i, j} = {
sequester_maxh{j, found}
ri = cast_i{i32, n} # Initialize to not-found
if (found & has_maxh) goto{dup}
has_maxh |= found
}
def set{j, h} = {
store{inds, j, i}
if (u1~~0) {
setlabel{dup}
ri = i32~~(load{inds, j} - m)
}
store{rp, i, ri}
}
insert_all{set, set_maxh, dup, uniq}
# Lookup places correct result index at each first occurrence
swap_sides{}; i=m
if (has_maxh) { ++uniq; unsequester_maxh{inds} }
ind_rev_lookup{uniq, has_maxh}
# Propagate to later occurrences
@for (r in rp over i to n) r = load{rp, min{i, u32~~r+n}}
}
{('⊒'), inds, link} => {
store{link,0,0}
store{inds, maxh>>sh + cast_i{T,ext}, 0}
def dup = makelabel{}
def set{j, h} = {
store{inds, j, load{inds,j} &- (h==maxh)}
setlabel{dup}
i1 := i+1
store{link, i1, load{inds,j}}
store{inds, j, i1}
}
def set_maxh{found, i, j} = sequester_maxh{j, found}
insert_all{set, set_maxh, dup}
unsequester_maxh{inds}
if (not swap) prog_lookup{0} else prog_lookup{1}
}
}
setlabel{abort}
hash_free{}
i == m # Whether it finished
}
def try_vec_memb{..._} = {}
def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
& hasarch{'SSE4.2'} & T==u32} = {
# Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints)
# i-h>>sh = i+((1<<sh-1)-h)>>sh = (((i+1)<<sh-1)-h)>>sh
# We maintain io = (i+1)<<sh-1
def vl = 4; def V = [vl]T
assert{sz > 4}
assert{sz%vl == 0}
io := make{V, each{{k} => T~~k<<sh - 1, 1+iota{vl}}}
id := V**(T~~vl<<sh)
mv := V**0
@for (h in *V~~hash over sz/vl) { mv=max{mv, io-min{h,io}}; io+=id }
max_off := vfold{max, mv} >> sh
# sz==1<<sh, so when i>=sz the above overflows
# And we have to handle maxh specially anyway
def mx{i,h} = { max_off = max{max_off, i-h>>sh} }
i:T=sz; h:T=i; while ((h=load{hash,i})<maxh) { mx{i,h}; ++i }
if (has_maxh) mx{i,maxh}
vswap := base{256,vl**1} * promote{T,swap}
def memb{test} = {
def R = i8; def rw = width{R}; def u = width{T}/rw
l := n/u
@for (r in *T~~rp over i to l) {
c := V**0 # Will combine u results to avoid folding too much
@unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f}
r = vswap ^ vfold{|, c}
}
@for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}}
goto{done}
}
def try{nv} = {
if (max_off < nv*vl) {
# Avoid matching maxh if it shouldn't be in the table
clear := V**(maxh &- ~has_maxh)
@for (hv in *V~~(hash + maxh>>sh) over vl) hv = hv &~ (hv == clear)
# Test against nv vectors
def test{x} = {
h := hash_val{x}; vh := V**h
def any{{...r,a}} = any{r}|a; def any{{a}}=a
any{@collect (k in *V~~(hash+h>>sh) over nv) vh == k}
}
memb{test}
}
}
each{try, tup{1,2}}
}
def exp_hash{name} = {
exportT{merge{'si_',name,'_c2_hash'}, each{hashtab{., name}, tup{u32,u64}}}
}
each{exp_hash, names}

View File

@ -0,0 +1,86 @@
include './base'
local include 'skin/cext'
include './hashtab'
# Resizing hash table, with fallback
def rty{name} = if (to_prim{name}=='∊') i8 else i32
fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = {
def wt = width{T}
def prim = to_prim{name}
def has_radix = if (prim=='⊐') 0 else wt==32
n64 := promote{u64,n}
def {res0,cc_stop} = if (prim=='∊') tup{1,n64>>has_radix} else tup{0,2*n64}
log := clzc{n}
# Max size
msl := clzc{n+n/2} + 1; if (has_radix and msl>20) msl=20
msz := usz~~1 << msl
# Starting log-size
sl := msl; if (msl>=14) sl = 12+(msl&1)
b:usz = 64 # Block size
# Filling e slots past the end requires e*(e+1)/2 collisions, so
# n entries with <2 each can fill <sqrt(4*n)
ext := tern{n<=b, n, b + (usz~~1 << (log/2 + 1))}
x0 := hash_val{xp->0}
rp <-{0} res0
def aux = prim!='∊'
def {{hash,...vals}, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{
sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, has_radix, 0
}
def {output, write_res} = match (prim) {
{('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }}
{('⊒')} => {
ctr0:u32 = 1; def {val} = vals
def res{j,h,k,x0} = {
e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0
hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0
vj + (c0 & -e0)
}
tup{{b}=>b, res}
}
{('⊐')} => {
ctr:u32 = 1; def {val} = vals
def res{j,h,k,x0} = {
if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h }
val->j
}
tup{{b}=>ctr & -promote{u32,b}, res}
}
}
def break=makelabel{}
cc:u64 = 0 # Collision counter
i:usz=1; while (1) {
e := tern{n-i>b, i+b, n}
while (i < e) {
h := hash_val{xp->i}; j0 := h>>sh; j := j0
k:=undefined{T}; while (((k=hash->j)!=h) & (k!=x0)) ++j
cc += cast_i{u64, j-j0}
rp <-{i} write_res{j,h,k,x0}
++i
}
if (i == n) goto{break}
# Check collision counter and possibly resize
def p64 = promote{u64,.}
dc := cc - p64{div_thresh{i}}
if (i64~~dc >= 0) {
if (sz == msz) goto{break} # Abort
if (has_radix and i < n/2 and sz >= 1<<18) goto{break}
# Avoid resizing if close to the end
if (cc>=cc_stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) {
hash_resize{cc, 2} # Factor of 4
if (cc >= cc_stop) goto{break}
}
}
}
setlabel{break}
hash_free{}
output{i == n} # Whether it finished, and unique count if ⊐
}
def exp_hash{T, name} = {
export{merge{name,'_c1_hash',fmtnat{width{T}}}, selfhashtab{T, name}}
}
each{{n}=>each{exp_hash{.,n},tup{u32,u64}}, names}

View File

@ -1,5 +1,4 @@
# Fold associative/commutative operation across a register # Fold associative/commutative operation across a register
# Used by squeeze.singeli, count.singeli
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
c:= x c:= x