diff --git a/build/src/build.bqn b/build/src/build.bqn index b03687f3..7b21fbcd 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -648,11 +648,11 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/search.c"‿"search", "xag"‿"src/builtins/selfsearch.c"‿"selfsearch" + "xag"‿"src/builtins/scan.c"‿"scan", "xa."‿"src/builtins/fold.c"‿"fold", "xag"‿"src/builtins/sort.c"‿"bins" - "2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan", - "2a."‿"src/builtins/slash.c"‿"constrep", + "2.."‿"src/builtins/select.c"‿"select", "2a."‿"src/builtins/slash.c"‿"constrep", "xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/src/builtins/search.c b/src/builtins/search.c index 1c618af3..5c6f9a23 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -1,29 +1,49 @@ // Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒) -// 𝕨⊐unit or unit∊𝕩: scalar loop with early-exit -// SHOULD use simd -// SHOULD unify implementations -// 𝕩⊒unit or 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩 - -// Both arguments with rank≥1: -// High-rank inputs: -// Convert to a (lower-rank) typed integer array if cells are ≤62 bits -// COULD have special hashing for equal type >64 bit cells, skipping squeezing -// COULD try conditionally squeezing ahead-of-time, and not squeezing in bqn_hash -// p⊐n & n∊p with short p & long n: n⊸=¨ p -// bitarr⊐𝕩: more special arithmetic -// SHOULD have impls for long p & short n -// ≤16-bit elements: lookup tables -// Character elements: reinterpret as integer elements -// Otherwise, generic hashtable -// SHOULD handle up to 64 bit cells via proper typed hash tables -// SHOULD have fast path when cell sizes or element types doesn't match +// 𝕨⊐unit or unit∊𝕩: SIMD shortcutting search +// 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩 +// High-rank inputs: +// Convert to a typed numeric list if cells are ≤62 bits +// COULD have hashing for equal-type >64 bit cells, to skip squeezing +// COULD try squeezing ahead-of-time to skip it in bqn_hash +// SHOULD have fast path when cell sizes don't match +// One input empty: fast not-found +// Character elements: +// Character versus number array is fast not-found for ∊ and ⊐ +// SHOULD have fast character-number path for ⊒ +// Reinterpret as integer elements +// COULD try p=⌜n when all arguments are short (may not be faster?) +// p⊐n & n∊p with short n: p⊸⊐¨n +// p⊐n & n∊p with boolean p: based on ⊑p and p⊐¬⊑p +// p⊐n & n∊p with short p: +// AVX2 binary search when applicable +// SHOULD apply vector binary search to characters (sort as ints) +// n⊸=¨p otherwise +// ≤16-bit elements: lookup tables +// 8-bit ∊ and ⊐: SSSE3 table +// SHOULD make 8-bit NEON table +// SHOULD have branchy reverse 16-bit table search +// 32- or 64-bit elements: hash tables +// Store hash in table and not element; Robin Hood ordering; resizing +// Reverse hash if searched-for is shorter +// Shortcutting for reverse hashes and non-reversed ⊒ +// SIMD lookup for 32-bit ∊ if chain length is small enough +// SHOULD partition if hash table size gets large +// SHOULD handle unequal search types better +// Otherwise, generic hashtable #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" #include "../utils/calls.h" +extern NOINLINE void memset16(u16* p, u16 v, usz l) { for (usz i=0; ixe) x=taga(cpyC16Arr(x)); \ - else w=taga(cpyC16Arr(w)); \ - we = xe = el_i16; \ - goto tyEls; \ - } \ - we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \ -} +#define CHECK_CHRS_ELSE \ + if (!elNum(we)) { \ + if (elChr(we)) { \ + if (elNum(xe)) goto none_found; \ + if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \ + if (we>xe) x=taga(cpyC16Arr(x)); \ + else w=taga(cpyC16Arr(w)); \ + we = xe = el_i16; \ + goto tyEls; \ + } \ + we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \ + } \ + } else if (!elNum(xe)) { \ + if (elChr(xe)) goto none_found; \ + } else B indexOf_c2(B t, B w, B x) { + bool split = 0; (void) split; if (RARE(!isArr(w) || RNK(w)!=1)) { + split = 1; B2 t = splitCells(x, w, 1); w = t.p; x = t.n; @@ -218,10 +262,23 @@ B indexOf_c2(B t, B w, B x) { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); if (wia==0 || xia==0) { - decG(w); return i64EachDec(0, x); + none_found: + decG(w); return i64EachDec(wia, x); } - - if (elNum(we) && elNum(xe)) { tyEls: + + CHECK_CHRS_ELSE { tyEls: // Both numbers + if (wia>32 && xia<=(we<=el_i8?1:3)) { + SGetU(x); + B r; + #define IND(T) \ + T* rp; r = m_##T##arrc(&rp, x); \ + for (usz i=0; i16) { + #if SINGELI_AVX2 + if (xia>=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<(xe==el_i8?64:16) && we<=xe && !elChr(TI(x,elType))) { + B g = C1(reverse, C1(gradeDown, incG(w))); + w = C2(select, incG(g), w); + switch (xe) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; } + i8* rp; B r = m_i8arrc(&rp, x); + avx2_indexOf_sort[xe-el_i8](rp, tyany_ptr(w), wia, tyany_ptr(x), xia); + r = C2(select, r, C2(join, g, m_i8(wia))); + decG(w); decG(x); return r; + } + #endif + if (wia<=4 && xia>16) { SGetU(w); #define XEQ(I) C2(ne, GetU(w,I), incG(x)) B r = XEQ(wia-1); @@ -241,23 +309,37 @@ B indexOf_c2(B t, B w, B x) { } if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { + B r; #if SINGELI if (wia>256 && we==el_i8 && xe==el_i8) { TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256); void* fp = tyany_ptr(x); simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind); decG(w); - i32* rp; B r = m_i32arrc(&rp, x); - for (usz i=0; i32 && wia<=(xe<=el_i8?1:xe==el_i32?4:6)) { + SGetU(w); + i8* rp; r = m_i8arrc(&rp, w); + for (usz i=0; ixe?we:xe; - if (xia<=(me==el_i8?1:me==el_i16?4:16) && wia>16) { + #if SINGELI_AVX2 + if (wia>=32>>(we-el_i8) && xia>1 && ((we==el_i16 && xia<32) || (we==el_i32 && xia<16)) && xe<=we && !elChr(TI(x,elType))) { + x = C1(and, x); // sort + if (xe16) { SGetU(x); r = WEQ(GetU(x,0)); for (usz i=1; i -static inline u32 hash32(u32 x) { return _mm_crc32_u32(0x973afb51, x); } +// From search.c +extern NOINLINE void memset32(u32* p, u32 v, usz l); +extern NOINLINE void memset64(u64* p, u64 v, usz l); + +#if SINGELI + #define SINGELI_FILE selfsearch + #include "../utils/includeSingeli.h" + + #define TRY_HASHTAB_RET(NAME, W, RET) \ + if (NAME##_c1_hash##W(rp, (u##W*)xv, n)) { decG(x); return RET; } #else -// Murmur3 -static inline u32 hash32(u32 x) { - x ^= x >> 16; x *= 0x85ebca6b; - x ^= x >> 13; x *= 0xc2b2ae35; - x ^= x >> 16; - return x; -} + #define TRY_HASHTAB_RET(NAME, W, RET) (void)0; #endif -static inline u64 hash64(u64 x) { - x ^= x >> 33; x *= 0xff51afd7ed558ccd; - x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53; - x ^= x >> 33; - return x; -} static inline bool use_sorted(B x, u8 logw) { if (!FL_HAS(x, fl_asc|fl_dsc)) return 0; @@ -146,73 +140,6 @@ u8 radix_offsets_2_usz(usz* c0, u32* v0, usz n) { } \ decG(x); TFREE(alloc); -static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i20) msl=20; \ - usz sh = W - (msl<14? msl : 12+(msl&1)); /* Shift to fit to table */ \ - usz sz = 1 << (W - sh); /* Initial size */ \ - usz msz = 1ull << msl; /* Max sz */ \ - usz b = 64; /* Block size */ \ - /* Resize or abort if more than 1/2^thresh collisions/element */ \ - usz thresh = THRESH; \ - /* Filling e slots past the end requires e*(e+1)/2 collisions, so */ \ - /* n entries with <2 each can fill b? i+b : n; \ - for (; i < e; i++) { \ - T h = hash##W(xp[i]); T j0 = h>>sh, j = j0; T k; \ - while (k=hash[j], k!=h & k!=x0) j++; \ - cc += j-j0; \ - RESWRITE \ - } \ - if (i == n) break; \ - i64 dc = (i64)cc - ((THRESHMUL*i)>>thresh); \ - if (dc >= 0) { \ - if (sz == msz || (RAD && i < n/2 && sz >= 1<<18)) break; /*Abort*/ \ - /* Avoid resizing if close to the end */ \ - if (cc>(5+log-(W-sh))) continue;\ - /* Resize hash, factor of 4 */ \ - usz m = 2; \ - usz dif = sz*((1<>sh, k = k0; while (hash[k]!=x0) k++; \ - cc += k-k0; \ - hash[k] = h; AUXMOVE \ - } \ - if (cc >= STOP) break; \ - thresh = THRESH; \ - } \ - } \ - TFREE(halloc); \ - if (i==n) { decG(x); return RESULT; } -#define SELFHASHTAB_VAL(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, INIT) \ - SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, \ - /*AUXSIZE*/sizeof(u32), \ - /* AUXINIT */ \ - u32* val = (u32*)(hash+sz+ext) + msz-sz; \ - memset32(val, 0, sz+ext); \ - INIT , \ - /*AUXEXTEND*/val -= dif; memset32(val, 0, dif); , \ - /*AUXMOVE*/u32 v = val[j]; val[j] = 0; val[k] = v;) - extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*); extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*); @@ -268,14 +195,11 @@ B memberOf_c1(B t, B x) { } } if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - #define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB( \ - T, W, RAD, STOP, \ - 1, taga(cpyBitArr(r)), hash[j]=h; rp[i]=k!=h;, \ - 1, THRESH, 0,,,) + #define TRY_HASHTAB(W) TRY_HASHTAB_RET(memberOf, W, taga(cpyBitArr(r))) if (lw==5) { if (n<12) { BRUTE(32); } i8* rp; B r = m_i8arrv(&rp, n); - HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<15)? 3 : 5) + TRY_HASHTAB(32) // Radix-assisted lookup when hash table gives up usz rx = 256, tn = 1<<16; // Radix; table length @@ -301,10 +225,10 @@ B memberOf_c1(B t, B x) { if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<20) { BRUTE(64); } i8* rp; B r = m_i8arrv(&rp, n); - HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE if (RNK(x)>1) { @@ -377,18 +301,11 @@ B count_c1(B t, B x) { if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - #define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB_VAL( \ - T, W, RAD, STOP, \ - /*RES0*/0, /*RETURN*/num_squeeze(r), \ - /* RESWRITE */ \ - bool e0=h==x0; rp[i]=val[j]+(ctr0&-(u32)e0); \ - hash[j]=h; val[j]+=!e0; ctr0+=e0; , \ - /*THRESHMUL*/1, THRESH, \ - /*INIT*/u32 ctr0 = 1;) + #define TRY_HASHTAB(W) TRY_HASHTAB_RET(count, W, num_squeeze(r)) if (lw==5) { if (n<20) { BRUTE(32); } i32* rp; B r = m_i32arrv(&rp, n); - HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(32) // Radix-assisted lookup usz rx = 256, tn = 1<<16; // Radix; table length u32* v0 = (u32*)xv; @@ -413,10 +330,10 @@ B count_c1(B t, B x) { if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<20) { BRUTE(64); } i32* rp; B r = m_i32arrv(&rp, n); - HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE if (RNK(x)>1) { @@ -485,18 +402,19 @@ B indexOf_c1(B t, B x) { if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - - #define HASHTAB(T, W, THRESH) SELFHASHTAB_VAL(T, W, 0, 2*n, \ - /*RES0*/0, /*RETURN*/reduceI32WidthBelow(r, ctr), \ - /* RESWRITE */ \ - if (k!=h) { val[j]=ctr++; hash[j]=h; } rp[i]=val[j]; , \ - /*THRESHMUL*/2, THRESH, \ - /*INIT*/u32 ctr = 1;) + + #if SINGELI + #define TRY_HASHTAB(W) \ + u32 ctr = indexOf_c1_hash##W(rp, (u##W*)xv, n); \ + if (ctr>0) { decG(x); return reduceI32WidthBelow(r, ctr); } + #else + #define TRY_HASHTAB(W) (void)0; + #endif if (lw==5) { if (n<12) { BRUTE(32); } B r; i32* rp; r = m_i32arrv(&rp, n); - i32* xp = tyany_ptr(x); + i32* xp = xv; i32 min=I32_MAX, max=I32_MIN; for (usz i = 0; i < n; i++) { i32 c = xp[i]; @@ -512,17 +430,16 @@ B indexOf_c1(B t, B x) { decG(x); return reduceI32WidthBelow(r, u); } - HASHTAB(u32, 32, sz==msz? 0 : sz>=(1<<18)? 1 : sz>=(1<<14)? 4 : 6) + TRY_HASHTAB(32) decG(r); // Fall through } if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<16) { BRUTE(64); } i32* rp; B r = m_i32arrv(&rp, n); - u64* xp = tyany_ptr(x); - HASHTAB(u64, 64, sz==msz? 0 : sz>=(1<<17)? 1 : sz>=(1<<13)? 4 : 6) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE #undef DOTAB diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index c0e5ed4c..2ed5648f 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -200,7 +200,9 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { } # Binary search within vector registers -def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { +def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { + def up = prim != '⍒' + def search = (prim=='∊') | (prim=='⊐') assert{wn > 1}; assert{wn < maxwn} def wd = width{T} def I = if (wd<32) u8 else u32; def wi = width{I} @@ -214,7 +216,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { log := ceil_log2{wn+1} gap := 1<=1) ms{wv}{1} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' # Offset at end - off := U~~V**cast_i{i8, gap-1} + off := U~~V**cast_i{i8, gap-(1-search)} # Midpoint bits for each step def lowbits = bb{copy{isub,isub}} bits := each{{j} => U**(lowbits << j), iota{lstep}} @@ -252,33 +254,61 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { if (this) @for_vec_overlap{vl} (j to xn) { xv:= load{*V~~(x+j), 0} s := U**bb{iota{isub}} # Select sequential bytes within each U - def ltx{se,ind} = lt{xv, V~~se{re_el{I,ind}}} + def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}} + def ltx = cmpx{lt}; def eqx = cmpx{==} @unroll (j to klog) { m := s | tupsel{klog-1-j,bits} s = homBlend{m, s, ltx{selw, m}} } r := if (isub==1) s else s>>(lb{isub}+wd-wi) + # b records if xv was found; c is added to the index + def r_out = prim!='∊' + def get_up{var,cmpx,use}{set,...a} = if (use) set{var, cmpx{...a}} + b := undefined{U}; def up_b = get_up{b, eqx, search} + c := undefined{U}; def up_c = get_up{c, ltx, r_out} + up_b{=, selw, s} # Extra selection lanes if (last and ex>=1 and log>=klog+1) { r += r - c := ltx{selw1,s} + def up_bc{set,se} = { up_b{|=,se,s}; up_c{set,se,s} } + up_bc{=,selw1} if (ex>=2 and log>=klog+2) { r += r - each{{se} => c += ltx{se,s}, selw2} + each{up_bc{+=, .}, selw2} } - r += c + if (r_out) r += c + } + if (r_out) { + r -= off + if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b} + rn := if (T==i8) r + else if (T==i16) half{narrow{u8, r}, 0} + else extract{re_el{i64, narrow{u8, r}}, 0} + rnp := *type{rn}~~(*i8~~rp+j) + if (isvec{type{rn}}) store{rnp, 0, rn} + else storeu{rnp, rn} + } else { + def B = ty_u{vl}; out := cast_i{B, homMask{b}} + store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)} } - r -= off - rn := if (T==i8) r - else if (T==i16) half{narrow{u8, r}, 0} - else extract{re_el{i64, narrow{u8, r}}, 0} - rnp := *type{rn}~~(*i8~~rp+j) - if (isvec{type{rn}}) store{rnp, 0, rn} - else storeu{rnp, rn} } } } +if (hasarch{'AVX2'}) { + fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = { + bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn} + } + exportT{ + 'avx2_member_sort', + each{avx2_search_bin{'∊',.,.}, tup{i16,i32}, tup{32,16}} + } + exportT{ + 'avx2_indexOf_sort', + each{avx2_search_bin{'⊐',.,.}, tup{i8,i16,i32}, tup{64,16,16}} + } +} + def unroll_sizes = tup{4,1} fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = { each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs} @@ -321,7 +351,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { # For >=8 i8 values, vector bit-table is as good as binary search def wn_vec = if (T==i8) 8 else 2*256/width{T} if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) { - bin_search_vec{T, ...param, wn_vec} + bin_search_vec{if (up) '⍋' else '⍒', T, ...slice{param,1}, wn_vec} # Lookup table threshold has to account for cost of # populating the table (proportional to wn until it's large), and # initializing the table (constant, much higher for i16) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli new file mode 100644 index 00000000..411a8d5d --- /dev/null +++ b/src/singeli/src/hashtab.singeli @@ -0,0 +1,108 @@ +local include 'skin/cext' +local include './cbqnDefs' # talloc/tfree + +# Search primitives +# Function params are passed as names to keep generated code readable +def names = tup{'memberOf', 'count', 'indexOf'} +def prims = tup{'∊', '⊒', '⊐' } +def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} +def to_prim = map{names, prims, .} + +# Defined in C +def memset{p:pT, v, l} = { + emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l} +} + +# These hashes are stored in tables and must be invertible! +# Murmur3 +def hash_val{x0:u32} = { + x := x0 + x ^= x >> 16; x *= 0x85ebca6b + x ^= x >> 13; x *= 0xc2b2ae35 + x ^= x >> 16; x +} +def hash_val{x0:u64} = { + x := x0 + x ^= x >> 33; x *= 0xff51afd7ed558ccd + x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53 + x ^= x >> 33; x +} +# CRC32 +if (hasarch{'SSE4.2'}) require{'x86intrin.h'} +def hash_val{x:u32 & hasarch{'SSE4.2'}} = { + emit{u32, '_mm_crc32_u32', 0x973afb51, x} +} + +# Allocate and initialize resizing hash table +# Initial size sz+ext and maximum size msz+ext +# One region for each type in Ts initialized with value from v0s +# Allocates the maximum at the start, resizes downwards within the existing allocation +def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { + def ws = each{width,Ts} + def wt = tupsel{0,ws} + each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment + # Variables updated on resize + sz := usz~~1 << logsz + sh := wt - logsz + + def add{}=0; def add{a,...r} = a+add{...r} + halloc := talloc{u8, (msz+ext)*add{...ws/8}} + szo := msz-sz # Beginning of allocation to initial table + sze := sz+ext # Initial table to end of allocation + def pe{{}} = halloc; def pl{{}} = tup{} + def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far + def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } + ptrs := pl{Ts} + def memset{_, ('any'), _} = {} # Indicates initialization not needed + each{memset{., ., sze}, ptrs, v0s} + + def hash_resize{cc, m} = { + dif := sz*((1< { p -= dif; memset{p, v, dif} }, ptrs, v0s} + def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s + s := sz+ext + i := dif + + # Iterate over existing elements in blocks of e<=32 + # Branchlessly skips over h0 + while (i < s) { + e := min{s-i, usz~~32} + b:u32 = 0 + @for_backwards (h in hash+i over e) b = 2*b + promote{u32, h!=h0} + while (b!=0) { + j := i + cast_i{usz,ctz{b}}; b &= b-1 + h := hash->j + hash <-{j} h0 + k0 := h>>sh + if (ordered) { k = max{k0, k+1} } # k0 can't be less than k + else { k = k0; while (hash->k!=h0) ++k } + cc += cast_i{ux, k-k0} + hash <-{k} h + def move{p,v0} = { + v := p->j; if (not same{v0,'any'}) p <-{j} v0; p <-{k} v + } + each{move, vals, v0r} + } + i += e + } + } + + # Test for resize if more than 1/2^t collisions/element + t:usz = 0 # Shift amount + def div_thresh{i} = i>>t + # Threshold setter, re-applied on resize + def set_thresh{} = { + if (sz==msz) t = 0 + else if ((not has_radix) and sz>=(1<<24)/wt) t = 0 + else if ( sz>=(1<<20)/wt) t = 3 + else t = 5 + } + tup{set_thresh, div_thresh} + set_thresh{} + + tup{ptrs, sz, sh, div_thresh, hash_resize, {}=>tfree{halloc}} +} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 21ef5e2d..8b44ec61 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -1,6 +1,7 @@ include './base' include './mask' include './vecfold' +include './hashtab' def findFirst{C, M, F, ...v1} = { def exit = makelabel{} @@ -329,3 +330,322 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { } exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} + + +# Hash tables +oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 + +def rty{name} = if (to_prim{name}=='∊') i8 else i32 +def ity{name} = (to_prim{name}=='⊒')**(*u32) +fn hashtab{T, name}(rp:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = { + # iv,mi/ip,m - searched-in; fv,ni/fp,n - searched-for; may get swapped around & back + def prim = to_prim{name} + def U = if (prim=='∊') usz else u32 + m := cast_i{U,mi}; n := cast_i{U,ni} + def wt = width{T} + ip := *T~~iv; fp := *T~~fv + def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} + swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4) + if (swap) swap_sides{} + + log := clzc{m} + # Max size + msl := max{clzc{(m+m/2)|4}+1, min{ux~~14,clzc{m+n/4}}} + msz := usz~~1 << msl + # Starting log-size (try_vec_memb requires size>4) + sl := msl; if (msl>=14) sl = 12+(msl&1) + b:U = 64 # Block size + + # Filling e slots past the end requires e*(e+1)/2 collisions, so + # m entries with <2 each can fill b, i+b, m} + while (i < e) { + def ii = if (prim!='⊒') i else m-i-1 + h := hash_val{load{ip,ii}}; j := h>>sh + set_maxh{h==maxh, i, j} + kv := each{load{.,j}, tabs}; def {k,...kr} = kv + # Robin Hood insertion + j0 := j; je := j # Save value; end of chain (insert at j) + if (k != maxh) { + if (k == h) goto{dup} + do { + ++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv + def c = promote{T, h >= k} + j += c + if (kn == h) goto{dup} + each{store{.,je-c,.}, tabs, kv} + each{=, kv, knv} + } while (k != maxh) + cc += cast_i{u64, je-j0} + } + each{{u} => { u += promote{usz,h!=maxh} }, uniq} + store{hash, j, h} + set_tab{j, h} + ++i + } + # Check collision counter and possibly resize + def p64 = promote{u64,.} + dc := p64{cc} - p64{div_thresh{i}} + if (tern{i=0, sz=cc_stop or p64{n/4}*p64{cc} + rdc >= mm>>(5+log-(wt-sh))) { + hash_resize{cc, 2} # Factor of 4 + if (i==m and sz= cc_stop) { i=0; goto{abort} } + } + } + } + } + def get_end{} = { + end := maxh>>sh + while (load{hash,end}!=maxh) ++end + end + } + def sequester_maxh{j, found} = { j += cast_i{T, ext &- found} } + def unsequester_maxh{tab} = { + store{tab, get_end{}, load{tab, maxh>>sh + cast_i{T,ext}}} + } + + def hash_remove{j,h} = { + do { + jp:=j; ++j + h=load{hash,j} + if (h>>sh == j) h = maxh + store{hash, jp, h} + each{{t} => store{t, jp, load{t,j}}, vals} + } while (h!=maxh) + } + def memb_remove{uniq, has_maxh} = { + @for (ip over m) { + h := hash_val{ip}; j := h>>sh + def shortcut = makelabel{} + if (h == maxh) { + if (uniq==0) goto{shortcut} + has_maxh = 0 + } else { + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + if (k == h) { + --uniq + if (uniq==0 and not has_maxh) { + setlabel{shortcut} + @for (rp over n) rp = 1 + goto{abort} + } + hash_remove{j,h} + } + } + } + } + } + def ind_rev_lookup{uniq, has_maxh} = { + def shortcut = makelabel{} + @for (ip over i to m) { + h := hash_val{ip}; j := h>>sh + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + def had_maxh{} = { s:=has_maxh; has_maxh=0; s } + if (k == h and (h>sh + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + if (k==h) { + def {inds} = vals; def {link} = links + ti := load{inds, j} + if (ti > 0) { + store{rp, ...rev{i, m-ti}} + --c; if (c==0) goto{shortcut} + ti = load{link, ti} + if (ti > 0 or h==maxh) store{inds, j, ti} + else hash_remove{j,h} + } + } + } + } + setlabel{shortcut} + } + + def lookup_all{get_res} = { + @for (rp, fp over n) { + h := hash_val{fp}; j := h>>sh + k := undefined{T}; while ((k=load{hash,j}) < h) ++j + rp = get_res{k==h, j} + } + } + + match (prim, ...vals, ...links) { + {('∊')} => { + has_maxh:u1 = 0 + uniq:usz = 0 # Uniques inserted + def dup = makelabel{} + insert_all{ + {j, h} => setlabel{dup}, + {found, i, j} => has_maxh |= found, + dup, uniq + } + if (swap) { + swap_sides{}; i=m # i==m return value is kind of dumb + memb_remove{uniq,has_maxh} # Remove values in ip from hash + } + try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} + end := get_end{} # Clip trailing maxh if it shouldn't be in the table + if (has_maxh) ++end # Don't clip + lookup_all{{found, j} => promote{i8, swap ^ (found & (j if (not swap) { + has_maxh:u1 = 0 + ind_maxh:u32 = 0 + def dup = makelabel{} + def set{j, h} = { + store{inds, j, m-i} # So it can be cleared with one &- in get{} + setlabel{dup} + } + def set_maxh{found, i, j} = { + ind_maxh |= i &- (found&~has_maxh) + has_maxh |= found + } + insert_all{set, set_maxh, dup} + store{inds, get_end{}, (m-ind_maxh) &- has_maxh} + lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))} + } else { # swap + # After insert_all, position i in rp contains: + # - ≠𝕨, if i is the first occurrence of its value in 𝕩, or + # - j-≠𝕩, where j { + store{link,0,0} + store{inds, maxh>>sh + cast_i{T,ext}, 0} + def dup = makelabel{} + def set{j, h} = { + store{inds, j, load{inds,j} &- (h==maxh)} + setlabel{dup} + i1 := i+1 + store{link, i1, load{inds,j}} + store{inds, j, i1} + } + def set_maxh{found, i, j} = sequester_maxh{j, found} + insert_all{set, set_maxh, dup} + unsequester_maxh{inds} + if (not swap) prog_lookup{0} else prog_lookup{1} + } + } + + setlabel{abort} + hash_free{} + i == m # Whether it finished +} + +def try_vec_memb{..._} = {} +def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done + & hasarch{'SSE4.2'} & T==u32} = { + # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) + # i-h>>sh = i+((1<>sh = (((i+1)<>sh + # We maintain io = (i+1)< 4} + assert{sz%vl == 0} + io := make{V, each{{k} => T~~k<> sh + # sz==1<=sz the above overflows + # And we have to handle maxh specially anyway + def mx{i,h} = { max_off = max{max_off, i-h>>sh} } + i:T=sz; h:T=i; while ((h=load{hash,i})>sh) over vl) hv = hv &~ (hv == clear) + # Test against nv vectors + def test{x} = { + h := hash_val{x}; vh := V**h + def any{{...r,a}} = any{r}|a; def any{{a}}=a + any{@collect (k in *V~~(hash+h>>sh) over nv) vh == k} + } + memb{test} + } + } + each{try, tup{1,2}} +} + +def exp_hash{name} = { + exportT{merge{'si_',name,'_c2_hash'}, each{hashtab{., name}, tup{u32,u64}}} +} +each{exp_hash, names} diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli new file mode 100644 index 00000000..fd789b63 --- /dev/null +++ b/src/singeli/src/selfsearch.singeli @@ -0,0 +1,86 @@ +include './base' +local include 'skin/cext' +include './hashtab' + +# Resizing hash table, with fallback +def rty{name} = if (to_prim{name}=='∊') i8 else i32 +fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { + def wt = width{T} + def prim = to_prim{name} + def has_radix = if (prim=='⊐') 0 else wt==32 + n64 := promote{u64,n} + def {res0,cc_stop} = if (prim=='∊') tup{1,n64>>has_radix} else tup{0,2*n64} + + log := clzc{n} + # Max size + msl := clzc{n+n/2} + 1; if (has_radix and msl>20) msl=20 + msz := usz~~1 << msl + # Starting log-size + sl := msl; if (msl>=14) sl = 12+(msl&1) + b:usz = 64 # Block size + + # Filling e slots past the end requires e*(e+1)/2 collisions, so + # n entries with <2 each can fill 0} + rp <-{0} res0 + def aux = prim!='∊' + def {{hash,...vals}, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{ + sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, has_radix, 0 + } + + def {output, write_res} = match (prim) { + {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} + {('⊒')} => { + ctr0:u32 = 1; def {val} = vals + def res{j,h,k,x0} = { + e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0 + hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0 + vj + (c0 & -e0) + } + tup{{b}=>b, res} + } + {('⊐')} => { + ctr:u32 = 1; def {val} = vals + def res{j,h,k,x0} = { + if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h } + val->j + } + tup{{b}=>ctr & -promote{u32,b}, res} + } + } + + def break=makelabel{} + cc:u64 = 0 # Collision counter + i:usz=1; while (1) { + e := tern{n-i>b, i+b, n} + while (i < e) { + h := hash_val{xp->i}; j0 := h>>sh; j := j0 + k:=undefined{T}; while (((k=hash->j)!=h) & (k!=x0)) ++j + cc += cast_i{u64, j-j0} + rp <-{i} write_res{j,h,k,x0} + ++i + } + if (i == n) goto{break} + # Check collision counter and possibly resize + def p64 = promote{u64,.} + dc := cc - p64{div_thresh{i}} + if (i64~~dc >= 0) { + if (sz == msz) goto{break} # Abort + if (has_radix and i < n/2 and sz >= 1<<18) goto{break} + # Avoid resizing if close to the end + if (cc>=cc_stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) { + hash_resize{cc, 2} # Factor of 4 + if (cc >= cc_stop) goto{break} + } + } + } + setlabel{break} + hash_free{} + output{i == n} # Whether it finished, and unique count if ⊐ +} + +def exp_hash{T, name} = { + export{merge{name,'_c1_hash',fmtnat{width{T}}}, selfhashtab{T, name}} +} +each{{n}=>each{exp_hash{.,n},tup{u32,u64}}, names} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index f8b85b26..f0aadea2 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -1,5 +1,4 @@ # Fold associative/commutative operation across a register -# Used by squeeze.singeli, count.singeli def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { c:= x