From 0fb845d33652146072a4ccbc8d762dda766d4c6e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 30 Oct 2023 19:25:05 -0400 Subject: [PATCH 01/40] Move self-search hash table code to Singeli --- build/src/build.bqn | 6 +- src/builtins/selfsearch.c | 130 +++++++-------------------- src/singeli/src/selfsearch.singeli | 138 +++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 100 deletions(-) create mode 100644 src/singeli/src/selfsearch.singeli diff --git a/build/src/build.bqn b/build/src/build.bqn index b03687f3..7b21fbcd 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -648,11 +648,11 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/search.c"‿"search", "xag"‿"src/builtins/selfsearch.c"‿"selfsearch" + "xag"‿"src/builtins/scan.c"‿"scan", "xa."‿"src/builtins/fold.c"‿"fold", "xag"‿"src/builtins/sort.c"‿"bins" - "2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan", - "2a."‿"src/builtins/slash.c"‿"constrep", + "2.."‿"src/builtins/select.c"‿"select", "2a."‿"src/builtins/slash.c"‿"constrep", "xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 95bb32c7..8c5ed6bc 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -63,6 +63,19 @@ static inline u64 hash64(u64 x) { return x; } +static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i20) msl=20; \ - usz sh = W - (msl<14? msl : 12+(msl&1)); /* Shift to fit to table */ \ - usz sz = 1 << (W - sh); /* Initial size */ \ - usz msz = 1ull << msl; /* Max sz */ \ - usz b = 64; /* Block size */ \ - /* Resize or abort if more than 1/2^thresh collisions/element */ \ - usz thresh = THRESH; \ - /* Filling e slots past the end requires e*(e+1)/2 collisions, so */ \ - /* n entries with <2 each can fill b? i+b : n; \ - for (; i < e; i++) { \ - T h = hash##W(xp[i]); T j0 = h>>sh, j = j0; T k; \ - while (k=hash[j], k!=h & k!=x0) j++; \ - cc += j-j0; \ - RESWRITE \ - } \ - if (i == n) break; \ - i64 dc = (i64)cc - ((THRESHMUL*i)>>thresh); \ - if (dc >= 0) { \ - if (sz == msz || (RAD && i < n/2 && sz >= 1<<18)) break; /*Abort*/ \ - /* Avoid resizing if close to the end */ \ - if (cc>(5+log-(W-sh))) continue;\ - /* Resize hash, factor of 4 */ \ - usz m = 2; \ - usz dif = sz*((1<>sh, k = k0; while (hash[k]!=x0) k++; \ - cc += k-k0; \ - hash[k] = h; AUXMOVE \ - } \ - if (cc >= STOP) break; \ - thresh = THRESH; \ - } \ - } \ - TFREE(halloc); \ - if (i==n) { decG(x); return RESULT; } -#define SELFHASHTAB_VAL(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, INIT) \ - SELFHASHTAB(T, W, RAD, STOP, RES0, RESULT, RESWRITE, THRESHMUL, THRESH, \ - /*AUXSIZE*/sizeof(u32), \ - /* AUXINIT */ \ - u32* val = (u32*)(hash+sz+ext) + msz-sz; \ - memset32(val, 0, sz+ext); \ - INIT , \ - /*AUXEXTEND*/val -= dif; memset32(val, 0, dif); , \ - /*AUXMOVE*/u32 v = val[j]; val[j] = 0; val[k] = v;) - extern void (*const simd_mark_firsts_u8)(void*,uint64_t,void*,void*); extern u64 (*const simd_deduplicate_u8)(void*,uint64_t,void*,void*); @@ -268,14 +214,11 @@ B memberOf_c1(B t, B x) { } } if (lw==4) { if (n<8) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - #define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB( \ - T, W, RAD, STOP, \ - 1, taga(cpyBitArr(r)), hash[j]=h; rp[i]=k!=h;, \ - 1, THRESH, 0,,,) + #define TRY_HASHTAB(W) TRY_HASHTAB_RET(memberOf, W, taga(cpyBitArr(r))) if (lw==5) { if (n<12) { BRUTE(32); } i8* rp; B r = m_i8arrv(&rp, n); - HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<15)? 3 : 5) + TRY_HASHTAB(32) // Radix-assisted lookup when hash table gives up usz rx = 256, tn = 1<<16; // Radix; table length @@ -301,10 +244,10 @@ B memberOf_c1(B t, B x) { if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<20) { BRUTE(64); } i8* rp; B r = m_i8arrv(&rp, n); - HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE if (RNK(x)>1) { @@ -377,18 +320,11 @@ B count_c1(B t, B x) { if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - #define HASHTAB(T, W, RAD, STOP, THRESH) T* xp = (T*)xv; SELFHASHTAB_VAL( \ - T, W, RAD, STOP, \ - /*RES0*/0, /*RETURN*/num_squeeze(r), \ - /* RESWRITE */ \ - bool e0=h==x0; rp[i]=val[j]+(ctr0&-(u32)e0); \ - hash[j]=h; val[j]+=!e0; ctr0+=e0; , \ - /*THRESHMUL*/1, THRESH, \ - /*INIT*/u32 ctr0 = 1;) + #define TRY_HASHTAB(W) TRY_HASHTAB_RET(count, W, num_squeeze(r)) if (lw==5) { if (n<20) { BRUTE(32); } i32* rp; B r = m_i32arrv(&rp, n); - HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(32) // Radix-assisted lookup usz rx = 256, tn = 1<<16; // Radix; table length u32* v0 = (u32*)xv; @@ -413,10 +349,10 @@ B count_c1(B t, B x) { if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<20) { BRUTE(64); } i32* rp; B r = m_i32arrv(&rp, n); - HASHTAB(u64, 64, 0, n, sz==msz? 0 : sz>=(1<<18)? 0 : sz>=(1<<14)? 3 : 5) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE if (RNK(x)>1) { @@ -485,18 +421,19 @@ B indexOf_c1(B t, B x) { if (lw==3) { if (n<12) { BRUTE(8); } else { LOOKUP(8); } } if (lw==4) { if (n<12) { BRUTE(16); } else { LOOKUP(16); } } #undef LOOKUP - - #define HASHTAB(T, W, THRESH) SELFHASHTAB_VAL(T, W, 0, 2*n, \ - /*RES0*/0, /*RETURN*/reduceI32WidthBelow(r, ctr), \ - /* RESWRITE */ \ - if (k!=h) { val[j]=ctr++; hash[j]=h; } rp[i]=val[j]; , \ - /*THRESHMUL*/2, THRESH, \ - /*INIT*/u32 ctr = 1;) + + #if SINGELI + #define TRY_HASHTAB(W) \ + u32 ctr = indexOf_c1_hash##W(rp, (u##W*)xv, n); \ + if (ctr>0) { decG(x); return reduceI32WidthBelow(r, ctr); } + #else + #define TRY_HASHTAB(W) (void)0; + #endif if (lw==5) { if (n<12) { BRUTE(32); } B r; i32* rp; r = m_i32arrv(&rp, n); - i32* xp = tyany_ptr(x); + i32* xp = xv; i32 min=I32_MAX, max=I32_MIN; for (usz i = 0; i < n; i++) { i32 c = xp[i]; @@ -512,17 +449,16 @@ B indexOf_c1(B t, B x) { decG(x); return reduceI32WidthBelow(r, u); } - HASHTAB(u32, 32, sz==msz? 0 : sz>=(1<<18)? 1 : sz>=(1<<14)? 4 : 6) + TRY_HASHTAB(32) decG(r); // Fall through } if (lw==6 && canCompare64_norm(&x, &xv, n)) { if (n<16) { BRUTE(64); } i32* rp; B r = m_i32arrv(&rp, n); - u64* xp = tyany_ptr(x); - HASHTAB(u64, 64, sz==msz? 0 : sz>=(1<<17)? 1 : sz>=(1<<13)? 4 : 6) + TRY_HASHTAB(64) decG(r); // Fall through } - #undef HASHTAB + #undef TRY_HASHTAB #undef BRUTE #undef DOTAB diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli new file mode 100644 index 00000000..8b959ee0 --- /dev/null +++ b/src/singeli/src/selfsearch.singeli @@ -0,0 +1,138 @@ +include './base' +include './cbqnDefs' +local include 'skin/cext' + +# Size-specific functions defined in C +def hash_val{v:T} = emit{T, merge{'hash',fmtnat{width{T}}}, v} +def memset{p:pT, v, l} = { + emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l} +} + +def names = tup{'memberOf', 'count', 'indexOf'} +def prims = tup{'∊', '⊒', '⊐' } +def map{f,t,v} = if (v==tupsel{0,f}) tupsel{0,t} else map{slice{f,1}, slice{t,1}, v} +def to_prim = map{names, prims, .} + +# Resizing hash table, with fallback +def rty{name} = if (to_prim{name}=='∊') i8 else i32 +fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { + def wt = width{T} + def prim = to_prim{name} + def has_radix = if (prim=='⊐') 0 else wt==32 + def {res0,stop} = if (prim=='∊') tup{1,n>>has_radix} else tup{0,2*n} + + log := clzc{n} + msl := clzc{n+n/2} + 1; if (has_radix and msl>20) msl=20 + sh := wt - tern{msl<14, msl, 12+(msl&1)} # Shift to fit to table + sz := usz~~1 << (wt - sh) # Initial size + msz := usz~~1 << msl # Max sz + b:usz = 64 # Block size + + # Resize or abort if more than 1/2^thresh collisions/element + def {set_thresh, div_thresh} = { + t:usz = 0 # Shift amount + def o = prim=='⊐' # Offset to allow effective t<0 + def div_thresh{i} = ((1<>t + def t0= has_radix*(prim=='∊') + def w = if (prim=='⊒') 1<<6 else wt<=(1<<24)/w) t = 0+o + else if ( sz>=(1<<20)/w) t = 3+o + else t = 5+o + } + tup{set_thresh, div_thresh} + } + set_thresh{} + + # Filling e slots past the end requires e*(e+1)/2 collisions, so + # n entries with <2 each can fill 0}; rp <-{0} res0 + def aux = prim!='∊' + halloc := talloc{u8, (msz+ext)*((wt + aux*width{u32})/8)} + szo := msz-sz # Beginning of allocation to initial table + sze := sz+ext # Initial table to end of allocation + hash := *T~~halloc + szo + memset{hash, x0, sze} + val := *u32~~(hash+sze) + szo + if (aux) { memset{val, 0, sze} } + + def {output, write_res} = match (prim) { + {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} + {('⊒')} => { + ctr0:u32 = 1 + tup{ + {b}=>b, + {j,h,k,x0} => { + e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0 + hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0 + vj + (c0 & -e0) + } + } + } + {('⊐')} => { + ctr:u32 = 1 + tup{ + {b}=>promote{u32,b}*ctr, + {j,h,k,x0} => { + if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h } + val->j + } + } + } + } + + def break=makelabel{} + cc:usz = 0 # Collision counter + i:usz=1; while (1) { + e := tern{n-i>b, i+b, n} + while (i < e) { + h := hash_val{xp->i}; j0 := h>>sh; j := j0 + k:=undefined{T}; while (((k=hash->j)!=h) & (k!=x0)) ++j + cc += cast_i{usz, j-j0} + rp <-{i} write_res{j,h,k,x0} + ++i + } + if (i == n) goto{break} + def p64{a} = i64~~promote{u64,a} + dc := p64{cc} - p64{div_thresh{i}} + if (dc >= 0) { + if (sz == msz) goto{break} # Abort + if (has_radix and i < n/2 and sz >= 1<<18) goto{break} + # Avoid resizing if close to the end + if (not (cc>(5+log-(wt-sh)))) { + # Resize hash, factor of 4 + def m = 2 + dif := sz*((1<j + if (h != x0) { + hash <-{j} x0 + k0 := h>>sh; k := k0; while (hash->k!=x0) ++k + cc += cast_i{usz, k-k0} + hash <-{k} h + if (aux) { v := val->j; val <-{j} 0; val <-{k} v } + } + } + if (cc >= stop) goto{break} + set_thresh{} + } + } + } + setlabel{break} + tfree{halloc} + output{i == n} # Whether it finished +} + +def exp_hash{T, name} = { + export{merge{name,'_c1_hash',fmtnat{width{T}}}, selfhashtab{T, name}} +} +each{{n}=>each{exp_hash{.,n},tup{u32,u64}}, names} From 0391cf4ca30ba9b037e391df3b95858b71204007 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 30 Oct 2023 22:11:56 -0400 Subject: [PATCH 02/40] Fast number-character (no matches) searches --- src/builtins/search.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 1c618af3..3ca88d2e 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -191,15 +191,21 @@ static NOINLINE usz indexOfOne(B l, B e) { #endif } -#define CHR_TO_INT if (elChr(we) && elChr(xe)) { \ - if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \ - if (we>xe) x=taga(cpyC16Arr(x)); \ - else w=taga(cpyC16Arr(w)); \ - we = xe = el_i16; \ - goto tyEls; \ - } \ - we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \ -} +#define CHECK_CHRS_ELSE \ + if (!elNum(we)) { \ + if (elChr(we)) { \ + if (elNum(xe)) goto none_found; \ + if (we!=xe && we<=el_c16 && xe<=el_c16) { /* LUT uses signed integers, so needs equal-width args if we're gonna give unsigned ones */ \ + if (we>xe) x=taga(cpyC16Arr(x)); \ + else w=taga(cpyC16Arr(w)); \ + we = xe = el_i16; \ + goto tyEls; \ + } \ + we-=el_c8-el_i8; xe-=el_c8-el_i8; goto tyEls; \ + } \ + } else if (!elNum(xe)) { \ + if (elChr(xe)) goto none_found; \ + } else B indexOf_c2(B t, B w, B x) { if (RARE(!isArr(w) || RNK(w)!=1)) { @@ -218,10 +224,11 @@ B indexOf_c2(B t, B w, B x) { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); if (wia==0 || xia==0) { - decG(w); return i64EachDec(0, x); + none_found: + decG(w); return i64EachDec(wia, x); } - - if (elNum(we) && elNum(xe)) { tyEls: + + CHECK_CHRS_ELSE { tyEls: // Both numbers if (we==el_bit) { u64* wp = bitarr_ptr(w); u64 w0 = 1 & wp[0]; @@ -257,7 +264,7 @@ B indexOf_c2(B t, B w, B x) { TABLE(w, x, i32, wia, i) return reduceI32Width(r, wia); } - } else { CHR_TO_INT; } + } i32* rp; B r = m_i32arrc(&rp, x); H_b2i* map = m_b2i(64); @@ -302,10 +309,11 @@ B memberOf_c2(B t, B w, B x) { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); if (wia==0 || xia==0) { + none_found: decG(x); return i64EachDec(0, w); } - - if (elNum(we) && elNum(xe)) { tyEls: + + CHECK_CHRS_ELSE { tyEls: // Both numbers #define WEQ(V) C2(eq, incG(w), V) if (xe==el_bit) { u64* xp = bitarr_ptr(x); @@ -337,7 +345,7 @@ B memberOf_c2(B t, B w, B x) { TABLE(x, w, i8, 0, 1) return taga(cpyBitArr(r)); } - } else { CHR_TO_INT; } + } H_Sb* set = m_Sb(64); SGetU(x) SGetU(w) @@ -353,7 +361,7 @@ B memberOf_c2(B t, B w, B x) { decG(x); return r; } -#undef CHR_TO_INT +#undef CHECK_CHRS_ELSE B count_c2(B t, B w, B x) { if (RARE(!isArr(w) || RNK(w)!=1)) { From ee856a4880af3770e80e7acc79df79a6dce2a86a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 31 Oct 2023 12:16:51 -0400 Subject: [PATCH 03/40] Move hash resizing to a generator, and other refactoring --- src/builtins/selfsearch.c | 20 ------ src/singeli/src/selfsearch.singeli | 101 +++++++++++++++++------------ 2 files changed, 59 insertions(+), 62 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 8c5ed6bc..93de7f93 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -43,26 +43,6 @@ extern B mul_c2(B, B, B); extern B scan_add_bool(B x, u64 ia); extern B scan_max_num(B x, u8 xe, u64 ia); -// These hashes are stored in tables and must be invertible! -#if defined(__SSE4_2__) -#include -static inline u32 hash32(u32 x) { return _mm_crc32_u32(0x973afb51, x); } -#else -// Murmur3 -static inline u32 hash32(u32 x) { - x ^= x >> 16; x *= 0x85ebca6b; - x ^= x >> 13; x *= 0xc2b2ae35; - x ^= x >> 16; - return x; -} -#endif -static inline u64 hash64(u64 x) { - x ^= x >> 33; x *= 0xff51afd7ed558ccd; - x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53; - x ^= x >> 33; - return x; -} - static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i> 16; x *= 0x85ebca6b + x ^= x >> 13; x *= 0xc2b2ae35 + x ^= x >> 16; x +} +def hash_val{x0:u64} = { + x := x0 + x ^= x >> 33; x *= 0xff51afd7ed558ccd + x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53 + x ^= x >> 33; x +} +# CRC32 +def hash_val{x:u32 & hasarch{'SSE4.2'}} = { + emit{u32, '_mm_crc32_u32', 0x973afb51, x} +} + +def hash_resize{hash,vals, x0,sh,sz,ext,cc, m} = { + dif := sz*((1< { val -= dif; memset{val, 0, dif} }, vals} + @for (j from dif to sz + ext) { + h := hash->j + if (h != x0) { + hash <-{j} x0 + k0 := h>>sh; k := k0; while (hash->k!=x0) ++k + cc += cast_i{usz, k-k0} + hash <-{k} h + each{{val} => { v := val->j; val <-{j} 0; val <-{k} v }, vals} + } + } +} + def names = tup{'memberOf', 'count', 'indexOf'} def prims = tup{'∊', '⊒', '⊐' } -def map{f,t,v} = if (v==tupsel{0,f}) tupsel{0,t} else map{slice{f,1}, slice{t,1}, v} +def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} def to_prim = map{names, prims, .} # Resizing hash table, with fallback @@ -56,31 +94,27 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { sze := sz+ext # Initial table to end of allocation hash := *T~~halloc + szo memset{hash, x0, sze} - val := *u32~~(hash+sze) + szo - if (aux) { memset{val, 0, sze} } + vals := if (aux) tup{*u32~~(hash+sze) + szo} else tup{} + each{memset{., 0, sze}, vals} def {output, write_res} = match (prim) { {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} {('⊒')} => { - ctr0:u32 = 1 - tup{ - {b}=>b, - {j,h,k,x0} => { - e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0 - hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0 - vj + (c0 & -e0) - } + ctr0:u32 = 1; def {val} = vals + def res{j,h,k,x0} = { + e0:=promote{u32,h==x0}; vj:=val->j; c0:=ctr0 + hash<-{j}h; val<-{j}vj+(e0^1); ctr0+=e0 + vj + (c0 & -e0) } + tup{{b}=>b, res} } {('⊐')} => { - ctr:u32 = 1 - tup{ - {b}=>promote{u32,b}*ctr, - {j,h,k,x0} => { - if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h } - val->j - } + ctr:u32 = 1; def {val} = vals + def res{j,h,k,x0} = { + if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h } + val->j } + tup{{b}=>promote{u32,b}*ctr, res} } } @@ -96,32 +130,15 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { ++i } if (i == n) goto{break} - def p64{a} = i64~~promote{u64,a} + # Check collision counter and possibly resize + def p64 = promote{u64,.} dc := p64{cc} - p64{div_thresh{i}} - if (dc >= 0) { + if (i64~~dc >= 0) { if (sz == msz) goto{break} # Abort if (has_radix and i < n/2 and sz >= 1<<18) goto{break} # Avoid resizing if close to the end - if (not (cc>(5+log-(wt-sh)))) { - # Resize hash, factor of 4 - def m = 2 - dif := sz*((1<j - if (h != x0) { - hash <-{j} x0 - k0 := h>>sh; k := k0; while (hash->k!=x0) ++k - cc += cast_i{usz, k-k0} - hash <-{k} h - if (aux) { v := val->j; val <-{j} 0; val <-{k} v } - } - } + if (cc>=stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) { + hash_resize{hash,vals, x0,sh,sz,ext,cc, 2} # Resize hash, factor of 4 if (cc >= stop) goto{break} set_thresh{} } From cb8be206be6883b4407e8b34fd1ba47d4ead36f8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 31 Oct 2023 15:31:53 -0400 Subject: [PATCH 04/40] Move hash allocation to a generator --- src/singeli/src/selfsearch.singeli | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index d3b275a9..27d55b43 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -26,6 +26,24 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = { emit{u32, '_mm_crc32_u32', 0x973afb51, x} } +# Allocate and initialize resizing hash table +# Initial size sz+ext and maximum size msz+ext +# One region for each type in Ts initialized with value from v0s +def hash_alloc{sz, msz, ext, Ts, v0s} = { + def ws = each{width,Ts} + each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment + def add{}=0; def add{a,...r} = a+add{...r} + halloc := talloc{u8, (msz+ext)*add{...ws/8}} + szo := msz-sz # Beginning of allocation to initial table + sze := sz+ext # Initial table to end of allocation + def pe{{}} = halloc; def pl{{}} = tup{} + def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far + def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } + ptrs := pl{Ts} + each{memset{., ., sze}, ptrs, v0s} + tup{halloc, ptrs} +} + def hash_resize{hash,vals, x0,sh,sz,ext,cc, m} = { dif := sz*((1<0}; rp <-{0} res0 + x0 := hash_val{xp->0} + rp <-{0} res0 def aux = prim!='∊' - halloc := talloc{u8, (msz+ext)*((wt + aux*width{u32})/8)} - szo := msz-sz # Beginning of allocation to initial table - sze := sz+ext # Initial table to end of allocation - hash := *T~~halloc + szo - memset{hash, x0, sze} - vals := if (aux) tup{*u32~~(hash+sze) + szo} else tup{} - each{memset{., 0, sze}, vals} + def {halloc,{hash,...vals}} = hash_alloc{ + sz, msz, ext, merge{tup{T}, aux**u32}, merge{tup{x0}, aux**0} + } def {output, write_res} = match (prim) { {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} From ed0372044138ee0a8645a60ef1815b742c6d3679 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 1 Nov 2023 15:43:26 -0400 Subject: [PATCH 05/40] Robin Hood hash table for 32-bit membership --- src/builtins/search.c | 11 +++++ src/builtins/selfsearch.c | 4 +- src/singeli/src/hashmap.singeli | 64 ++++++++++++++++++++++++++ src/singeli/src/search.singeli | 71 +++++++++++++++++++++++++++++ src/singeli/src/selfsearch.singeli | 72 +++--------------------------- 5 files changed, 153 insertions(+), 69 deletions(-) create mode 100644 src/singeli/src/hashmap.singeli diff --git a/src/builtins/search.c b/src/builtins/search.c index 3ca88d2e..2c7a7563 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -24,6 +24,10 @@ #include "../utils/talloc.h" #include "../utils/calls.h" +// From selfsearch.c +extern NOINLINE void memset32(u32* p, u32 v, usz l); +extern NOINLINE void memset64(u64* p, u64 v, usz l); + RangeFn getRange_fns[el_f64+1]; #if SINGELI extern RangeFn* const simd_getRangeRaw; @@ -345,6 +349,13 @@ B memberOf_c2(B t, B w, B x) { TABLE(x, w, i8, 0, 1) return taga(cpyBitArr(r)); } + if (we==el_i32 && xe==el_i32) { + i8* rp; B r = m_i8arrc(&rp, w); + if (memberOf_c2_hash32(rp, tyany_ptr(x), xia, tyany_ptr(w), wia)) { + decG(w); decG(x); return taga(cpyBitArr(r)); + } + decG(r); + } } H_Sb* set = m_Sb(64); diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 93de7f93..c238d595 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -43,8 +43,8 @@ extern B mul_c2(B, B, B); extern B scan_add_bool(B x, u64 ia); extern B scan_max_num(B x, u8 xe, u64 ia); -static NOINLINE void memset32(u32* p, u32 v, usz l) { for (usz i=0; i> 16; x *= 0x85ebca6b + x ^= x >> 13; x *= 0xc2b2ae35 + x ^= x >> 16; x +} +def hash_val{x0:u64} = { + x := x0 + x ^= x >> 33; x *= 0xff51afd7ed558ccd + x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53 + x ^= x >> 33; x +} +# CRC32 +def hash_val{x:u32 & hasarch{'SSE4.2'}} = { + emit{u32, '_mm_crc32_u32', 0x973afb51, x} +} + +# Allocate and initialize resizing hash table +# Initial size sz+ext and maximum size msz+ext +# One region for each type in Ts initialized with value from v0s +def hash_alloc{sz, msz, ext, Ts, v0s} = { + def ws = each{width,Ts} + each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment + def add{}=0; def add{a,...r} = a+add{...r} + halloc := talloc{u8, (msz+ext)*add{...ws/8}} + szo := msz-sz # Beginning of allocation to initial table + sze := sz+ext # Initial table to end of allocation + def pe{{}} = halloc; def pl{{}} = tup{} + def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far + def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } + ptrs := pl{Ts} + each{memset{., ., sze}, ptrs, v0s} + tup{ptrs, {}=>tfree{halloc}} +} + +def hash_resize{hash,vals, x0,sh,sz,ext,cc, m} = { + dif := sz*((1< { val -= dif; memset{val, 0, dif} }, vals} + @for (j from dif to sz + ext) { + h := hash->j + if (h != x0) { + hash <-{j} x0 + k0 := h>>sh; k := k0; while (hash->k!=x0) ++k + cc += cast_i{usz, k-k0} + hash <-{k} h + each{{val} => { v := val->j; val <-{j} 0; val <-{k} v }, vals} + } + } +} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 21ef5e2d..e59f3adf 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -1,6 +1,7 @@ include './base' include './mask' include './vecfold' +include './hashmap' def findFirst{C, M, F, ...v1} = { def exit = makelabel{} @@ -329,3 +330,73 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { } exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} + + +# Hash tables +fn hashtab_memberOf{T}(rp:*i8, ip:*T, m:usz, fp:*T, n:usz) = { + def wt = width{T} + def stop = m + + log := clzc{m} + msl := clzc{m} + 2 + sh := wt - tern{msl<14, msl, 12+(msl&1)} # Shift to fit to table + sz := usz~~1 << (wt - sh) # Initial size + msz := usz~~1 << msl # Max sz + b:usz = 64 # Block size + + # Filling e slots past the end requires e*(e+1)/2 collisions, so + # m entries with <2 each can fill b, i+b, m} + while (i < e) { + h := hash_val{load{ip,i}}; j := h>>sh + k := load{hash, j} + # Robin Hood insertion + if (k == maxh) { + has_maxh |= h==maxh + store{hash, j, h} + } else if (k != h) { + j0 := j; ji := j # Save value; insertion point + do { + ++j; kn := load{hash,j}; if (kn==h) goto{dup} + def c = promote{T, h >= k} + store{hash, j-c, k} + ji += c + k = kn + } while (k != maxh) + store{hash, ji, h} + cc += cast_i{usz, j-j0} + setlabel{dup} + } + ++i + } + if (i == m) goto{lookup} + # Check collision counter and possibly resize (needs tuning) + if (cc >= stop and sz == msz) goto{abort} + if (16*cc >= stop and sz < msz) { + hash_resize{hash,vals, maxh,sh,sz,ext,cc, 2} # Resize hash, factor of 4 + if (cc >= stop) goto{abort} + } + } + setlabel{lookup} + end := maxh>>sh + promote{T,ext} # Unreachable + if (has_maxh) { end=maxh>>sh; while (load{hash,end}!=maxh) ++end } + @for (rp, fp over n) { + h := hash_val{fp}; j := h>>sh + k:=undefined{T}; while ((k=load{hash,j}) < h) ++j + rp = promote{i8, (k==h) & (j> 16; x *= 0x85ebca6b - x ^= x >> 13; x *= 0xc2b2ae35 - x ^= x >> 16; x -} -def hash_val{x0:u64} = { - x := x0 - x ^= x >> 33; x *= 0xff51afd7ed558ccd - x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53 - x ^= x >> 33; x -} -# CRC32 -def hash_val{x:u32 & hasarch{'SSE4.2'}} = { - emit{u32, '_mm_crc32_u32', 0x973afb51, x} -} - -# Allocate and initialize resizing hash table -# Initial size sz+ext and maximum size msz+ext -# One region for each type in Ts initialized with value from v0s -def hash_alloc{sz, msz, ext, Ts, v0s} = { - def ws = each{width,Ts} - each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment - def add{}=0; def add{a,...r} = a+add{...r} - halloc := talloc{u8, (msz+ext)*add{...ws/8}} - szo := msz-sz # Beginning of allocation to initial table - sze := sz+ext # Initial table to end of allocation - def pe{{}} = halloc; def pl{{}} = tup{} - def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far - def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } - ptrs := pl{Ts} - each{memset{., ., sze}, ptrs, v0s} - tup{halloc, ptrs} -} - -def hash_resize{hash,vals, x0,sh,sz,ext,cc, m} = { - dif := sz*((1< { val -= dif; memset{val, 0, dif} }, vals} - @for (j from dif to sz + ext) { - h := hash->j - if (h != x0) { - hash <-{j} x0 - k0 := h>>sh; k := k0; while (hash->k!=x0) ++k - cc += cast_i{usz, k-k0} - hash <-{k} h - each{{val} => { v := val->j; val <-{j} 0; val <-{k} v }, vals} - } - } -} +include './hashmap' def names = tup{'memberOf', 'count', 'indexOf'} def prims = tup{'∊', '⊒', '⊐' } @@ -108,8 +46,8 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { x0 := hash_val{xp->0} rp <-{0} res0 def aux = prim!='∊' - def {halloc,{hash,...vals}} = hash_alloc{ - sz, msz, ext, merge{tup{T}, aux**u32}, merge{tup{x0}, aux**0} + def {{hash,...vals}, hash_free} = hash_alloc{ + sz, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0} } def {output, write_res} = match (prim) { @@ -160,8 +98,8 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { } } setlabel{break} - tfree{halloc} - output{i == n} # Whether it finished + hash_free{} + output{i == n} # Whether it finished, and unique count if ⊐ } def exp_hash{T, name} = { From d54621b5dd1b4fac58164412ad058a3e1a222345 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 2 Nov 2023 12:00:32 -0400 Subject: [PATCH 06/40] Extend hash table to 32-bit Index-of --- src/builtins/search.c | 7 ++++ src/singeli/src/search.singeli | 59 +++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 2c7a7563..b264f4e3 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -268,6 +268,13 @@ B indexOf_c2(B t, B w, B x) { TABLE(w, x, i32, wia, i) return reduceI32Width(r, wia); } + if (we==el_i32 && xe==el_i32) { + i32* rp; B r = m_i32arrc(&rp, x); + if (indexOf_c2_hash32(rp, tyany_ptr(w), wia, tyany_ptr(x), xia)) { + decG(w); decG(x); return reduceI32Width(r, wia); + } + decG(r); + } } i32* rp; B r = m_i32arrc(&rp, x); diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index e59f3adf..b238fbf9 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -333,8 +333,17 @@ exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} # Hash tables -fn hashtab_memberOf{T}(rp:*i8, ip:*T, m:usz, fp:*T, n:usz) = { +def names = tup{'memberOf', 'count', 'indexOf'} +def prims = tup{'∊', '⊒', '⊐' } +def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} +def to_prim = map{names, prims, .} + +oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 + +def rty{name} = if (to_prim{name}=='∊') i8 else i32 +fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { def wt = width{T} + def prim = to_prim{name} def stop = m log := clzc{m} @@ -348,35 +357,39 @@ fn hashtab_memberOf{T}(rp:*i8, ip:*T, m:usz, fp:*T, n:usz) = { # m entries with <2 each can fill {mi:u32=0}, vals} i:usz=0; while (1) { e := tern{m-i>b, i+b, m} while (i < e) { h := hash_val{load{ip,i}}; j := h>>sh - k := load{hash, j} + each{{mi}=>{mi|=i&-((h==maxh)&~has_maxh)}, ind_maxh} + has_maxh |= h==maxh + kv := each{load{.,j}, tabs}; def {k,..._} = kv # Robin Hood insertion - if (k == maxh) { - has_maxh |= h==maxh - store{hash, j, h} - } else if (k != h) { - j0 := j; ji := j # Save value; insertion point + if (k != maxh) { + if (k == h) goto{dup} + j0 := j; je := j # Save value; end of chain (insert at j) do { - ++j; kn := load{hash,j}; if (kn==h) goto{dup} + ++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv + if (kn == h) goto{dup} def c = promote{T, h >= k} - store{hash, j-c, k} - ji += c - k = kn + each{store{.,je-c,.}, tabs, kv} + j += c + each{=, kv, knv} } while (k != maxh) - store{hash, ji, h} - cc += cast_i{usz, j-j0} - setlabel{dup} + cc += cast_i{usz, je-j0} } + each{store{., j, .}, tabs, tup{h,...aux**(m-i)}} + setlabel{dup} ++i } if (i == m) goto{lookup} @@ -389,14 +402,22 @@ fn hashtab_memberOf{T}(rp:*i8, ip:*T, m:usz, fp:*T, n:usz) = { } setlabel{lookup} end := maxh>>sh + promote{T,ext} # Unreachable - if (has_maxh) { end=maxh>>sh; while (load{hash,end}!=maxh) ++end } + if (has_maxh) { + end=maxh>>sh; while (load{hash,end}!=maxh) ++end + each{store{.,end,.}, vals, ind_maxh} + } @for (rp, fp over n) { h := hash_val{fp}; j := h>>sh k:=undefined{T}; while ((k=load{hash,j}) < h) ++j - rp = promote{i8, (k==h) & (jeach{exp_hash{.,n},tup{u32}}, tup{'memberOf', 'indexOf'}} From 64f2a10fbba5b64a4d3eb72599ae70829a095150 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 2 Nov 2023 12:55:13 -0400 Subject: [PATCH 07/40] Return hash_resize{} from hash_alloc{} --- src/singeli/src/hashmap.singeli | 42 ++++++++++++++++-------------- src/singeli/src/search.singeli | 13 ++++----- src/singeli/src/selfsearch.singeli | 29 +++++++++++---------- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/singeli/src/hashmap.singeli b/src/singeli/src/hashmap.singeli index 1301fb51..0e7ca77a 100644 --- a/src/singeli/src/hashmap.singeli +++ b/src/singeli/src/hashmap.singeli @@ -28,9 +28,13 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = { # Allocate and initialize resizing hash table # Initial size sz+ext and maximum size msz+ext # One region for each type in Ts initialized with value from v0s -def hash_alloc{sz, msz, ext, Ts, v0s} = { +def hash_alloc{logsz, msz, ext, Ts, v0s} = { def ws = each{width,Ts} each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment + # Variables updated on resize + sz := usz~~1 << logsz + sh := tupsel{0,ws} - logsz + def add{}=0; def add{a,...r} = a+add{...r} halloc := talloc{u8, (msz+ext)*add{...ws/8}} szo := msz-sz # Beginning of allocation to initial table @@ -40,25 +44,25 @@ def hash_alloc{sz, msz, ext, Ts, v0s} = { def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } ptrs := pl{Ts} each{memset{., ., sze}, ptrs, v0s} - tup{ptrs, {}=>tfree{halloc}} -} -def hash_resize{hash,vals, x0,sh,sz,ext,cc, m} = { - dif := sz*((1< { val -= dif; memset{val, 0, dif} }, vals} - @for (j from dif to sz + ext) { - h := hash->j - if (h != x0) { - hash <-{j} x0 - k0 := h>>sh; k := k0; while (hash->k!=x0) ++k - cc += cast_i{usz, k-k0} - hash <-{k} h - each{{val} => { v := val->j; val <-{j} 0; val <-{k} v }, vals} + def hash_resize{cc, m} = { + dif := sz*((1< { p -= dif; memset{p, v, dif} }, ptrs, v0s} + def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s + @for (j from dif to sz + ext) { + h := hash->j + if (h != h0) { + hash <-{j} h0 + k0 := h>>sh; k := k0; while (hash->k!=h0) ++k + cc += cast_i{usz, k-k0} + hash <-{k} h + each{{p,v0} => { v := p->j; p <-{j} v0; p <-{k} v }, vals, v0r} + } } } + + tup{ptrs, sz, sh, hash_resize, {}=>tfree{halloc}} } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index b238fbf9..36c9ff85 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -347,10 +347,11 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { def stop = m log := clzc{m} + # Max size msl := clzc{m} + 2 - sh := wt - tern{msl<14, msl, 12+(msl&1)} # Shift to fit to table - sz := usz~~1 << (wt - sh) # Initial size - msz := usz~~1 << msl # Max sz + msz := usz~~1 << msl + # Starting log-size + sl := msl; if (msl>=14) sl = 12+(msl&1) b:usz = 64 # Block size # Filling e slots past the end requires e*(e+1)/2 collisions, so @@ -358,8 +359,8 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { ext := tern{m<=b, m, b + (usz~~1 << (log/2 + 1))} maxh := T~~maxvalue{T} def aux = prim!='∊' - def {tabs, hash_free} = hash_alloc{ - sz, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**0} + def {tabs, sz, sh, hash_resize, hash_free} = hash_alloc{ + sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**0} } def {hash,...vals} = tabs @@ -396,7 +397,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { # Check collision counter and possibly resize (needs tuning) if (cc >= stop and sz == msz) goto{abort} if (16*cc >= stop and sz < msz) { - hash_resize{hash,vals, maxh,sh,sz,ext,cc, 2} # Resize hash, factor of 4 + hash_resize{cc, 2} # Factor of 4 if (cc >= stop) goto{abort} } } diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index 5600825c..37571407 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -16,12 +16,23 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { def {res0,stop} = if (prim=='∊') tup{1,n>>has_radix} else tup{0,2*n} log := clzc{n} + # Max size msl := clzc{n+n/2} + 1; if (has_radix and msl>20) msl=20 - sh := wt - tern{msl<14, msl, 12+(msl&1)} # Shift to fit to table - sz := usz~~1 << (wt - sh) # Initial size - msz := usz~~1 << msl # Max sz + msz := usz~~1 << msl + # Starting log-size + sl := msl; if (msl>=14) sl = 12+(msl&1) b:usz = 64 # Block size + # Filling e slots past the end requires e*(e+1)/2 collisions, so + # n entries with <2 each can fill 0} + rp <-{0} res0 + def aux = prim!='∊' + def {{hash,...vals}, sz, sh, hash_resize, hash_free} = hash_alloc{ + sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0} + } + # Resize or abort if more than 1/2^thresh collisions/element def {set_thresh, div_thresh} = { t:usz = 0 # Shift amount @@ -40,16 +51,6 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { } set_thresh{} - # Filling e slots past the end requires e*(e+1)/2 collisions, so - # n entries with <2 each can fill 0} - rp <-{0} res0 - def aux = prim!='∊' - def {{hash,...vals}, hash_free} = hash_alloc{ - sz, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0} - } - def {output, write_res} = match (prim) { {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} {('⊒')} => { @@ -91,7 +92,7 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { if (has_radix and i < n/2 and sz >= 1<<18) goto{break} # Avoid resizing if close to the end if (cc>=stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) { - hash_resize{hash,vals, x0,sh,sz,ext,cc, 2} # Resize hash, factor of 4 + hash_resize{cc, 2} # Factor of 4 if (cc >= stop) goto{break} set_thresh{} } From 8aa6c0719955be699fafb25175b53982aea57007 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 2 Nov 2023 20:29:54 -0400 Subject: [PATCH 08/40] And hashed 32-bit Progressive Index-of --- src/builtins/search.c | 5 ++++ src/singeli/src/search.singeli | 51 +++++++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index b264f4e3..5d6f3723 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -422,6 +422,10 @@ B count_c2(B t, B w, B x) { we-= el_c8-el_i8; xe-= el_c8-el_i8; goto el8or16; } else { + if (we==el_i32 && xe==el_i32 && + count_c2_hash32(rp, tyany_ptr(w), wia, tyany_ptr(x), xia, wnext)) { + goto dec_nwx; + } H_b2i* map = m_b2i(64); SGetU(x) SGetU(w) @@ -438,6 +442,7 @@ B count_c2(B t, B w, B x) { } free_b2i(map); } + dec_nwx:; TFREE(wnext); decG(w); decG(x); return reduceI32Width(r, wia); } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 36c9ff85..4f544160 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -341,7 +341,8 @@ def to_prim = map{names, prims, .} oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 def rty{name} = if (to_prim{name}=='∊') i8 else i32 -fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { +def ity{name} = (to_prim{name}=='⊒')**(*u32) +fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, ilink:ity{name}) = { def wt = width{T} def prim = to_prim{name} def stop = m @@ -364,33 +365,47 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { } def {hash,...vals} = tabs + each{store{.,0,0}, ilink} + def lookup=makelabel{}; def abort=makelabel{}; def dup=makelabel{} cc:usz = 0 # Collision counter has_maxh:u1 = 0 - def ind_maxh = each{{_}=>{mi:u32=0}, vals} + def ind_maxh = @collect (prim=='⊐') {mi:u32=0} i:usz=0; while (1) { e := tern{m-i>b, i+b, m} while (i < e) { - h := hash_val{load{ip,i}}; j := h>>sh + h := hash_val{load{ip,if (prim!='⊒') i else (m-i-1)}}; j := h>>sh each{{mi}=>{mi|=i&-((h==maxh)&~has_maxh)}, ind_maxh} has_maxh |= h==maxh - kv := each{load{.,j}, tabs}; def {k,..._} = kv + kv := each{load{.,j}, tabs}; def {k,...kr} = kv # Robin Hood insertion + j0 := j; je := j # Save value; end of chain (insert at j) if (k != maxh) { if (k == h) goto{dup} - j0 := j; je := j # Save value; end of chain (insert at j) do { ++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv - if (kn == h) goto{dup} def c = promote{T, h >= k} - each{store{.,je-c,.}, tabs, kv} j += c + if (kn == h) goto{dup} + each{store{.,je-c,.}, tabs, kv} each{=, kv, knv} } while (k != maxh) cc += cast_i{usz, je-j0} } - each{store{., j, .}, tabs, tup{h,...aux**(m-i)}} - setlabel{dup} + if (prim!='⊒') { + each{store{., j, .}, tabs, tup{h,...aux**(m-i)}} + setlabel{dup} + } else { + # maxh entry either stores 0 or is the last entry and stores an + # index, so this preserves all values + ++je; each{{p,v}=>store{p,je,v|load{p,je}}, vals, kr} + each{store{.,j,0}, vals} + store{hash, j, h} + setlabel{dup} + i1 := i+1 + each{{p,q} => store{p, i1, i1-load{q,j}}, ilink, vals} + each{store{., j, .}, vals, tup{i1}} + } ++i } if (i == m) goto{lookup} @@ -401,18 +416,28 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { if (cc >= stop) goto{abort} } } + setlabel{lookup} end := maxh>>sh + promote{T,ext} # Unreachable - if (has_maxh) { + if ((prim!='⊒') and has_maxh) { end=maxh>>sh; while (load{hash,end}!=maxh) ++end each{store{.,end,.}, vals, ind_maxh} } @for (rp, fp over n) { h := hash_val{fp}; j := h>>sh k:=undefined{T}; while ((k=load{hash,j}) < h) ++j - rp = (if (prim=='∊') promote{i8, (k==h) & (j promote{i8, (k==h) & (j i32~~(m - (load{tupsel{0,vals}, j} &- (k==h))) + {('⊒')} => { + def {inds} = vals + ti := load{inds, j}; mi := ti &- (k==h) + store{inds, j, ti - load{tupsel{0,ilink},mi}} + i32~~(m - mi) + } + }) } + setlabel{abort} hash_free{} i == m # Whether it finished @@ -421,4 +446,4 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz) = { def exp_hash{T, name} = { export{merge{name,'_c2_hash',fmtnat{width{T}}}, hashtab{T, name}} } -each{{n}=>each{exp_hash{.,n},tup{u32}}, tup{'memberOf', 'indexOf'}} +each{{n}=>each{exp_hash{.,n},tup{u32}}, names} From a9f33c1454c606410c0eb229f515e297b3c4a9b2 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 3 Nov 2023 16:10:44 -0400 Subject: [PATCH 09/40] Refactoring --- .../src/{hashmap.singeli => hashtab.singeli} | 7 ++ src/singeli/src/search.singeli | 108 +++++++++++------- src/singeli/src/selfsearch.singeli | 9 +- 3 files changed, 74 insertions(+), 50 deletions(-) rename src/singeli/src/{hashmap.singeli => hashtab.singeli} (87%) diff --git a/src/singeli/src/hashmap.singeli b/src/singeli/src/hashtab.singeli similarity index 87% rename from src/singeli/src/hashmap.singeli rename to src/singeli/src/hashtab.singeli index 0e7ca77a..f07e2a05 100644 --- a/src/singeli/src/hashmap.singeli +++ b/src/singeli/src/hashtab.singeli @@ -1,6 +1,13 @@ local include 'skin/cext' local include './cbqnDefs' # talloc/tfree +# Search primitives +# Function params are passed as names to keep generated code readable +def names = tup{'memberOf', 'count', 'indexOf'} +def prims = tup{'∊', '⊒', '⊐' } +def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} +def to_prim = map{names, prims, .} + # Defined in C def memset{p:pT, v, l} = { emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 4f544160..40b2e35c 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -1,7 +1,7 @@ include './base' include './mask' include './vecfold' -include './hashmap' +include './hashtab' def findFirst{C, M, F, ...v1} = { def exit = makelabel{} @@ -333,16 +333,11 @@ exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} # Hash tables -def names = tup{'memberOf', 'count', 'indexOf'} -def prims = tup{'∊', '⊒', '⊐' } -def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} -def to_prim = map{names, prims, .} - oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 def rty{name} = if (to_prim{name}=='∊') i8 else i32 def ity{name} = (to_prim{name}=='⊒')**(*u32) -fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, ilink:ity{name}) = { +fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) = { def wt = width{T} def prim = to_prim{name} def stop = m @@ -365,18 +360,70 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, ilink:ity{name}) } def {hash,...vals} = tabs - each{store{.,0,0}, ilink} + def {set_tab, set_maxh, fixup, get_res} = match (prim) { + {('∊')} => { + has_maxh:u1 = 0 + def set{j,je,kr,dup} = { setlabel{dup} } + def setm{found, i} = has_maxh |= found + end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table + def usem{} = { + if (has_maxh) { while (load{hash,end}!=maxh) ++end } + else end += promote{T,ext} # Unreachable + } + def get{found, j} = promote{i8, found & (j { + def {inds} = vals + has_maxh:u1 = 0 + ind_maxh:u32 = 0 + def set{j,je,kr,dup} = { + store{inds, j, m-i} # So it can be cleared with one &- in get{} + setlabel{dup} + } + def setm{found, i} = { + ind_maxh |= i &- (found&~has_maxh) + has_maxh |= found + } + def usem{} = { + if (has_maxh) { + end := maxh>>sh; while (load{hash,end}!=maxh) ++end + store{inds, end, ind_maxh} + } + } + def get{found, j} = i32~~(m - (load{inds, j} &- found)) + tup{set, setm, usem, get} + } + {('⊒')} => { + def {inds} = vals; def {link} = links + store{link,0,0} + def set{j,je,{kr},dup} = { + # maxh entry either stores 0 or is the last entry and stores an + # index, so this preserves all values + ++je; store{inds, je, kr|load{inds,je}} + store{inds, j, 0} + setlabel{dup} + i1 := i+1 + store{link, i1, i1-load{inds,j}} + store{inds, j, i1} + } + def get{found, j} = { + ti := load{inds, j}; mi := ti &- found + store{inds, j, ti - load{link,mi}} + i32~~(m - mi) + } + tup{set, {found,i}=>{}, {}=>{}, get} + } + } def lookup=makelabel{}; def abort=makelabel{}; def dup=makelabel{} cc:usz = 0 # Collision counter - has_maxh:u1 = 0 - def ind_maxh = @collect (prim=='⊐') {mi:u32=0} i:usz=0; while (1) { e := tern{m-i>b, i+b, m} while (i < e) { - h := hash_val{load{ip,if (prim!='⊒') i else (m-i-1)}}; j := h>>sh - each{{mi}=>{mi|=i&-((h==maxh)&~has_maxh)}, ind_maxh} - has_maxh |= h==maxh + def ii = if (prim!='⊒') i else m-i-1 + h := hash_val{load{ip,ii}}; j := h>>sh + set_maxh{h==maxh, i} kv := each{load{.,j}, tabs}; def {k,...kr} = kv # Robin Hood insertion j0 := j; je := j # Save value; end of chain (insert at j) @@ -392,20 +439,8 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, ilink:ity{name}) } while (k != maxh) cc += cast_i{usz, je-j0} } - if (prim!='⊒') { - each{store{., j, .}, tabs, tup{h,...aux**(m-i)}} - setlabel{dup} - } else { - # maxh entry either stores 0 or is the last entry and stores an - # index, so this preserves all values - ++je; each{{p,v}=>store{p,je,v|load{p,je}}, vals, kr} - each{store{.,j,0}, vals} - store{hash, j, h} - setlabel{dup} - i1 := i+1 - each{{p,q} => store{p, i1, i1-load{q,j}}, ilink, vals} - each{store{., j, .}, vals, tup{i1}} - } + store{hash, j, h} + set_tab{j, je, kr, dup} ++i } if (i == m) goto{lookup} @@ -418,24 +453,11 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, ilink:ity{name}) } setlabel{lookup} - end := maxh>>sh + promote{T,ext} # Unreachable - if ((prim!='⊒') and has_maxh) { - end=maxh>>sh; while (load{hash,end}!=maxh) ++end - each{store{.,end,.}, vals, ind_maxh} - } + fixup{} @for (rp, fp over n) { h := hash_val{fp}; j := h>>sh - k:=undefined{T}; while ((k=load{hash,j}) < h) ++j - rp = (match (prim) { - {('∊')} => promote{i8, (k==h) & (j i32~~(m - (load{tupsel{0,vals}, j} &- (k==h))) - {('⊒')} => { - def {inds} = vals - ti := load{inds, j}; mi := ti &- (k==h) - store{inds, j, ti - load{tupsel{0,ilink},mi}} - i32~~(m - mi) - } - }) + k := undefined{T}; while ((k=load{hash,j}) < h) ++j + rp = get_res{k==h, j} } setlabel{abort} diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index 37571407..834de1f3 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -1,11 +1,6 @@ include './base' local include 'skin/cext' -include './hashmap' - -def names = tup{'memberOf', 'count', 'indexOf'} -def prims = tup{'∊', '⊒', '⊐' } -def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} -def to_prim = map{names, prims, .} +include './hashtab' # Resizing hash table, with fallback def rty{name} = if (to_prim{name}=='∊') i8 else i32 @@ -68,7 +63,7 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { if (k!=h) { val<-{j}ctr; ++ctr; hash<-{j}h } val->j } - tup{{b}=>promote{u32,b}*ctr, res} + tup{{b}=>ctr & -promote{u32,b}, res} } } From 84f32055471bf3b3080014115b0dfccc6f5cc963 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 3 Nov 2023 21:50:38 -0400 Subject: [PATCH 10/40] Use a larger initial hash table if non-hashed argument is large --- src/singeli/src/search.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 40b2e35c..d963262a 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -344,7 +344,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) log := clzc{m} # Max size - msl := clzc{m} + 2 + msl := max{clzc{m+m/2}+1, min{ux~~14,clzc{m+n/4}}} msz := usz~~1 << msl # Starting log-size sl := msl; if (msl>=14) sl = 12+(msl&1) From 1286ee32eca3ee73d0a05277906f1a9908e46af2 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 3 Nov 2023 21:58:17 -0400 Subject: [PATCH 11/40] Reverse lookup for hashed Progressive Index-of --- src/singeli/src/search.singeli | 39 ++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index d963262a..3f127954 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -340,6 +340,10 @@ def ity{name} = (to_prim{name}=='⊒')**(*u32) fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) = { def wt = width{T} def prim = to_prim{name} + swap:u1 = 0 + if (prim=='⊒' and (swap = n < m)) { + each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} + } def stop = m log := clzc{m} @@ -454,10 +458,37 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) setlabel{lookup} fixup{} - @for (rp, fp over n) { - h := hash_val{fp}; j := h>>sh - k := undefined{T}; while ((k=load{hash,j}) < h) ++j - rp = get_res{k==h, j} + if (prim=='⊒' and swap) { + @for (rp over m) rp = i32~~n + @for (fp over i to n) { + h := hash_val{fp}; j := h>>sh + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + if (k==h) { + def {inds} = vals; def {link} = links + ti := load{inds, j} + if (ti>0) { + store{inds, j, ti - load{link,ti}} + } else { + do { + jp:=j; ++j + h=load{hash,j} + if (h>>sh == j) h = maxh + store{hash, jp, h} + store{inds, jp, load{inds,j}} + } while (h!=maxh) + } + store{rp, m-ti, i} + } + } + } + } else { + @for (rp, fp over n) { + h := hash_val{fp}; j := h>>sh + k := undefined{T}; while ((k=load{hash,j}) < h) ++j + rp = get_res{k==h, j} + } } setlabel{abort} From af81da50e4af934e31084874f0651779440103e7 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 4 Nov 2023 17:52:22 -0400 Subject: [PATCH 12/40] Don't need to initialize indices for hashed Index-of --- src/singeli/src/hashtab.singeli | 6 +++++- src/singeli/src/search.singeli | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index f07e2a05..92c200b7 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -50,6 +50,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s} = { def pe{{..._, p}} = p+sze # Next unallocated space given pointers so far def pl{{...R, T}} = { def ps=pl{R}; tup{...ps, *T~~pe{ps}+szo} } ptrs := pl{Ts} + def memset{_, ('any'), _} = {} # Indicates initialization not needed each{memset{., ., sze}, ptrs, v0s} def hash_resize{cc, m} = { @@ -66,7 +67,10 @@ def hash_alloc{logsz, msz, ext, Ts, v0s} = { k0 := h>>sh; k := k0; while (hash->k!=h0) ++k cc += cast_i{usz, k-k0} hash <-{k} h - each{{p,v0} => { v := p->j; p <-{j} v0; p <-{k} v }, vals, v0r} + def move{p,v0} = { + v := p->j; if (not same{v0,'any'}) p <-{j} v0; p <-{k} v + } + each{move, vals, v0r} } } } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 3f127954..5689d391 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -360,7 +360,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) maxh := T~~maxvalue{T} def aux = prim!='∊' def {tabs, sz, sh, hash_resize, hash_free} = hash_alloc{ - sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**0} + sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**(if (prim=='⊐') 'any' else 0)} } def {hash,...vals} = tabs From ec3154360459bd8a1414480854713b3981e769b3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 5 Nov 2023 15:04:30 -0500 Subject: [PATCH 13/40] Vector Member-of search up to an offset of 4 --- src/singeli/src/search.singeli | 38 +++++++++++++++++++++++++++++++++ src/singeli/src/vecfold.singeli | 1 - 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 5689d391..83f9f612 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -371,6 +371,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) def setm{found, i} = has_maxh |= found end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table def usem{} = { + try_vec_memb{T, hash, sz+ext, sh, maxh, has_maxh, rp, fp, n, abort} if (has_maxh) { while (load{hash,end}!=maxh) ++end } else end += promote{T,ext} # Unreachable } @@ -496,6 +497,43 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) i == m # Whether it finished } +def try_vec_memb{} = {} +def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done + & hasarch{'SSE4.2'} & T==u32} = { + # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) + # i-h>>sh = i+((1<>sh = (((i+1)<>sh + # We maintain io = (i+1)< (o+k)<> sh + + def memb{test} = { + def R = i8; def rw = width{R}; def u = width{T}/rw + l := n/u + @for (r in *T~~rp over i to l) { + c := V**0 # Will combine u results to avoid folding too much + @unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f} + r = vfold{|, c} + } + @for (rp, fp over _ from u*l to n) rp = promote{R, homAny{test{fp}}} + goto{done} + } + if (max_off < vl) { + hp := *V~~(hash + maxh>>sh) + hv := load{hp} + store{hp, 0, hv &~ (hv == V**(maxh &- ~has_maxh))} + def test{x} = { + h := hash_val{x}; j := h>>sh + V**h == load{*V~~(hash+j), 0} + } + memb{test} + } +} + def exp_hash{T, name} = { export{merge{name,'_c2_hash',fmtnat{width{T}}}, hashtab{T, name}} } diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index f8b85b26..f0aadea2 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -1,5 +1,4 @@ # Fold associative/commutative operation across a register -# Used by squeeze.singeli, count.singeli def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { c:= x From 75aed91f32a1c389be8d91dcb27922c3c0a64eba Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 5 Nov 2023 15:46:11 -0500 Subject: [PATCH 14/40] And max offset of 8 by testing two vectors --- src/singeli/src/search.singeli | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 83f9f612..4b8ec4c3 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -522,16 +522,21 @@ def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done @for (rp, fp over _ from u*l to n) rp = promote{R, homAny{test{fp}}} goto{done} } - if (max_off < vl) { - hp := *V~~(hash + maxh>>sh) - hv := load{hp} - store{hp, 0, hv &~ (hv == V**(maxh &- ~has_maxh))} - def test{x} = { - h := hash_val{x}; j := h>>sh - V**h == load{*V~~(hash+j), 0} + def try{nv} = { + if (max_off < nv*vl) { + # Avoid matching maxh if it shouldn't be in the table + clear := V**(maxh &- ~has_maxh) + @for (hv in *V~~(hash + maxh>>sh) over vl) hv = hv &~ (hv == clear) + # Test against nv vectors + def test{x} = { + h := hash_val{x}; vh := V**h + def any{{...r,a}} = any{r}|a; def any{{a}}=a + any{@collect (k in *V~~(hash+h>>sh) over nv) vh == k} + } + memb{test} } - memb{test} } + each{try, tup{1,2}} } def exp_hash{T, name} = { From 11245d385e349d0956e107905b32fbc49eda4794 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 5 Nov 2023 22:10:39 -0500 Subject: [PATCH 15/40] Vector binary search Member-of --- src/builtins/search.c | 21 +++++++++++++--- src/singeli/src/bins.singeli | 49 +++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 5d6f3723..117dbdf1 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -46,9 +46,13 @@ RangeFn getRange_fns[el_f64+1]; GETRANGE(i32,) GETRANGE(f64, if (!q_fi64(c)) return 0) #endif +#if SINGELI_AVX2 + extern void (*const avx2_member_sort_i32)(uint64_t*,int32_t*,uint64_t,int32_t*,uint64_t); +#endif #define C2i(F, W, X) C2(F, m_i32(W), X) +extern B and_c1(B,B); extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); extern B or_c2(B,B,B); @@ -335,10 +339,19 @@ B memberOf_c2(B t, B w, B x) { } u8 me = we>xe?we:xe; - if (xia<=(me==el_i8?1:me==el_i16?4:16) && wia>16) { - SGetU(x); - r = WEQ(GetU(x,0)); - for (usz i=1; i16) { + #if SINGELI_AVX2 + if (we==xe && we==el_i32 && xia>1) { + x = C1(and, x); // sort + u64* rp; r = m_bitarrc(&rp, w); + avx2_member_sort_i32(rp, tyany_ptr(x), xia, tyany_ptr(w), wia); + } else + #endif + { + SGetU(x); + r = WEQ(GetU(x,0)); + for (usz i=1; i 1}; assert{wn < maxwn} def wd = width{T} def I = if (wd<32) u8 else u32; def wi = width{I} @@ -214,7 +215,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { log := ceil_log2{wn+1} gap := 1<>(lb{isub}+wd-wi) + # b records if xv was found; c is added to the index + def r_out = prim!='∊' + def get_up{var,cmpx,use}{set,...a} = if (use) set{var, cmpx{...a}} + b := undefined{U}; def up_b = get_up{b, eqx, prim=='∊'} + c := undefined{U}; def up_c = get_up{c, ltx, r_out} + up_b{=, selw, s} # Extra selection lanes if (last and ex>=1 and log>=klog+1) { r += r - c := ltx{selw1,s} + def up_bc{set,se} = { up_b{|=,se,s}; up_c{set,se,s} } + up_bc{=,selw1} if (ex>=2 and log>=klog+2) { r += r - each{{se} => c += ltx{se,s}, selw2} + each{up_bc{+=, .}, selw2} } - r += c + if (r_out) r += c + } + if (r_out) { + r -= off + rn := if (T==i8) r + else if (T==i16) half{narrow{u8, r}, 0} + else extract{re_el{i64, narrow{u8, r}}, 0} + rnp := *type{rn}~~(*i8~~rp+j) + if (isvec{type{rn}}) store{rnp, 0, rn} + else storeu{rnp, rn} + } else { + out := homMask{b}; def B = type{out}; def wb = width{B} + store{*B~~rp, cdiv{j,wb}, out>>((-j)%wb)} } - r -= off - rn := if (T==i8) r - else if (T==i16) half{narrow{u8, r}, 0} - else extract{re_el{i64, narrow{u8, r}}, 0} - rnp := *type{rn}~~(*i8~~rp+j) - if (isvec{type{rn}}) store{rnp, 0, rn} - else storeu{rnp, rn} } } } +if (hasarch{'AVX2'}) { + fn avx2_search_bin{prim, T}(rp:*u64, w:*T, wn:u64, x:*T, xn:u64) : void = { + bin_search_vec{prim, T, w, wn, x, xn, rp, 16} + } + export{'avx2_member_sort_i32', avx2_search_bin{'∊',i32}} +} + def unroll_sizes = tup{4,1} fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = { each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs} @@ -321,7 +342,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { # For >=8 i8 values, vector bit-table is as good as binary search def wn_vec = if (T==i8) 8 else 2*256/width{T} if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) { - bin_search_vec{T, ...param, wn_vec} + bin_search_vec{if (up) '⍋' else '⍒', T, ...slice{param,1}, wn_vec} # Lookup table threshold has to account for cost of # populating the table (proportional to wn until it's large), and # initializing the table (constant, much higher for i16) From d154ece51abb966278226bff389e660007f01455 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 6 Nov 2023 09:37:58 -0500 Subject: [PATCH 16/40] Vector binary search for 16-bit Member-of in addition to 32 (8 is too slow) --- src/builtins/search.c | 29 ++++++++++++++--------------- src/singeli/src/bins.singeli | 13 ++++++++----- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 117dbdf1..0da3b466 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -47,7 +47,7 @@ RangeFn getRange_fns[el_f64+1]; GETRANGE(f64, if (!q_fi64(c)) return 0) #endif #if SINGELI_AVX2 - extern void (*const avx2_member_sort_i32)(uint64_t*,int32_t*,uint64_t,int32_t*,uint64_t); + extern void (**const avx2_member_sort)(uint64_t*,void*,uint64_t,void*,uint64_t); #endif @@ -338,20 +338,19 @@ B memberOf_c2(B t, B w, B x) { decG(w); goto dec_x; } - u8 me = we>xe?we:xe; - if (xia<=(me==el_i8?1:me==el_i16?4:15) && wia>16) { - #if SINGELI_AVX2 - if (we==xe && we==el_i32 && xia>1) { - x = C1(and, x); // sort - u64* rp; r = m_bitarrc(&rp, w); - avx2_member_sort_i32(rp, tyany_ptr(x), xia, tyany_ptr(w), wia); - } else - #endif - { - SGetU(x); - r = WEQ(GetU(x,0)); - for (usz i=1; i=32>>(we-el_i8) && xia>1 && ((we==el_i16 && xia<32) || (we==el_i32 && xia<16)) && xe<=we && !elChr(TI(x,elType))) { + x = C1(and, x); // sort + if (xe16) { + SGetU(x); + r = WEQ(GetU(x,0)); + for (usz i=1; i>((-j)%wb)} + def B = ty_u{vl}; out := cast_i{B, homMask{b}} + store{*B~~rp, cdiv{j,vl}, out>>((-j)%vl)} } } } } if (hasarch{'AVX2'}) { - fn avx2_search_bin{prim, T}(rp:*u64, w:*T, wn:u64, x:*T, xn:u64) : void = { - bin_search_vec{prim, T, w, wn, x, xn, rp, 16} + fn avx2_search_bin{prim, T, maxwn}(rp:*u64, w:*void, wn:u64, x:*void, xn:u64) : void = { + bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn} + } + exportT{ + 'avx2_member_sort', + each{avx2_search_bin{'∊',.,.}, tup{i16,i32}, tup{32,16}} } - export{'avx2_member_sort_i32', avx2_search_bin{'∊',i32}} } def unroll_sizes = tup{4,1} From 52bca6a55ccaa624c5637b0dcb5c87fdd9d4dd2c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 6 Nov 2023 12:16:59 -0500 Subject: [PATCH 17/40] Multiple IndexOfOne to search for a few values in a large array --- src/builtins/search.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/builtins/search.c b/src/builtins/search.c index 0da3b466..fcb89479 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -237,6 +237,18 @@ B indexOf_c2(B t, B w, B x) { } CHECK_CHRS_ELSE { tyEls: // Both numbers + if (wia>32 && xia<=(we<=el_i8?1:6)) { + SGetU(x); + B r; + #define IND(T) \ + T* rp; r = m_##T##arrc(&rp, x); \ + for (usz i=0; i32 && wia<=(xe<=el_i8?1:6)) { + SGetU(w); + i8* rp; r = m_i8arrc(&rp, w); + for (usz i=0; i Date: Mon, 6 Nov 2023 15:43:26 -0500 Subject: [PATCH 18/40] Vector binary search for integer Index-of --- src/builtins/search.c | 16 ++++++++++++++++ src/singeli/src/bins.singeli | 14 ++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index fcb89479..90ad0e4a 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -48,17 +48,22 @@ RangeFn getRange_fns[el_f64+1]; #endif #if SINGELI_AVX2 extern void (**const avx2_member_sort)(uint64_t*,void*,uint64_t,void*,uint64_t); + extern void (**const avx2_indexOf_sort)(int8_t*,void*,uint64_t,void*,uint64_t); #endif #define C2i(F, W, X) C2(F, m_i32(W), X) extern B and_c1(B,B); +extern B gradeDown_c1(B,B); +extern B reverse_c1(B,B); extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); extern B or_c2(B,B,B); extern B add_c2(B,B,B); extern B sub_c2(B,B,B); extern B mul_c2(B,B,B); +extern B join_c2(B,B,B); +extern B select_c2(B,B,B); static u64 elRange(u8 eltype) { return 1ull<<(1<=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<64>>(xe-el_i8) && we<=xe && !elChr(TI(x,elType))) { + B g = C1(reverse, C1(gradeDown, incG(w))); + w = C2(select, incG(g), w); + switch (xe) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; } + i8* rp; B r = m_i8arrc(&rp, x); + avx2_indexOf_sort[xe-el_i8](rp, tyany_ptr(w), wia, tyany_ptr(x), xia); + r = C2(select, r, C2(join, g, m_i8(wia))); + decG(w); decG(x); return r; + } + #endif if (wia<=(we<=el_i16?4:16) && xia>16) { SGetU(w); #define XEQ(I) C2(ne, GetU(w,I), incG(x)) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 157fcaed..89274f39 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -202,6 +202,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { # Binary search within vector registers def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def up = prim != '⍒' + def search = (prim=='∊') | (prim=='⊐') assert{wn > 1}; assert{wn < maxwn} def wd = width{T} def I = if (wd<32) u8 else u32; def wi = width{I} @@ -215,7 +216,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { log := ceil_log2{wn+1} gap := 1<=1) ms{wv}{1} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' # Offset at end - off := U~~V**cast_i{i8, gap-1} + off := U~~V**cast_i{i8, gap-(1-search)} # Midpoint bits for each step def lowbits = bb{copy{isub,isub}} bits := each{{j} => U**(lowbits << j), iota{lstep}} @@ -263,7 +264,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { # b records if xv was found; c is added to the index def r_out = prim!='∊' def get_up{var,cmpx,use}{set,...a} = if (use) set{var, cmpx{...a}} - b := undefined{U}; def up_b = get_up{b, eqx, prim=='∊'} + b := undefined{U}; def up_b = get_up{b, eqx, search} c := undefined{U}; def up_c = get_up{c, ltx, r_out} up_b{=, selw, s} # Extra selection lanes @@ -279,6 +280,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { } if (r_out) { r -= off + if (prim=='⊐') r = homBlend{U**cast_i{u8,wn}, r, b} rn := if (T==i8) r else if (T==i16) half{narrow{u8, r}, 0} else extract{re_el{i64, narrow{u8, r}}, 0} @@ -294,13 +296,17 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { } if (hasarch{'AVX2'}) { - fn avx2_search_bin{prim, T, maxwn}(rp:*u64, w:*void, wn:u64, x:*void, xn:u64) : void = { + fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = { bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn} } exportT{ 'avx2_member_sort', each{avx2_search_bin{'∊',.,.}, tup{i16,i32}, tup{32,16}} } + exportT{ + 'avx2_indexOf_sort', + each{avx2_search_bin{'⊐',.,.}, tup{i8,i16,i32}, tup{64,32,16}} + } } def unroll_sizes = tup{4,1} From 30331eb800e95c6b860dfcb10546993e209762ce Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 07:15:17 -0500 Subject: [PATCH 19/40] =?UTF-8?q?Fix=20write=20off=20end=20for=20reverse?= =?UTF-8?q?=20=E2=8A=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/search.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 4b8ec4c3..fa12f6c1 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -471,6 +471,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) ti := load{inds, j} if (ti>0) { store{inds, j, ti - load{link,ti}} + store{rp, m-ti, i} } else { do { jp:=j; ++j @@ -480,7 +481,6 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) store{inds, jp, load{inds,j}} } while (h!=maxh) } - store{rp, m-ti, i} } } } From 289658a59ccf8966bca9619f08b07ca849872348 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 07:53:45 -0500 Subject: [PATCH 20/40] Delete hash entry as soon as it's cleared, not on next access --- src/singeli/src/search.singeli | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index fa12f6c1..16ac8f57 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -469,9 +469,10 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) if (k==h) { def {inds} = vals; def {link} = links ti := load{inds, j} - if (ti>0) { - store{inds, j, ti - load{link,ti}} - store{rp, m-ti, i} + store{rp, m-ti, i} + ti -= load{link, ti} + if (ti > 0) { + store{inds, j, ti} } else { do { jp:=j; ++j From 4bf4149d4040644b6b361d321c0b71bb01490015 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 09:33:46 -0500 Subject: [PATCH 21/40] =?UTF-8?q?Use=20hash=20deletion=20for=20forward=20?= =?UTF-8?q?=E2=8A=92=20as=20well=20as=20reverse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/search.singeli | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 16ac8f57..84a9b61e 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -409,14 +409,10 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) store{inds, j, 0} setlabel{dup} i1 := i+1 - store{link, i1, i1-load{inds,j}} + store{link, i1, load{inds,j}} store{inds, j, i1} } - def get{found, j} = { - ti := load{inds, j}; mi := ti &- found - store{inds, j, ti - load{link,mi}} - i32~~(m - mi) - } + def get{found, j} = assert{0} tup{set, {found,i}=>{}, {}=>{}, get} } } @@ -459,8 +455,9 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) setlabel{lookup} fixup{} - if (prim=='⊒' and swap) { - @for (rp over m) rp = i32~~n + def prog_lookup{swap} = { # Progressive Index-of lookup + def rev{a,b} = if (swap) tup{b,a} else tup{a,b} + memset{*u32~~rp, ...rev{m,n}} @for (fp over i to n) { h := hash_val{fp}; j := h>>sh k := load{hash,j} @@ -469,8 +466,8 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) if (k==h) { def {inds} = vals; def {link} = links ti := load{inds, j} - store{rp, m-ti, i} - ti -= load{link, ti} + store{rp, ...rev{i, m-ti}} + ti = load{link, ti} if (ti > 0) { store{inds, j, ti} } else { @@ -485,6 +482,9 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) } } } + } + if (prim=='⊒') { + if (not swap) prog_lookup{0} else prog_lookup{1} } else { @for (rp, fp over n) { h := hash_val{fp}; j := h>>sh From a4290b1f5c300fbeba737a9c38d3c77f06a41073 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 09:59:53 -0500 Subject: [PATCH 22/40] =?UTF-8?q?Shortcut=20for=20hash-table=20dyadic=20?= =?UTF-8?q?=E2=8A=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/search.singeli | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 84a9b61e..0108283d 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -458,6 +458,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) def prog_lookup{swap} = { # Progressive Index-of lookup def rev{a,b} = if (swap) tup{b,a} else tup{a,b} memset{*u32~~rp, ...rev{m,n}} + c:usz = m; def shortcut = makelabel{} @for (fp over i to n) { h := hash_val{fp}; j := h>>sh k := load{hash,j} @@ -467,6 +468,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) def {inds} = vals; def {link} = links ti := load{inds, j} store{rp, ...rev{i, m-ti}} + c -= promote{usz,ti>0}; if (c==0) goto{shortcut} ti = load{link, ti} if (ti > 0) { store{inds, j, ti} @@ -482,6 +484,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) } } } + setlabel{shortcut} } if (prim=='⊒') { if (not swap) prog_lookup{0} else prog_lookup{1} From 8a2190b09253fe5e3a584f98a82c5cf3a02d2408 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 14:50:26 -0500 Subject: [PATCH 23/40] 64-bit hashes with float normalization --- src/builtins/search.c | 31 +++++++++++++++++++++++++------ src/singeli/src/search.singeli | 11 ++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 90ad0e4a..bd910283 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -65,6 +65,13 @@ extern B mul_c2(B,B,B); extern B join_c2(B,B,B); extern B select_c2(B,B,B); +B asNormalized(B x, usz n, bool nanBad); +SHOULD_INLINE bool canCompare64_norm(B* w, usz wia, B* x, usz xia) { + B wn=asNormalized(*w,wia,true); if (wn.u == m_f64(0).u) return 0; *w=wn; + B xn=asNormalized(*x,xia,true); if (xn.u == m_f64(0).u) return 0; *x=xn; + return 1; +} + static u64 elRange(u8 eltype) { return 1ull<<(1< v & -promote{T,m}) infix left 35 def rty{name} = if (to_prim{name}=='∊') i8 else i32 def ity{name} = (to_prim{name}=='⊒')**(*u32) -fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) = { +fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{name}) = { def wt = width{T} def prim = to_prim{name} + ip := *T~~iv; fp := *T~~fv swap:u1 = 0 if (prim=='⊒' and (swap = n < m)) { each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} @@ -501,7 +502,7 @@ fn hashtab{T, name}(rp:*rty{name}, ip:*T, m:usz, fp:*T, n:usz, links:ity{name}) i == m # Whether it finished } -def try_vec_memb{} = {} +def try_vec_memb{..._} = {} def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done & hasarch{'SSE4.2'} & T==u32} = { # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) @@ -543,7 +544,7 @@ def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done each{try, tup{1,2}} } -def exp_hash{T, name} = { - export{merge{name,'_c2_hash',fmtnat{width{T}}}, hashtab{T, name}} +def exp_hash{name} = { + exportT{merge{'si_',name,'_c2_hash'}, each{hashtab{., name}, tup{u32,u64}}} } -each{{n}=>each{exp_hash{.,n},tup{u32}}, names} +each{exp_hash, names} From c75fdbc27795f6654a75e6dcdbecefd6fe6bfa9a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 7 Nov 2023 22:06:34 -0500 Subject: [PATCH 24/40] Reverse Member-of hash lookup --- src/builtins/search.c | 2 +- src/singeli/src/search.singeli | 52 +++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index bd910283..191f2b0c 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -370,7 +370,7 @@ B memberOf_c2(B t, B w, B x) { } CHECK_CHRS_ELSE { tyEls: // Both numbers - if (xia>32 && wia<=(xe<=el_i8?1:6)) { + if (xia>32 && wia<=(xe<=el_i8?1:xe==el_i32?4:6)) { SGetU(w); i8* rp; r = m_i8arrc(&rp, w); for (usz i=0; i{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} swap:u1 = 0 - if (prim=='⊒' and (swap = n < m)) { - each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} - } + if ((prim=='⊒' or prim=='∊') and (swap = n < m)) swap_sides{} def stop = m log := clzc{m} @@ -372,11 +371,24 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n def setm{found, i} = has_maxh |= found end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table def usem{} = { - try_vec_memb{T, hash, sz+ext, sh, maxh, has_maxh, rp, fp, n, abort} + if (swap) { + swap_sides{}; i=m # i==m return value is kind of dumb + @for (ip over m) { # Remove values in ip from hash + h := hash_val{ip}; j := h>>sh + if (h == maxh) { has_maxh = 0 } else { + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + if (k==h) hash_remove{j,h} + } + } + } + } + try_vec_memb{T, hash, sz+ext, sh, maxh, has_maxh, swap, rp, fp, n, abort} if (has_maxh) { while (load{hash,end}!=maxh) ++end } else end += promote{T,ext} # Unreachable } - def get{found, j} = promote{i8, found & (j { @@ -454,6 +466,16 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } } + def hash_remove{j,h} = { + do { + jp:=j; ++j + h=load{hash,j} + if (h>>sh == j) h = maxh + store{hash, jp, h} + each{{t} => store{t, jp, load{t,j}}, vals} + } while (h!=maxh) + } + setlabel{lookup} fixup{} def prog_lookup{swap} = { # Progressive Index-of lookup @@ -471,17 +493,8 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n store{rp, ...rev{i, m-ti}} c -= promote{usz,ti>0}; if (c==0) goto{shortcut} ti = load{link, ti} - if (ti > 0) { - store{inds, j, ti} - } else { - do { - jp:=j; ++j - h=load{hash,j} - if (h>>sh == j) h = maxh - store{hash, jp, h} - store{inds, jp, load{inds,j}} - } while (h!=maxh) - } + if (ti > 0) store{inds, j, ti} + else hash_remove{j,h} } } } @@ -503,7 +516,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } def try_vec_memb{..._} = {} -def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done +def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, swap, rp, fp, n, done & hasarch{'SSE4.2'} & T==u32} = { # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) # i-h>>sh = i+((1<>sh = (((i+1)<>sh @@ -515,6 +528,7 @@ def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done mv := V**0 @for (h in *V~~(hash+o) over nh) { mv=max{mv, io-min{h,io}}; io+=id } max_off := vfold{max, mv} >> sh + vswap := base{256,vl**1} * promote{T,swap} def memb{test} = { def R = i8; def rw = width{R}; def u = width{T}/rw @@ -522,9 +536,9 @@ def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, rp, fp, n, done @for (r in *T~~rp over i to l) { c := V**0 # Will combine u results to avoid folding too much @unroll (f in fp+u*i over a to u) c |= V**(1<<(rw*a)) & test{f} - r = vfold{|, c} + r = vswap ^ vfold{|, c} } - @for (rp, fp over _ from u*l to n) rp = promote{R, homAny{test{fp}}} + @for (rp, fp over _ from u*l to n) rp = promote{R, swap ^ homAny{test{fp}}} goto{done} } def try{nv} = { From 8e9c04523a00bde0ddbd9ca417e115a77c1af496 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 8 Nov 2023 10:28:14 -0500 Subject: [PATCH 25/40] Fix overflow bug in hashed Member-of max_off calculation --- src/singeli/src/search.singeli | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 825bcaa8..89507f37 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -384,7 +384,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } } } - try_vec_memb{T, hash, sz+ext, sh, maxh, has_maxh, swap, rp, fp, n, abort} + try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} if (has_maxh) { while (load{hash,end}!=maxh) ++end } else end += promote{T,ext} # Unreachable } @@ -516,20 +516,25 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } def try_vec_memb{..._} = {} -def try_vec_memb{T, hash, hashlen, sh, maxh, has_maxh, swap, rp, fp, n, done +def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done & hasarch{'SSE4.2'} & T==u32} = { # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) # i-h>>sh = i+((1<>sh = (((i+1)<>sh # We maintain io = (i+1)< (o+k)< T~~k<> sh - vswap := base{256,vl**1} * promote{T,swap} + # sz==1<=sz the above overflows + # And we have to handle maxh specially anyway + def mx{i,h} = { max_off = max{max_off, i-h>>sh} } + i:T=sz; h:T=i; while ((h=load{hash,i}) Date: Wed, 8 Nov 2023 17:34:33 -0500 Subject: [PATCH 26/40] Shortcutting for reverse Member-of hash --- src/singeli/src/search.singeli | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 89507f37..66fe18f4 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -375,11 +375,23 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n swap_sides{}; i=m # i==m return value is kind of dumb @for (ip over m) { # Remove values in ip from hash h := hash_val{ip}; j := h>>sh - if (h == maxh) { has_maxh = 0 } else { + def shortcut = makelabel{} + if (h == maxh) { + if (uniq==0) goto{shortcut} + has_maxh = 0 + } else { k := load{hash,j} if (k <= h) { while (k < h) { ++j; k = load{hash,j} } - if (k==h) hash_remove{j,h} + if (k == h) { + --uniq + if (uniq==0 and not has_maxh) { + setlabel{shortcut} + @for (rp over n) rp = 1 + goto{abort} + } + hash_remove{j,h} + } } } } @@ -432,6 +444,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n def lookup=makelabel{}; def abort=makelabel{}; def dup=makelabel{} cc:usz = 0 # Collision counter + uniq:usz = 0 # Uniques inserted i:usz=0; while (1) { e := tern{m-i>b, i+b, m} while (i < e) { @@ -453,6 +466,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } while (k != maxh) cc += cast_i{usz, je-j0} } + if (prim=='∊') uniq += promote{usz,h!=maxh} store{hash, j, h} set_tab{j, je, kr, dup} ++i From f1485ca98cf5ae81d716683f3b7f5e0c1024844a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 8 Nov 2023 21:07:03 -0500 Subject: [PATCH 27/40] Turn hash function definition structure inside-out --- src/singeli/src/search.singeli | 239 +++++++++++++++++---------------- 1 file changed, 122 insertions(+), 117 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 66fe18f4..cb065b20 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -364,120 +364,46 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } def {hash,...vals} = tabs - def {set_tab, set_maxh, fixup, get_res} = match (prim) { - {('∊')} => { - has_maxh:u1 = 0 - def set{j,je,kr,dup} = { setlabel{dup} } - def setm{found, i} = has_maxh |= found - end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table - def usem{} = { - if (swap) { - swap_sides{}; i=m # i==m return value is kind of dumb - @for (ip over m) { # Remove values in ip from hash - h := hash_val{ip}; j := h>>sh - def shortcut = makelabel{} - if (h == maxh) { - if (uniq==0) goto{shortcut} - has_maxh = 0 - } else { - k := load{hash,j} - if (k <= h) { - while (k < h) { ++j; k = load{hash,j} } - if (k == h) { - --uniq - if (uniq==0 and not has_maxh) { - setlabel{shortcut} - @for (rp over n) rp = 1 - goto{abort} - } - hash_remove{j,h} - } - } - } - } + def abort = makelabel{} + i:usz = 0 # Saved to determine if hashing finished + def insert_all{set_tab, set_maxh, dup, ...uniq} = { + cc:usz = 0 # Collision counter + def insert_finish = makelabel{} + while (1) { + e := tern{m-i>b, i+b, m} + while (i < e) { + def ii = if (prim!='⊒') i else m-i-1 + h := hash_val{load{ip,ii}}; j := h>>sh + set_maxh{h==maxh, i} + kv := each{load{.,j}, tabs}; def {k,...kr} = kv + # Robin Hood insertion + j0 := j; je := j # Save value; end of chain (insert at j) + if (k != maxh) { + if (k == h) goto{dup} + do { + ++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv + def c = promote{T, h >= k} + j += c + if (kn == h) goto{dup} + each{store{.,je-c,.}, tabs, kv} + each{=, kv, knv} + } while (k != maxh) + cc += cast_i{usz, je-j0} } - try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} - if (has_maxh) { while (load{hash,end}!=maxh) ++end } - else end += promote{T,ext} # Unreachable + each{{u} => { u += promote{usz,h!=maxh} }, uniq} + store{hash, j, h} + set_tab{j, je, kr} + ++i + } + if (i == m) goto{insert_finish} + # Check collision counter and possibly resize (needs tuning) + if (cc >= stop and sz == msz) goto{abort} + if (16*cc >= stop and sz < msz) { + hash_resize{cc, 2} # Factor of 4 + if (cc >= stop) goto{abort} } - def get{found, j} = promote{i8, swap ^ (found & (j { - def {inds} = vals - has_maxh:u1 = 0 - ind_maxh:u32 = 0 - def set{j,je,kr,dup} = { - store{inds, j, m-i} # So it can be cleared with one &- in get{} - setlabel{dup} - } - def setm{found, i} = { - ind_maxh |= i &- (found&~has_maxh) - has_maxh |= found - } - def usem{} = { - if (has_maxh) { - end := maxh>>sh; while (load{hash,end}!=maxh) ++end - store{inds, end, ind_maxh} - } - } - def get{found, j} = i32~~(m - (load{inds, j} &- found)) - tup{set, setm, usem, get} - } - {('⊒')} => { - def {inds} = vals; def {link} = links - store{link,0,0} - def set{j,je,{kr},dup} = { - # maxh entry either stores 0 or is the last entry and stores an - # index, so this preserves all values - ++je; store{inds, je, kr|load{inds,je}} - store{inds, j, 0} - setlabel{dup} - i1 := i+1 - store{link, i1, load{inds,j}} - store{inds, j, i1} - } - def get{found, j} = assert{0} - tup{set, {found,i}=>{}, {}=>{}, get} - } - } - - def lookup=makelabel{}; def abort=makelabel{}; def dup=makelabel{} - cc:usz = 0 # Collision counter - uniq:usz = 0 # Uniques inserted - i:usz=0; while (1) { - e := tern{m-i>b, i+b, m} - while (i < e) { - def ii = if (prim!='⊒') i else m-i-1 - h := hash_val{load{ip,ii}}; j := h>>sh - set_maxh{h==maxh, i} - kv := each{load{.,j}, tabs}; def {k,...kr} = kv - # Robin Hood insertion - j0 := j; je := j # Save value; end of chain (insert at j) - if (k != maxh) { - if (k == h) goto{dup} - do { - ++je; knv := each{load{.,je}, tabs}; def {kn,..._} = knv - def c = promote{T, h >= k} - j += c - if (kn == h) goto{dup} - each{store{.,je-c,.}, tabs, kv} - each{=, kv, knv} - } while (k != maxh) - cc += cast_i{usz, je-j0} - } - if (prim=='∊') uniq += promote{usz,h!=maxh} - store{hash, j, h} - set_tab{j, je, kr, dup} - ++i - } - if (i == m) goto{lookup} - # Check collision counter and possibly resize (needs tuning) - if (cc >= stop and sz == msz) goto{abort} - if (16*cc >= stop and sz < msz) { - hash_resize{cc, 2} # Factor of 4 - if (cc >= stop) goto{abort} } + setlabel{insert_finish} } def hash_remove{j,h} = { @@ -489,9 +415,30 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n each{{t} => store{t, jp, load{t,j}}, vals} } while (h!=maxh) } - - setlabel{lookup} - fixup{} + def memb_remove{uniq, has_maxh} = { + @for (ip over m) { + h := hash_val{ip}; j := h>>sh + def shortcut = makelabel{} + if (h == maxh) { + if (uniq==0) goto{shortcut} + has_maxh = 0 + } else { + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + if (k == h) { + --uniq + if (uniq==0 and not has_maxh) { + setlabel{shortcut} + @for (rp over n) rp = 1 + goto{abort} + } + hash_remove{j,h} + } + } + } + } + } def prog_lookup{swap} = { # Progressive Index-of lookup def rev{a,b} = if (swap) tup{b,a} else tup{a,b} memset{*u32~~rp, ...rev{m,n}} @@ -514,9 +461,8 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } setlabel{shortcut} } - if (prim=='⊒') { - if (not swap) prog_lookup{0} else prog_lookup{1} - } else { + + def lookup_all{get_res} = { @for (rp, fp over n) { h := hash_val{fp}; j := h>>sh k := undefined{T}; while ((k=load{hash,j}) < h) ++j @@ -524,6 +470,65 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } } + match (prim, ...vals, ...links) { + {('∊')} => { + has_maxh:u1 = 0 + uniq:usz = 0 # Uniques inserted + def dup = makelabel{} + insert_all{ + {j,je,kr} => setlabel{dup}, + {found, i} => has_maxh |= found, + dup, uniq + } + if (swap) { + swap_sides{}; i=m # i==m return value is kind of dumb + memb_remove{uniq,has_maxh} # Remove values in ip from hash + } + try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} + end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table + if (has_maxh) { while (load{hash,end}!=maxh) ++end } + else end += promote{T,ext} # Unreachable + lookup_all{{found, j} => promote{i8, swap ^ (found & (j { + has_maxh:u1 = 0 + ind_maxh:u32 = 0 + def dup = makelabel{} + def set{j,je,kr} = { + store{inds, j, m-i} # So it can be cleared with one &- in get{} + setlabel{dup} + } + def set_maxh{found, i} = { + ind_maxh |= i &- (found&~has_maxh) + has_maxh |= found + } + insert_all{set, set_maxh, dup} + if (has_maxh) { + end := maxh>>sh; while (load{hash,end}!=maxh) ++end + store{inds, end, ind_maxh} + } + lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))} + } + + {('⊒'), inds, link} => { + store{link,0,0} + def dup = makelabel{} + def set{j,je,{kr}} = { + # maxh entry either stores 0 or is the last entry and stores an + # index, so this preserves all values + ++je; store{inds, je, kr|load{inds,je}} + store{inds, j, 0} + setlabel{dup} + i1 := i+1 + store{link, i1, load{inds,j}} + store{inds, j, i1} + } + insert_all{set, {found,i}=>{}, dup} + if (not swap) prog_lookup{0} else prog_lookup{1} + } + } + setlabel{abort} hash_free{} i == m # Whether it finished From 2c625127d559fbc8cfef2bf63775eac9a44f7066 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 9 Nov 2023 10:39:03 -0500 Subject: [PATCH 28/40] Reverse hashed Index-of --- src/builtins/search.c | 4 +-- src/singeli/src/search.singeli | 61 ++++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 191f2b0c..a1eaf467 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -251,7 +251,7 @@ B indexOf_c2(B t, B w, B x) { } CHECK_CHRS_ELSE { tyEls: // Both numbers - if (wia>32 && xia<=(we<=el_i8?1:6)) { + if (wia>32 && xia<=(we<=el_i8?1:3)) { SGetU(x); B r; #define IND(T) \ @@ -283,7 +283,7 @@ B indexOf_c2(B t, B w, B x) { decG(w); decG(x); return r; } #endif - if (wia<=(we<=el_i16?4:16) && xia>16) { + if (wia<=4 && xia>16) { SGetU(w); #define XEQ(I) C2(ne, GetU(w,I), incG(x)) B r = XEQ(wia-1); diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index cb065b20..a2649a33 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -342,8 +342,8 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n def prim = to_prim{name} ip := *T~~iv; fp := *T~~fv def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} - swap:u1 = 0 - if ((prim=='⊒' or prim=='∊') and (swap = n < m)) swap_sides{} + swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4) + if (swap) swap_sides{} def stop = m log := clzc{m} @@ -439,6 +439,24 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } } } + def ind_rev_lookup{uniq, has_maxh} = { + def shortcut = makelabel{} + @for (ip over i to m) { + h := hash_val{ip}; j := h>>sh + k := load{hash,j} + if (k <= h) { + while (k < h) { ++j; k = load{hash,j} } + def had_maxh{} = { s:=has_maxh; has_maxh=0; s } + if (k == h and (h promote{i8, swap ^ (found & (j { + {('⊐'), inds} => if (not swap) { has_maxh:u1 = 0 ind_maxh:u32 = 0 def dup = makelabel{} @@ -509,6 +527,43 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n store{inds, end, ind_maxh} } lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))} + } else { # swap + # After insert_all, position i in rp contains: + # - ≠𝕨, if i is the first occurrence of its value in 𝕩, or + # - j-≠𝕩, where j>sh; while (load{hash,end}!=maxh) ++end + store{inds, end, ind_maxh} + } + ind_rev_lookup{uniq, has_maxh} + # Propagate to later occurrences + @for (r in rp over i to n) r = load{rp, min{i, u32~~r+n}} } {('⊒'), inds, link} => { From 7b79dfc94e4355702964529846acb682ec0624e2 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 9 Nov 2023 15:18:28 -0500 Subject: [PATCH 29/40] Update search implementation comments --- src/builtins/search.c | 47 ++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index a1eaf467..d8d88a1e 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -1,23 +1,34 @@ // Dyadic search functions: Member Of (∊), Index of (⊐), Progressive Index of (⊒) -// 𝕨⊐unit or unit∊𝕩: scalar loop with early-exit -// SHOULD use simd -// SHOULD unify implementations -// 𝕩⊒unit or 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩 - -// Both arguments with rank≥1: -// High-rank inputs: -// Convert to a (lower-rank) typed integer array if cells are ≤62 bits -// COULD have special hashing for equal type >64 bit cells, skipping squeezing -// COULD try conditionally squeezing ahead-of-time, and not squeezing in bqn_hash -// p⊐n & n∊p with short p & long n: n⊸=¨ p -// bitarr⊐𝕩: more special arithmetic -// SHOULD have impls for long p & short n -// ≤16-bit elements: lookup tables -// Character elements: reinterpret as integer elements -// Otherwise, generic hashtable -// SHOULD handle up to 64 bit cells via proper typed hash tables -// SHOULD have fast path when cell sizes or element types doesn't match +// 𝕨⊐unit or unit∊𝕩: SIMD shortcutting search +// 𝕨⊒𝕩 where 1≥≠𝕩: defer to 𝕨⊐𝕩 +// High-rank inputs: +// Convert to a typed numeric list if cells are ≤62 bits +// COULD have hashing for equal-type >64 bit cells, to skip squeezing +// COULD try squeezing ahead-of-time to skip it in bqn_hash +// SHOULD have fast path when cell sizes don't match +// One input empty: fast not-found +// Character elements: +// Character versus number array is fast not-found for ∊ and ⊐ +// SHOULD have fast character-number path for ⊒ +// Reinterpret as integer elements +// COULD try p=⌜n when all arguments are short (may not be faster?) +// p⊐n & n∊p with short n: p⊸⊐¨n +// p⊐n & n∊p with boolean p: based on ⊑p and p⊐¬⊑p +// p⊐n & n∊p with short p: +// AVX2 binary search when applicable +// n⊸=¨p otherwise +// ≤16-bit elements: lookup tables +// 8-bit ∊ and ⊐: SSSE3 table +// SHOULD make 8-bit NEON table +// 32- or 64-bit elements: hash tables +// Store hash in table and not element; Robin Hood ordering; resizing +// Reverse hash if searched-for is shorter +// Shortcutting for reverse hashes and non-reversed ⊒ +// SIMD lookup for 32-bit ∊ if chain length is small enough +// SHOULD partition if hash table size gets large +// SHOULD handle unequal search types better +// Otherwise, generic hashtable #include "../core.h" #include "../utils/hash.h" From e1b0c53c92806c88d25337847863345048168d81 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 9 Nov 2023 20:24:46 -0500 Subject: [PATCH 30/40] Fix various sentinel bugs --- src/singeli/src/search.singeli | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index a2649a33..61579c49 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -348,9 +348,9 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n log := clzc{m} # Max size - msl := max{clzc{m+m/2}+1, min{ux~~14,clzc{m+n/4}}} + msl := max{clzc{(m+m/2)|4}+1, min{ux~~14,clzc{m+n/4}}} msz := usz~~1 << msl - # Starting log-size + # Starting log-size (try_vec_memb requires size>4) sl := msl; if (msl>=14) sl = 12+(msl&1) b:usz = 64 # Block size @@ -504,8 +504,8 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table - if (has_maxh) { while (load{hash,end}!=maxh) ++end } - else end += promote{T,ext} # Unreachable + while (load{hash,end}!=maxh) ++end + if (has_maxh) ++end # Don't clip lookup_all{{found, j} => promote{i8, swap ^ (found & (j>sh; while (load{hash,end}!=maxh) ++end - store{inds, end, ind_maxh} - } + end := maxh>>sh; while (load{hash,end}!=maxh) ++end + store{inds, end, (m-ind_maxh) &- has_maxh} lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))} } else { # swap # After insert_all, position i in rp contains: @@ -596,6 +594,7 @@ def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done # i-h>>sh = i+((1<>sh = (((i+1)<>sh # We maintain io = (i+1)< 4} assert{sz%vl == 0} io := make{V, each{{k} => T~~k< Date: Fri, 10 Nov 2023 08:22:52 -0500 Subject: [PATCH 31/40] =?UTF-8?q?Insert=20maxh=20to=20a=20special=20end=20?= =?UTF-8?q?bucket=20for=20=E2=8A=92=20and=20reverse=20=E2=8A=90=20(fixes?= =?UTF-8?q?=20=E2=8A=92=20maxh=20bugs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/search.singeli | 77 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 61579c49..beca3e04 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -360,7 +360,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n maxh := T~~maxvalue{T} def aux = prim!='∊' def {tabs, sz, sh, hash_resize, hash_free} = hash_alloc{ - sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**(if (prim=='⊐') 'any' else 0)} + sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**'any'} } def {hash,...vals} = tabs @@ -374,7 +374,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n while (i < e) { def ii = if (prim!='⊒') i else m-i-1 h := hash_val{load{ip,ii}}; j := h>>sh - set_maxh{h==maxh, i} + set_maxh{h==maxh, i, j} kv := each{load{.,j}, tabs}; def {k,...kr} = kv # Robin Hood insertion j0 := j; je := j # Save value; end of chain (insert at j) @@ -392,7 +392,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } each{{u} => { u += promote{usz,h!=maxh} }, uniq} store{hash, j, h} - set_tab{j, je, kr} + set_tab{j, h} ++i } if (i == m) goto{insert_finish} @@ -405,6 +405,15 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n } setlabel{insert_finish} } + def get_end{} = { + end := maxh>>sh + while (load{hash,end}!=maxh) ++end + end + } + def sequester_maxh{j, found} = { j += cast_i{T, ext &- found} } + def unsequester_maxh{tab} = { + store{tab, get_end{}, load{tab, maxh>>sh + cast_i{T,ext}}} + } def hash_remove{j,h} = { do { @@ -469,11 +478,13 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n if (k==h) { def {inds} = vals; def {link} = links ti := load{inds, j} - store{rp, ...rev{i, m-ti}} - c -= promote{usz,ti>0}; if (c==0) goto{shortcut} - ti = load{link, ti} - if (ti > 0) store{inds, j, ti} - else hash_remove{j,h} + if (ti > 0) { + store{rp, ...rev{i, m-ti}} + --c; if (c==0) goto{shortcut} + ti = load{link, ti} + if (ti > 0 or h==maxh) store{inds, j, ti} + else hash_remove{j,h} + } } } } @@ -494,8 +505,8 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n uniq:usz = 0 # Uniques inserted def dup = makelabel{} insert_all{ - {j,je,kr} => setlabel{dup}, - {found, i} => has_maxh |= found, + {j, h} => setlabel{dup}, + {found, i, j} => has_maxh |= found, dup, uniq } if (swap) { @@ -503,8 +514,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n memb_remove{uniq,has_maxh} # Remove values in ip from hash } try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, abort} - end := maxh>>sh # Clip trailing maxh if it shouldn't be in the table - while (load{hash,end}!=maxh) ++end + end := get_end{} # Clip trailing maxh if it shouldn't be in the table if (has_maxh) ++end # Don't clip lookup_all{{found, j} => promote{i8, swap ^ (found & (j>sh; while (load{hash,end}!=maxh) ++end - store{inds, end, (m-ind_maxh) &- has_maxh} + store{inds, get_end{}, (m-ind_maxh) &- has_maxh} lookup_all{{found, j} => i32~~(m - (load{inds, j} &- found))} } else { # swap # After insert_all, position i in rp contains: # - ≠𝕨, if i is the first occurrence of its value in 𝕩, or # - j-≠𝕩, where j>sh; while (load{hash,end}!=maxh) ++end - store{inds, end, ind_maxh} - } + if (has_maxh) { ++uniq; unsequester_maxh{inds} } ind_rev_lookup{uniq, has_maxh} # Propagate to later occurrences @for (r in rp over i to n) r = load{rp, min{i, u32~~r+n}} @@ -566,18 +567,18 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n {('⊒'), inds, link} => { store{link,0,0} + store{inds, maxh>>sh + cast_i{T,ext}, 0} def dup = makelabel{} - def set{j,je,{kr}} = { - # maxh entry either stores 0 or is the last entry and stores an - # index, so this preserves all values - ++je; store{inds, je, kr|load{inds,je}} - store{inds, j, 0} + def set{j, h} = { + store{inds, j, load{inds,j} &- (h==maxh)} setlabel{dup} i1 := i+1 store{link, i1, load{inds,j}} store{inds, j, i1} } - insert_all{set, {found,i}=>{}, dup} + def set_maxh{found, i, j} = sequester_maxh{j, found} + insert_all{set, set_maxh, dup} + unsequester_maxh{inds} if (not swap) prog_lookup{0} else prog_lookup{1} } } From 3a50cda5d3d23ce87341be1919b693dfcfe308a0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 10 Nov 2023 15:41:57 -0500 Subject: [PATCH 32/40] Specialize Index-of tables on result type --- src/builtins/search.c | 32 +++++++++++++++++++++----------- src/builtins/selfsearch.c | 5 +++-- src/singeli/src/bins.singeli | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index d8d88a1e..61a626d6 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -17,10 +17,12 @@ // p⊐n & n∊p with boolean p: based on ⊑p and p⊐¬⊑p // p⊐n & n∊p with short p: // AVX2 binary search when applicable +// SHOULD apply vector binary search to characters (sort as ints) // n⊸=¨p otherwise // ≤16-bit elements: lookup tables // 8-bit ∊ and ⊐: SSSE3 table // SHOULD make 8-bit NEON table +// SHOULD have branchy reverse 16-bit table search // 32- or 64-bit elements: hash tables // Store hash in table and not element; Robin Hood ordering; resizing // Reverse hash if searched-for is shorter @@ -35,9 +37,12 @@ #include "../utils/talloc.h" #include "../utils/calls.h" -// From selfsearch.c -extern NOINLINE void memset32(u32* p, u32 v, usz l); -extern NOINLINE void memset64(u64* p, u64 v, usz l); +extern NOINLINE void memset16(u16* p, u16 v, usz l) { for (usz i=0; i=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<64>>(xe-el_i8) && we<=xe && !elChr(TI(x,elType))) { + if (xia>=32 && wia>1 && el_i8<=xe && xe<=el_i32 && wia<(xe==el_i8?64:16) && we<=xe && !elChr(TI(x,elType))) { B g = C1(reverse, C1(gradeDown, incG(w))); w = C2(select, incG(g), w); switch (xe) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; } @@ -304,21 +309,26 @@ B indexOf_c2(B t, B w, B x) { } if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { + B r; #if SINGELI if (wia>256 && we==el_i8 && xe==el_i8) { TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256); void* fp = tyany_ptr(x); simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind); decG(w); - i32* rp; B r = m_i32arrc(&rp, x); - for (usz i=0; i Date: Sat, 11 Nov 2023 12:14:01 -0500 Subject: [PATCH 33/40] Eliminate a loop when resizing ordered hash --- src/singeli/src/hashtab.singeli | 11 ++++++----- src/singeli/src/search.singeli | 2 +- src/singeli/src/selfsearch.singeli | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 92c200b7..85c8142d 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -35,7 +35,7 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = { # Allocate and initialize resizing hash table # Initial size sz+ext and maximum size msz+ext # One region for each type in Ts initialized with value from v0s -def hash_alloc{logsz, msz, ext, Ts, v0s} = { +def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { def ws = each{width,Ts} each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment # Variables updated on resize @@ -56,15 +56,16 @@ def hash_alloc{logsz, msz, ext, Ts, v0s} = { def hash_resize{cc, m} = { dif := sz*((1< { p -= dif; memset{p, v, dif} }, ptrs, v0s} def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s - @for (j from dif to sz + ext) { - h := hash->j + @for (h in hash over j from dif to sz + ext) { if (h != h0) { hash <-{j} h0 - k0 := h>>sh; k := k0; while (hash->k!=h0) ++k + k0 := h>>sh + if (ordered) { k = max{k0, k+1} } # k0 can't be less than k + else { k = k0; while (hash->k!=h0) ++k } cc += cast_i{usz, k-k0} hash <-{k} h def move{p,v0} = { diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index beca3e04..a46e37fd 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -360,7 +360,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n maxh := T~~maxvalue{T} def aux = prim!='∊' def {tabs, sz, sh, hash_resize, hash_free} = hash_alloc{ - sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**'any'} + sl, msz, ext, tup{T, ...aux**u32}, tup{maxh, ...aux**'any'}, 1 } def {hash,...vals} = tabs diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index 834de1f3..49dc9948 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -25,7 +25,7 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { rp <-{0} res0 def aux = prim!='∊' def {{hash,...vals}, sz, sh, hash_resize, hash_free} = hash_alloc{ - sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0} + sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, 0 } # Resize or abort if more than 1/2^thresh collisions/element From 5957487b70465b80a279824145954cfadaebd7e1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 11 Nov 2023 16:37:00 -0500 Subject: [PATCH 34/40] Faster iteration over non-empty entries for hash table resizing --- src/singeli/src/hashtab.singeli | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 85c8142d..a67cff5f 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -60,8 +60,15 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { k:tupsel{0,Ts} = 0; --k each{{p,v} => { p -= dif; memset{p, v, dif} }, ptrs, v0s} def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s - @for (h in hash over j from dif to sz + ext) { - if (h != h0) { + s := sz+ext + i := dif + while (i < s) { + e := min{s-i, usz~~32} + b:u32 = 0 + @for_backwards (h in hash+i over e) b = 2*b + promote{u32, h!=h0} + while (b!=0) { + j := i + cast_i{usz,ctz{b}}; b &= b-1 + h := hash->j hash <-{j} h0 k0 := h>>sh if (ordered) { k = max{k0, k+1} } # k0 can't be less than k @@ -73,6 +80,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { } each{move, vals, v0r} } + i += e } } From ee7a11a2796ccf63823a44da2bc2706d91b5a396 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 12 Nov 2023 14:35:25 -0500 Subject: [PATCH 35/40] Unify self-search hash thresholds across primitives; doesn't seem to hurt performance --- src/singeli/src/selfsearch.singeli | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index 49dc9948..9aacc666 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -31,16 +31,13 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { # Resize or abort if more than 1/2^thresh collisions/element def {set_thresh, div_thresh} = { t:usz = 0 # Shift amount - def o = prim=='⊐' # Offset to allow effective t<0 - def div_thresh{i} = ((1<>t - def t0= has_radix*(prim=='∊') - def w = if (prim=='⊒') 1<<6 else wt<>t # Threshold setter, re-applied on resize def set_thresh{} = { - if (sz==msz) t = t0 - else if ((not has_radix) and sz>=(1<<24)/w) t = 0+o - else if ( sz>=(1<<20)/w) t = 3+o - else t = 5+o + if (sz==msz) t = 0 + else if ((not has_radix) and sz>=(1<<24)/wt) t = 0 + else if ( sz>=(1<<20)/wt) t = 3 + else t = 5 } tup{set_thresh, div_thresh} } From 09572de49aa035d30537d80e16cf711f007d21df Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 12 Nov 2023 16:17:19 -0500 Subject: [PATCH 36/40] Credible resizing policy for dyadic search hashes --- src/singeli/src/hashtab.singeli | 21 +++++++++++++++++--- src/singeli/src/search.singeli | 31 +++++++++++++++++------------- src/singeli/src/selfsearch.singeli | 26 +++++-------------------- 3 files changed, 41 insertions(+), 37 deletions(-) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index a67cff5f..a3335894 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -35,12 +35,13 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = { # Allocate and initialize resizing hash table # Initial size sz+ext and maximum size msz+ext # One region for each type in Ts initialized with value from v0s -def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { +def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { def ws = each{width,Ts} + def wt = tupsel{0,ws} each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment # Variables updated on resize sz := usz~~1 << logsz - sh := tupsel{0,ws} - logsz + sh := wt - logsz def add{}=0; def add{a,...r} = a+add{...r} halloc := talloc{u8, (msz+ext)*add{...ws/8}} @@ -56,6 +57,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { def hash_resize{cc, m} = { dif := sz*((1< { p -= dif; memset{p, v, dif} }, ptrs, v0s} @@ -84,5 +86,18 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, ordered} = { } } - tup{ptrs, sz, sh, hash_resize, {}=>tfree{halloc}} + # Test for resize if more than 1/2^t collisions/element + t:usz = 0 # Shift amount + def div_thresh{i} = i>>t + # Threshold setter, re-applied on resize + def set_thresh{} = { + if (sz==msz) t = 0 + else if ((not has_radix) and sz>=(1<<24)/wt) t = 0 + else if ( sz>=(1<<20)/wt) t = 3 + else t = 5 + } + tup{set_thresh, div_thresh} + set_thresh{} + + tup{ptrs, sz, sh, div_thresh, hash_resize, {}=>tfree{halloc}} } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index a46e37fd..c5c032b0 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -344,7 +344,6 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4) if (swap) swap_sides{} - def stop = m log := clzc{m} # Max size @@ -356,11 +355,12 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n # Filling e slots past the end requires e*(e+1)/2 collisions, so # m entries with <2 each can fill b, i+b, m} while (i < e) { def ii = if (prim!='⊒') i else m-i-1 @@ -395,15 +394,21 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n set_tab{j, h} ++i } - if (i == m) goto{insert_finish} - # Check collision counter and possibly resize (needs tuning) - if (cc >= stop and sz == msz) goto{abort} - if (16*cc >= stop and sz < msz) { - hash_resize{cc, 2} # Factor of 4 - if (cc >= stop) goto{abort} + # Check collision counter and possibly resize + def p64 = promote{u64,.} + dc := p64{cc} - p64{div_thresh{i}} + if (tern{i=0, sz=cc_stop or p64{n/4}*p64{cc} + rdc >= mm>>(5+log-(wt-sh))) { + hash_resize{cc, 2} # Factor of 4 + if (i==m and sz= cc_stop) { i=0; goto{abort} } + } } } - setlabel{insert_finish} } def get_end{} = { end := maxh>>sh diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index 9aacc666..fcf3b0b5 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -8,7 +8,7 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { def wt = width{T} def prim = to_prim{name} def has_radix = if (prim=='⊐') 0 else wt==32 - def {res0,stop} = if (prim=='∊') tup{1,n>>has_radix} else tup{0,2*n} + def {res0,cc_stop} = if (prim=='∊') tup{1,n>>has_radix} else tup{0,2*n} log := clzc{n} # Max size @@ -24,25 +24,10 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { x0 := hash_val{xp->0} rp <-{0} res0 def aux = prim!='∊' - def {{hash,...vals}, sz, sh, hash_resize, hash_free} = hash_alloc{ - sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, 0 + def {{hash,...vals}, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{ + sl, msz, ext, tup{T, ...aux**u32}, tup{x0, ...aux**0}, has_radix, 0 } - # Resize or abort if more than 1/2^thresh collisions/element - def {set_thresh, div_thresh} = { - t:usz = 0 # Shift amount - def div_thresh{i} = i>>t - # Threshold setter, re-applied on resize - def set_thresh{} = { - if (sz==msz) t = 0 - else if ((not has_radix) and sz>=(1<<24)/wt) t = 0 - else if ( sz>=(1<<20)/wt) t = 3 - else t = 5 - } - tup{set_thresh, div_thresh} - } - set_thresh{} - def {output, write_res} = match (prim) { {('∊')} => tup{{b}=>b, {j,h,k,x0} => { hash<-{j}h; k!=h }} {('⊒')} => { @@ -83,10 +68,9 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { if (sz == msz) goto{break} # Abort if (has_radix and i < n/2 and sz >= 1<<18) goto{break} # Avoid resizing if close to the end - if (cc>=stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) { + if (cc>=cc_stop or p64{n-i}*dc >= (p64{i}*p64{n+i})>>(5+log-(wt-sh))) { hash_resize{cc, 2} # Factor of 4 - if (cc >= stop) goto{break} - set_thresh{} + if (cc >= cc_stop) goto{break} } } } From 5910c93fcb33c799c03c316d0c3138eeab13c758 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 12 Nov 2023 23:37:47 +0200 Subject: [PATCH 37/40] silence some warnings & errors in various build types --- src/builtins/search.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 61a626d6..2f8e57da 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -82,7 +82,7 @@ extern B join_c2(B,B,B); extern B select_c2(B,B,B); B asNormalized(B x, usz n, bool nanBad); -SHOULD_INLINE bool canCompare64_norm(B* w, usz wia, B* x, usz xia) { +SHOULD_INLINE bool canCompare64_norm2(B* w, usz wia, B* x, usz xia) { B wn=asNormalized(*w,wia,true); if (wn.u == m_f64(0).u) return 0; *w=wn; B xn=asNormalized(*x,xia,true); if (xn.u == m_f64(0).u) return 0; *x=xn; return 1; @@ -244,7 +244,7 @@ static NOINLINE usz indexOfOne(B l, B e) { } else B indexOf_c2(B t, B w, B x) { - bool split = 0; + bool split = 0; (void) split; if (RARE(!isArr(w) || RNK(w)!=1)) { split = 1; B2 t = splitCells(x, w, 1); @@ -331,7 +331,7 @@ B indexOf_c2(B t, B w, B x) { return r; } #if SINGELI - if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm(&w,wia,&x,xia))))) { + if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) { i32* rp; B r = m_i32arrc(&rp, x); if (si_indexOf_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia)) { decG(w); decG(x); return reduceI32Width(r, wia); @@ -357,7 +357,7 @@ B indexOf_c2(B t, B w, B x) { B enclosed_0, enclosed_1; B memberOf_c2(B t, B w, B x) { - bool split = 0; + bool split = 0; (void) split; if (isAtm(x) || RNK(x)!=1) { split = 1; B2 t = splitCells(w, x, false); @@ -438,7 +438,7 @@ B memberOf_c2(B t, B w, B x) { return taga(cpyBitArr(r)); } #if SINGELI - if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm(&w,wia,&x,xia))))) { + if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) { i8* rp; B r = m_i8arrc(&rp, w); if (si_memberOf_c2_hash[we-el_i32](rp, tyany_ptr(x), xia, tyany_ptr(w), wia)) { decG(w); decG(x); return taga(cpyBitArr(r)); @@ -465,7 +465,7 @@ B memberOf_c2(B t, B w, B x) { #undef CHECK_CHRS_ELSE B count_c2(B t, B w, B x) { - bool split = 0; + bool split = 0; (void) split; if (RARE(!isArr(w) || RNK(w)!=1)) { split = 1; B2 t = splitCells(x, w, 2); @@ -508,7 +508,7 @@ B count_c2(B t, B w, B x) { goto el8or16; } else { #if SINGELI - if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm(&w,wia,&x,xia)))) && + if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia)))) && si_count_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia, wnext)) { goto dec_nwx; } @@ -529,7 +529,9 @@ B count_c2(B t, B w, B x) { } free_b2i(map); } + #if SINGELI dec_nwx:; + #endif TFREE(wnext); decG(w); decG(x); return reduceI32Width(r, wia); } From 4fb6ce602fcca86dc26cc0dcc15d165d652055fe Mon Sep 17 00:00:00 2001 From: dzaima Date: Mon, 13 Nov 2023 12:56:48 +0200 Subject: [PATCH 38/40] add a require{} for CRC32 --- src/singeli/src/hashtab.singeli | 1 + 1 file changed, 1 insertion(+) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index a3335894..ff24f349 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -28,6 +28,7 @@ def hash_val{x0:u64} = { x ^= x >> 33; x } # CRC32 +if (hasarch{'SSE4.2'}) require{'x86intrin.h'} def hash_val{x:u32 & hasarch{'SSE4.2'}} = { emit{u32, '_mm_crc32_u32', 0x973afb51, x} } From c94d51dcafb3d7da5cbc24f59f4b0396de4b88f5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 13 Nov 2023 08:24:51 -0500 Subject: [PATCH 39/40] Fixes for 64-bit usz --- src/builtins/search.c | 6 +++--- src/singeli/src/hashtab.singeli | 2 +- src/singeli/src/search.singeli | 20 +++++++++++--------- src/singeli/src/selfsearch.singeli | 9 +++++---- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 2f8e57da..5c6f9a23 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -331,7 +331,7 @@ B indexOf_c2(B t, B w, B x) { return r; } #if SINGELI - if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) { + if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia))))) { i32* rp; B r = m_i32arrc(&rp, x); if (si_indexOf_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia)) { decG(w); decG(x); return reduceI32Width(r, wia); @@ -508,8 +508,8 @@ B count_c2(B t, B w, B x) { goto el8or16; } else { #if SINGELI - if (we==xe && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia)))) && - si_count_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia, wnext)) { + if (we==xe && wia<=INT32_MAX && (we==el_i32 || (we==el_f64 && (split || canCompare64_norm2(&w,wia,&x,xia)))) && + si_count_c2_hash[we-el_i32](rp, tyany_ptr(w), wia, tyany_ptr(x), xia, (u32*)wnext)) { goto dec_nwx; } #endif diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index ff24f349..4c34caf4 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -76,7 +76,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { k0 := h>>sh if (ordered) { k = max{k0, k+1} } # k0 can't be less than k else { k = k0; while (hash->k!=h0) ++k } - cc += cast_i{usz, k-k0} + cc += cast_i{ux, k-k0} hash <-{k} h def move{p,v0} = { v := p->j; if (not same{v0,'any'}) p <-{j} v0; p <-{k} v diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index c5c032b0..1a0739c6 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -337,9 +337,11 @@ oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 def rty{name} = if (to_prim{name}=='∊') i8 else i32 def ity{name} = (to_prim{name}=='⊒')**(*u32) -fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{name}) = { - def wt = width{T} +fn hashtab{T, name}(rp:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = { def prim = to_prim{name} + def U = if (prim=='∊') usz else u32 + m := cast_i{U,mi}; n := cast_i{U,ni} + def wt = width{T} ip := *T~~iv; fp := *T~~fv def swap_sides{} = each{{a,b}=>{t:=a; a=b; b=t}, tup{ip,m}, tup{fp,n}} swap:u1 = n+(1024*(prim!='⊒')) < (if (prim!='∊') m else m-m/4) @@ -351,12 +353,12 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n msz := usz~~1 << msl # Starting log-size (try_vec_memb requires size>4) sl := msl; if (msl>=14) sl = 12+(msl&1) - b:usz = 64 # Block size + b:U = 64 # Block size # Filling e slots past the end requires e*(e+1)/2 collisions, so # m entries with <2 each can fill b, i+b, m} while (i < e) { @@ -387,7 +389,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n each{store{.,je-c,.}, tabs, kv} each{=, kv, knv} } while (k != maxh) - cc += cast_i{usz, je-j0} + cc += cast_i{u64, je-j0} } each{{u} => { u += promote{usz,h!=maxh} }, uniq} store{hash, j, h} @@ -474,7 +476,7 @@ fn hashtab{T, name}(rp:*rty{name}, iv:*void, m:usz, fv:*void, n:usz, links:ity{n def prog_lookup{swap} = { # Progressive Index-of lookup def rev{a,b} = if (swap) tup{b,a} else tup{a,b} memset{*u32~~rp, ...rev{m,n}} - c:usz = m; def shortcut = makelabel{} + c := m; def shortcut = makelabel{} @for (fp over i to n) { h := hash_val{fp}; j := h>>sh k := load{hash,j} diff --git a/src/singeli/src/selfsearch.singeli b/src/singeli/src/selfsearch.singeli index fcf3b0b5..fd789b63 100644 --- a/src/singeli/src/selfsearch.singeli +++ b/src/singeli/src/selfsearch.singeli @@ -8,7 +8,8 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { def wt = width{T} def prim = to_prim{name} def has_radix = if (prim=='⊐') 0 else wt==32 - def {res0,cc_stop} = if (prim=='∊') tup{1,n>>has_radix} else tup{0,2*n} + n64 := promote{u64,n} + def {res0,cc_stop} = if (prim=='∊') tup{1,n64>>has_radix} else tup{0,2*n64} log := clzc{n} # Max size @@ -50,20 +51,20 @@ fn selfhashtab{T, name}(rp:*rty{name}, xp:*T, n:usz) = { } def break=makelabel{} - cc:usz = 0 # Collision counter + cc:u64 = 0 # Collision counter i:usz=1; while (1) { e := tern{n-i>b, i+b, n} while (i < e) { h := hash_val{xp->i}; j0 := h>>sh; j := j0 k:=undefined{T}; while (((k=hash->j)!=h) & (k!=x0)) ++j - cc += cast_i{usz, j-j0} + cc += cast_i{u64, j-j0} rp <-{i} write_res{j,h,k,x0} ++i } if (i == n) goto{break} # Check collision counter and possibly resize def p64 = promote{u64,.} - dc := p64{cc} - p64{div_thresh{i}} + dc := cc - p64{div_thresh{i}} if (i64~~dc >= 0) { if (sz == msz) goto{break} # Abort if (has_radix and i < n/2 and sz >= 1<<18) goto{break} From b4e53abbe4253f3dff212a9838633957f5ad5c34 Mon Sep 17 00:00:00 2001 From: dzaima Date: Mon, 13 Nov 2023 16:14:04 +0200 Subject: [PATCH 40/40] some comments --- src/singeli/src/hashtab.singeli | 10 +++++++--- src/singeli/src/search.singeli | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 4c34caf4..411a8d5d 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -36,6 +36,7 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = { # Allocate and initialize resizing hash table # Initial size sz+ext and maximum size msz+ext # One region for each type in Ts initialized with value from v0s +# Allocates the maximum at the start, resizes downwards within the existing allocation def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { def ws = each{width,Ts} def wt = tupsel{0,ws} @@ -56,15 +57,18 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = { each{memset{., ., sze}, ptrs, v0s} def hash_resize{cc, m} = { - dif := sz*((1< { p -= dif; memset{p, v, dif} }, ptrs, v0s} def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s s := sz+ext i := dif + + # Iterate over existing elements in blocks of e<=32 + # Branchlessly skips over h0 while (i < s) { e := min{s-i, usz~~32} b:u32 = 0 diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 1a0739c6..8b44ec61 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -338,6 +338,7 @@ oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 def rty{name} = if (to_prim{name}=='∊') i8 else i32 def ity{name} = (to_prim{name}=='⊒')**(*u32) fn hashtab{T, name}(rp:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = { + # iv,mi/ip,m - searched-in; fv,ni/fp,n - searched-for; may get swapped around & back def prim = to_prim{name} def U = if (prim=='∊') usz else u32 m := cast_i{U,mi}; n := cast_i{U,ni}