From 8edc965686c4e42dba849d0c26d1cece13c2df4e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 24 Nov 2023 13:35:46 -0500 Subject: [PATCH 01/11] =?UTF-8?q?=E2=80=A2HashMap=20stub=20with=20linear?= =?UTF-8?q?=20lookup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins.h | 1 + src/builtins/sysfn.c | 33 +++++++++++++++++++++++++++++++++ test/cases/hash.bqn | 4 ++++ 3 files changed, 38 insertions(+) diff --git a/src/builtins.h b/src/builtins.h index a6109e04..2688661e 100644 --- a/src/builtins.h +++ b/src/builtins.h @@ -15,6 +15,7 @@ /* sysfn.c*/D(nGet,"•ns.Get") D(nHas,"•ns.Has") M(nKeys,"•ns.Keys") \ /* sysfn.c*/M(fName,"•file.Name") M(fParent,"•file.Parent") \ /* sysfn.c*/M(tRawMode,"•term.RawMode") M(tFlush,"•term.Flush") M(tCharB,"•term.CharB") M(tCharN,"•term.CharN") M(tOutRaw,"•term.OutRaw") M(tErrRaw,"•term.ErrRaw") \ +/* sysfn.c*/D(hashMap,"•HashMap") \ /* inverse.c*/M(setInvReg,"(SetInvReg)") M(setInvSwap,"(SetInvSwap)") M(nativeInvReg,"(NativeInvReg)") M(nativeInvSwap,"(NativeInvSwap)") \ /*internal.c*/M(itype,"•internal.Type") M(elType,"•internal.ElType") M(refc,"•internal.Refc") M(isPure,"•internal.IsPure") A(info,"•internal.Info") \ /*internal.c*/M(heapDump,"•internal.HeapDump") M(internalGC,"•internal.GC") M(heapStats,"•internal.HeapStats") \ diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index f11264d1..3049cc09 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -707,6 +707,38 @@ B currentError_c1(B t, B x) { B currentError_c1(B t, B x) { thrM("•CurrentError: No errors as error catching has been disabled"); } #endif +static Body* hashmap_ns; +static B hashmap_getName; static NFnDesc* hashmap_getDesc; +B hashmap_get_c2(B t, B w, B x) { + Scope* sc = c(NS,nfn_objU(t))->sc; + B k = sc->vars[0]; + SGetU(k) + usz l = IA(k); + for (usz i=0; ivars[1], i); + } + if (q_N(w)) thrM("(hashmap).Get: key not found"); + dec(x); return w; +} +B hashmap_get_c1(B t, B x) { return hashmap_get_c2(t, bi_N, x); } +static NOINLINE void hashmap_init() { + hashmap_ns = m_nnsDesc("keys", "vals", "get"); + NSDesc* d = hashmap_ns->nsDesc; + d->expGIDs[0] = d->expGIDs[1] = -1; + hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); +} +B hashMap_c2(B t, B w, B x) { + if (!isArr(w) || RNK(w)!=1 || !isArr(x) || RNK(x)!=1) thrF("•HashMap: Arguments must be lists (%H≡≢𝕨, %H≡≢𝕩)", w, x); + usz n = IA(w); + if (n != IA(x)) thrF("•HashMap: 𝕨 and 𝕩 must have the same length (%s≡≠𝕨, %s≡≠𝕩)", n, IA(x)); + if (hashmap_ns==NULL) hashmap_init(); + B ns = m_nns(hashmap_ns, w, x, m_nfn(hashmap_getDesc, bi_N)); + Scope* sc = c(NS,ns)->sc; + for (i32 i = 2; i < 3; i++) nfn_swapObj(sc->vars[i], incG(ns)); + return ns; +} + static NFnDesc* fileAtDesc; B fileAt_c1(B d, B x) { return path_rel(nfn_objU(d), x, "•file.At"); @@ -1623,6 +1655,7 @@ static Body* file_nsGen; F("fromutf8", U"•FromUTF8", bi_fromUtf8) \ F("toutf8", U"•ToUTF8", bi_toUtf8) \ F("currenterror", U"•CurrentError", bi_currentError) \ + F("hashmap", U"•HashMap", bi_hashMap) \ F("math", U"•math", tag(0,VAR_TAG)) \ F("rand", U"•rand", tag(1,VAR_TAG)) \ F("term", U"•term", tag(2,VAR_TAG)) \ diff --git a/test/cases/hash.bqn b/test/cases/hash.bqn index e1d4c982..2baaf73a 100644 --- a/test/cases/hash.bqn +++ b/test/cases/hash.bqn @@ -44,3 +44,7 @@ a←(•ParseFloat¨ "1.2"‿"-0"‿"0")∾-⊸⋈0÷0 ⋄ ! (0‿3‿3‿3⊏a) !"∊: Rank of 𝕨 must be at least the cell rank of 𝕩 (⟨2⟩ ≡ ≢𝕨, 2‿2‿2 ≡ ≢𝕩)" % 1‿2∊2‿2‿2⥊1‿2 !"⊐: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊐1‿2 !"⊒: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊒1‿2 + +⟨"get"⟩ %% •ns.Keys "abc"‿"de"‿"fgh" •HashMap ↕3 +⟨2⟩ %% ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fgh" +∞ %% ∞ ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fghi" From 77bfc7dd1a1eccc54de3444d99448a3d19372edf Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 25 Nov 2023 22:15:19 -0500 Subject: [PATCH 02/11] Non-resizing static Robin Hood hashmap --- src/builtins/sysfn.c | 58 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 3049cc09..6fca306e 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -707,35 +707,75 @@ B currentError_c1(B t, B x) { B currentError_c1(B t, B x) { thrM("•CurrentError: No errors as error catching has been disabled"); } #endif +typedef struct HashMap { + struct Value; + u64 pop; // count of defined entries + u64 sh; // shift to turn hash into index + u64 sz; // count of allocated entries, a power of 2 + u64 a[]; +} HashMap; static Body* hashmap_ns; static B hashmap_getName; static NFnDesc* hashmap_getDesc; B hashmap_get_c2(B t, B w, B x) { Scope* sc = c(NS,nfn_objU(t))->sc; + HashMap* map = c(HashMap, sc->vars[2]); + u64* hp = map->a; + u64 h = bqn_hash(x, wy_secret); + u64 m = ~(u32)0; + u64 v = h &~ m; + u64 j = h >> map->sh; + u64 u; while ((u=hp[j]) < v) j++; B k = sc->vars[0]; - SGetU(k) - usz l = IA(k); - for (usz i=0; ivars[1], i); + while (u < (v|m)) { + usz i = u&m; + if (equal(x, IGetU(k, i))) { + dec(x); dec(w); + return IGet(sc->vars[1], i); + } + u = hp[++j]; } if (q_N(w)) thrM("(hashmap).Get: key not found"); dec(x); return w; } B hashmap_get_c1(B t, B x) { return hashmap_get_c2(t, bi_N, x); } static NOINLINE void hashmap_init() { - hashmap_ns = m_nnsDesc("keys", "vals", "get"); + hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get"); NSDesc* d = hashmap_ns->nsDesc; - d->expGIDs[0] = d->expGIDs[1] = -1; + for (usz i = 0; i < 3; i++) d->expGIDs[i] = -1; hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); } +B hashmap_build(B keys, usz n) { + usz ext = 32; + usz sh = CLZ(n|16)-1; + u64 l = (u64)1 << (64-sh); + HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,l+ext), t_hashmap); + map->pop = n; map->sh = sh; map->sz = l; + u64* hp = map->a; + SGetU(keys) + u64 e = ~(u64)0; + u64 m = ~(u32)0; + for (u64 j=0; j>sh; u64 j = j0; + u64 u; while ((u=hp[j]) < v) j++; + if (u < (v|m) && equal(key, GetU(keys,u&m))) thrM("•HashMap: 𝕨 contained duplicate keys"); + u64 je=j; while (u!=e) { u64 s=u; je++; u=hp[je]; hp[je]=s; } + hp[j] = v; + } + return tag(map, OBJ_TAG); +} B hashMap_c2(B t, B w, B x) { if (!isArr(w) || RNK(w)!=1 || !isArr(x) || RNK(x)!=1) thrF("•HashMap: Arguments must be lists (%H≡≢𝕨, %H≡≢𝕩)", w, x); usz n = IA(w); if (n != IA(x)) thrF("•HashMap: 𝕨 and 𝕩 must have the same length (%s≡≠𝕨, %s≡≠𝕩)", n, IA(x)); if (hashmap_ns==NULL) hashmap_init(); - B ns = m_nns(hashmap_ns, w, x, m_nfn(hashmap_getDesc, bi_N)); + B h = hashmap_build(w, n); + B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N)); Scope* sc = c(NS,ns)->sc; - for (i32 i = 2; i < 3; i++) nfn_swapObj(sc->vars[i], incG(ns)); + for (usz i = 3; i < 4; i++) nfn_swapObj(sc->vars[i], incG(ns)); return ns; } From 96c7f2eb1a4a3cdc4bb5373bf6d7c7b2d6486b08 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 27 Nov 2023 19:14:32 -0500 Subject: [PATCH 03/11] =?UTF-8?q?Move=20=E2=80=A2HashMap=20implementation?= =?UTF-8?q?=20details=20to=20search.c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 55 +++++++++++++++++++++++++++++++++++++++++ src/builtins/sysfn.c | 57 ++++++------------------------------------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 5c6f9a23..90036edd 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -601,3 +601,58 @@ void search_init(void) { getRange_fns[4] = getRange_f64; #endif } + + +// •HashMap implementation +typedef struct HashMap { + struct Value; + u64 pop; // count of defined entries + u64 sh; // shift to turn hash into index + u64 sz; // count of allocated entries, a power of 2 + u64 a[]; +} HashMap; + +B hashmap_build(B keys, usz n) { + usz ext = 32; + usz sh = CLZ(n|16)-1; + u64 l = (u64)1 << (64-sh); + HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,l+ext), t_hashmap); + map->pop = n; map->sh = sh; map->sz = l; + u64* hp = map->a; + SGetU(keys) + u64 e = ~(u64)0; + u64 m = ~(u32)0; + memset64(hp, e, l+ext); + for (usz i=0; i>sh; u64 j = j0; + u64 u; while ((u=hp[j]) < v) j++; + if (u < (v|m) && equal(key, GetU(keys,u&m))) thrM("•HashMap: 𝕨 contained duplicate keys"); + u64 je=j; while (u!=e) { u64 s=u; je++; u=hp[je]; hp[je]=s; } + hp[j] = v; + } + return tag(map, OBJ_TAG); +} + +B hashmap_lookup(B* vars, B w, B x) { + HashMap* map = c(HashMap, vars[2]); + u64* hp = map->a; + u64 h = bqn_hash(x, wy_secret); + u64 m = ~(u32)0; + u64 v = h &~ m; + u64 j = h >> map->sh; + u64 u; while ((u=hp[j]) < v) j++; + B k = vars[0]; + while (u < (v|m)) { + usz i = u&m; + if (equal(x, IGetU(k, i))) { + dec(x); dec(w); + return IGet(vars[1], i); + } + u = hp[++j]; + } + if (q_N(w)) thrM("(hashmap).Get: key not found"); + dec(x); return w; +} diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 6fca306e..0a22855f 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -707,66 +707,23 @@ B currentError_c1(B t, B x) { B currentError_c1(B t, B x) { thrM("•CurrentError: No errors as error catching has been disabled"); } #endif -typedef struct HashMap { - struct Value; - u64 pop; // count of defined entries - u64 sh; // shift to turn hash into index - u64 sz; // count of allocated entries, a power of 2 - u64 a[]; -} HashMap; static Body* hashmap_ns; static B hashmap_getName; static NFnDesc* hashmap_getDesc; +// Hash object handling defined in search.c +extern B hashmap_build(B keys, usz n); +extern B hashmap_lookup(B* vars, B w, B x); B hashmap_get_c2(B t, B w, B x) { - Scope* sc = c(NS,nfn_objU(t))->sc; - HashMap* map = c(HashMap, sc->vars[2]); - u64* hp = map->a; - u64 h = bqn_hash(x, wy_secret); - u64 m = ~(u32)0; - u64 v = h &~ m; - u64 j = h >> map->sh; - u64 u; while ((u=hp[j]) < v) j++; - B k = sc->vars[0]; - while (u < (v|m)) { - usz i = u&m; - if (equal(x, IGetU(k, i))) { - dec(x); dec(w); - return IGet(sc->vars[1], i); - } - u = hp[++j]; - } - if (q_N(w)) thrM("(hashmap).Get: key not found"); - dec(x); return w; + return hashmap_lookup(c(NS,nfn_objU(t))->sc->vars, w, x); +} +B hashmap_get_c1(B t, B x) { + return hashmap_lookup(c(NS,nfn_objU(t))->sc->vars, bi_N, x); } -B hashmap_get_c1(B t, B x) { return hashmap_get_c2(t, bi_N, x); } static NOINLINE void hashmap_init() { hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get"); NSDesc* d = hashmap_ns->nsDesc; for (usz i = 0; i < 3; i++) d->expGIDs[i] = -1; hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); } -B hashmap_build(B keys, usz n) { - usz ext = 32; - usz sh = CLZ(n|16)-1; - u64 l = (u64)1 << (64-sh); - HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,l+ext), t_hashmap); - map->pop = n; map->sh = sh; map->sz = l; - u64* hp = map->a; - SGetU(keys) - u64 e = ~(u64)0; - u64 m = ~(u32)0; - for (u64 j=0; j>sh; u64 j = j0; - u64 u; while ((u=hp[j]) < v) j++; - if (u < (v|m) && equal(key, GetU(keys,u&m))) thrM("•HashMap: 𝕨 contained duplicate keys"); - u64 je=j; while (u!=e) { u64 s=u; je++; u=hp[je]; hp[je]=s; } - hp[j] = v; - } - return tag(map, OBJ_TAG); -} B hashMap_c2(B t, B w, B x) { if (!isArr(w) || RNK(w)!=1 || !isArr(x) || RNK(x)!=1) thrF("•HashMap: Arguments must be lists (%H≡≢𝕨, %H≡≢𝕩)", w, x); usz n = IA(w); From 016e428db65bfb6e11819be30ae34920eb369934 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 27 Nov 2023 19:19:24 -0500 Subject: [PATCH 04/11] =?UTF-8?q?Always=20store=20=E2=80=A2HashMap=20keys?= =?UTF-8?q?=20and=20values=20as=20HArrs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 12 ++++++------ src/builtins/sysfn.c | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 90036edd..22f05477 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -619,17 +619,17 @@ B hashmap_build(B keys, usz n) { HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,l+ext), t_hashmap); map->pop = n; map->sh = sh; map->sz = l; u64* hp = map->a; - SGetU(keys) + B* kp = harr_ptr(keys); u64 e = ~(u64)0; u64 m = ~(u32)0; memset64(hp, e, l+ext); for (usz i=0; i>sh; u64 j = j0; u64 u; while ((u=hp[j]) < v) j++; - if (u < (v|m) && equal(key, GetU(keys,u&m))) thrM("•HashMap: 𝕨 contained duplicate keys"); + if (u < (v|m) && equal(key, kp[u&m])) thrM("•HashMap: 𝕨 contained duplicate keys"); u64 je=j; while (u!=e) { u64 s=u; je++; u=hp[je]; hp[je]=s; } hp[j] = v; } @@ -644,12 +644,12 @@ B hashmap_lookup(B* vars, B w, B x) { u64 v = h &~ m; u64 j = h >> map->sh; u64 u; while ((u=hp[j]) < v) j++; - B k = vars[0]; + B* k = harr_ptr(vars[0]); // keys while (u < (v|m)) { usz i = u&m; - if (equal(x, IGetU(k, i))) { + if (equal(x, k[i])) { dec(x); dec(w); - return IGet(vars[1], i); + return inc(harr_ptr(vars[1])[i]); } u = hp[++j]; } diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 0a22855f..a55bcc41 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -729,6 +729,7 @@ B hashMap_c2(B t, B w, B x) { usz n = IA(w); if (n != IA(x)) thrF("•HashMap: 𝕨 and 𝕩 must have the same length (%s≡≠𝕨, %s≡≠𝕩)", n, IA(x)); if (hashmap_ns==NULL) hashmap_init(); + w = taga(toHArr(w)); x = taga(toHArr(x)); B h = hashmap_build(w, n); B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N)); Scope* sc = c(NS,ns)->sc; From ce9f460e2cb4d97dbf15d3dc5cc5443935e66bad Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 27 Nov 2023 21:46:51 -0500 Subject: [PATCH 05/11] =?UTF-8?q?=E2=80=A2HashMap=20count=20and=20set=20fu?= =?UTF-8?q?nctions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 109 +++++++++++++++++++++++++++++------------- src/builtins/sysfn.c | 33 +++++++++---- test/cases/hash.bqn | 3 +- 3 files changed, 102 insertions(+), 43 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 22f05477..b6f580a0 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -611,48 +611,93 @@ typedef struct HashMap { u64 sz; // count of allocated entries, a power of 2 u64 a[]; } HashMap; +static u64 hashmap_size(usz sh) { return ((u64)1 << (64-sh)) + 32; } +static const u64 empty = ~(u64)0; +static const u64 hmask = ~(u32)0; -B hashmap_build(B keys, usz n) { - usz ext = 32; +usz hashmap_count(B hash) { return c(HashMap, hash)->pop; } + +HashMap* hashmap_alloc(usz pop, usz sh, u64 sz) { + HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,sz), t_hashmap); + map->pop = pop; map->sh = sh; map->sz = sz; + memset64(map->a, empty, sz); + return map; +} + +NOINLINE HashMap* hashmap_resize(HashMap* old_map) { + usz sh = old_map->sh - 1; + if (sh < 32) thrM("•HashMap: hash size maximum exceeded"); + u64 sz = hashmap_size(sh); + HashMap* map = hashmap_alloc(old_map->pop, sh, sz); + for (u64 i=0, j=-(u64)1; isz; i++) { + u64 h = old_map->a[i]; + if (h == empty) continue; + j++; u64 jh = h>>sh; if (j < jh) j = jh; + map->a[j] = h; + } + mm_free((Value*)old_map); + return map->a[sz-1]==empty ? map : hashmap_resize(map); +} + +// Expects already defined: u64* hp, u64 sh, B* keys +#define HASHMAP_FIND(X, FOUND) \ + u64 h = bqn_hash(X, wy_secret); \ + u64 m = hmask; \ + u64 v = h &~ m; \ + u64 j = h >> sh; \ + u64 u; while ((u=hp[j]) < v) j++; \ + while (u < (v|m)) { \ + usz i = u&m; \ + if (equal(X, keys[i])) { FOUND } \ + u = hp[++j]; \ + } +// Expects usz i to be the index if new (i in FOUND is index where found) +#define HASHMAP_INSERT(X, FOUND) \ + HASHMAP_FIND(X, FOUND) \ + u64 je=j; while (u!=empty) { u64 s=u; je++; u=hp[je]; hp[je]=s; } \ + hp[j] = v | i; + +B hashmap_build(B key_arr, usz n) { usz sh = CLZ(n|16)-1; - u64 l = (u64)1 << (64-sh); - HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,l+ext), t_hashmap); - map->pop = n; map->sh = sh; map->sz = l; + u64 sz = hashmap_size(sh); + HashMap* map = hashmap_alloc(n, sh, sz); u64* hp = map->a; - B* kp = harr_ptr(keys); - u64 e = ~(u64)0; - u64 m = ~(u32)0; - memset64(hp, e, l+ext); + B* keys = harr_ptr(key_arr); for (usz i=0; i>sh; u64 j = j0; - u64 u; while ((u=hp[j]) < v) j++; - if (u < (v|m) && equal(key, kp[u&m])) thrM("•HashMap: 𝕨 contained duplicate keys"); - u64 je=j; while (u!=e) { u64 s=u; je++; u=hp[je]; hp[je]=s; } - hp[j] = v; + B key = keys[i]; + HASHMAP_INSERT(key, thrM("•HashMap: 𝕨 contained duplicate keys");) + if (RARE(je == sz-1)) { + map=hashmap_resize(map); + hp=map->a; sh=map->sh; sz=map->sz; + } } return tag(map, OBJ_TAG); } B hashmap_lookup(B* vars, B w, B x) { HashMap* map = c(HashMap, vars[2]); - u64* hp = map->a; - u64 h = bqn_hash(x, wy_secret); - u64 m = ~(u32)0; - u64 v = h &~ m; - u64 j = h >> map->sh; - u64 u; while ((u=hp[j]) < v) j++; - B* k = harr_ptr(vars[0]); // keys - while (u < (v|m)) { - usz i = u&m; - if (equal(x, k[i])) { - dec(x); dec(w); - return inc(harr_ptr(vars[1])[i]); - } - u = hp[++j]; - } + u64* hp = map->a; u64 sh = map->sh; + B* keys = harr_ptr(vars[0]); + HASHMAP_FIND(x, dec(w); dec(x); return inc(harr_ptr(vars[1])[i]);) if (q_N(w)) thrM("(hashmap).Get: key not found"); dec(x); return w; } + +void hashmap_set(B* vars, B w, B x) { + HashMap* map = c(HashMap, vars[2]); + u64* hp = map->a; u64 sh = map->sh; + usz i = map->pop; + if (i>>(64-3-sh)>7 || hp[map->sz-1]!=empty) { // keep load <= 7/8 + map=hashmap_resize(map); + vars[2] = tag(map, OBJ_TAG); + hp=map->a; sh=map->sh; + } + B* keys = harr_ptr(vars[0]); + HASHMAP_INSERT( + w, + B* s = harr_ptr(vars[1])+i; dec(*s); dec(w); *s=x; return; + ) + map->pop++; + vars[0] = vec_addN(vars[0], w); + vars[1] = vec_addN(vars[1], x); +} diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index a55bcc41..493f686a 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -708,21 +708,34 @@ B currentError_c1(B t, B x) { thrM("•CurrentError: No errors as error catching #endif static Body* hashmap_ns; -static B hashmap_getName; static NFnDesc* hashmap_getDesc; +static B hashmap_getName; static NFnDesc* hashmap_getDesc; +static B hashmap_hasName; static NFnDesc* hashmap_hasDesc; +static B hashmap_setName; static NFnDesc* hashmap_setDesc; +static B hashmap_countName; static NFnDesc* hashmap_countDesc; // Hash object handling defined in search.c extern B hashmap_build(B keys, usz n); extern B hashmap_lookup(B* vars, B w, B x); -B hashmap_get_c2(B t, B w, B x) { - return hashmap_lookup(c(NS,nfn_objU(t))->sc->vars, w, x); -} -B hashmap_get_c1(B t, B x) { - return hashmap_lookup(c(NS,nfn_objU(t))->sc->vars, bi_N, x); +extern void hashmap_set(B* vars, B w, B x); +extern usz hashmap_count(B hash); +#define VARS c(NS,nfn_objU(t))->sc->vars +B hashmap_get_c1(B t, B x ) { return hashmap_lookup(VARS, bi_N, x); } +B hashmap_get_c2(B t, B w, B x) { return hashmap_lookup(VARS, w, x); } +B hashmap_set_c2(B t, B w, B x) { hashmap_set(VARS, w, x); return inc(nfn_objU(t)); } +B hashmap_count_c1(B t, B x) { dec(x); return m_usz(hashmap_count(VARS[2])); } +B hashmap_has_c1(B t, B x) { + B l = hashmap_lookup(VARS, bi_noVar, x); + if (l.u==bi_noVar.u) return m_f64(0); + dec(l); return m_f64(1); } +#undef VARS static NOINLINE void hashmap_init() { - hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get"); + hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get", "has", "set", "count"); NSDesc* d = hashmap_ns->nsDesc; for (usz i = 0; i < 3; i++) d->expGIDs[i] = -1; - hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); + hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); + hashmap_hasName = m_c8vec_0("has"); gc_add(hashmap_hasName); hashmap_hasDesc = registerNFn(m_c8vec_0("(hashmap).Has"), hashmap_has_c1, c2_bad); + hashmap_setName = m_c8vec_0("set"); gc_add(hashmap_setName); hashmap_setDesc = registerNFn(m_c8vec_0("(hashmap).Set"), c1_bad, hashmap_set_c2); + hashmap_countName = m_c8vec_0("count"); gc_add(hashmap_countName); hashmap_countDesc = registerNFn(m_c8vec_0("(hashmap).Count"), hashmap_count_c1, c2_bad); } B hashMap_c2(B t, B w, B x) { if (!isArr(w) || RNK(w)!=1 || !isArr(x) || RNK(x)!=1) thrF("•HashMap: Arguments must be lists (%H≡≢𝕨, %H≡≢𝕩)", w, x); @@ -731,9 +744,9 @@ B hashMap_c2(B t, B w, B x) { if (hashmap_ns==NULL) hashmap_init(); w = taga(toHArr(w)); x = taga(toHArr(x)); B h = hashmap_build(w, n); - B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N)); + B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N), m_nfn(hashmap_hasDesc, bi_N), m_nfn(hashmap_setDesc, bi_N), m_nfn(hashmap_countDesc, bi_N)); Scope* sc = c(NS,ns)->sc; - for (usz i = 3; i < 4; i++) nfn_swapObj(sc->vars[i], incG(ns)); + for (usz i = 3; i < 7; i++) nfn_swapObj(sc->vars[i], incG(ns)); return ns; } diff --git a/test/cases/hash.bqn b/test/cases/hash.bqn index 2baaf73a..7f571d49 100644 --- a/test/cases/hash.bqn +++ b/test/cases/hash.bqn @@ -45,6 +45,7 @@ a←(•ParseFloat¨ "1.2"‿"-0"‿"0")∾-⊸⋈0÷0 ⋄ ! (0‿3‿3‿3⊏a) !"⊐: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊐1‿2 !"⊒: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊒1‿2 -⟨"get"⟩ %% •ns.Keys "abc"‿"de"‿"fgh" •HashMap ↕3 ⟨2⟩ %% ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fgh" ∞ %% ∞ ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fghi" +3 %% ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Count@ +"B" %% ("b" ("a"‿"b" •HashMap "A"‿"C").Set "B").Get "b" From 95f5e65b6ccc54eda537d5cd26dabede767293d1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 27 Nov 2023 22:10:00 -0500 Subject: [PATCH 06/11] =?UTF-8?q?=E2=80=A2HashMap=20delete=20with=20backwa?= =?UTF-8?q?rds=20shift;=20no=20key/value=20compaction=20yet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 25 +++++++++++++++++++++++-- src/builtins/sysfn.c | 10 +++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index b6f580a0..2cb5081d 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -686,12 +686,12 @@ B hashmap_lookup(B* vars, B w, B x) { void hashmap_set(B* vars, B w, B x) { HashMap* map = c(HashMap, vars[2]); u64* hp = map->a; u64 sh = map->sh; - usz i = map->pop; - if (i>>(64-3-sh)>7 || hp[map->sz-1]!=empty) { // keep load <= 7/8 + if (map->pop>>(64-3-sh)>7 || hp[map->sz-1]!=empty) { // keep load <= 7/8 map=hashmap_resize(map); vars[2] = tag(map, OBJ_TAG); hp=map->a; sh=map->sh; } + usz i = IA(vars[0]); B* keys = harr_ptr(vars[0]); HASHMAP_INSERT( w, @@ -701,3 +701,24 @@ void hashmap_set(B* vars, B w, B x) { vars[0] = vec_addN(vars[0], w); vars[1] = vec_addN(vars[1], x); } + +void hashmap_delete(B* vars, B x) { + HashMap* map = c(HashMap, vars[2]); + u64* hp = map->a; u64 sh = map->sh; + B kb = vars[0]; B* keys = harr_ptr(kb); + HASHMAP_FIND(x, + dec(x); + do { + u64 jp=j; j++; + u=hp[j]; if (u>>sh==j) u=empty; + hp[jp]=u; + } while (u!=empty); + B vb = vars[1]; + if (!reusable(kb)) { vars[0]=kb=taga(cpyHArr(kb)); keys=harr_ptr(kb); } + if (!reusable(vb)) { vars[1]=vb=taga(cpyHArr(vb)); } + usz p = --(map->pop); dec(keys[i]); keys[i]=bi_N; + B* s = harr_ptr(vb)+i; dec(*s); *s=bi_N; + return; + ) + thrM("(hashmap).Delete: key not found"); +} diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 493f686a..836e0b68 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -711,16 +711,19 @@ static Body* hashmap_ns; static B hashmap_getName; static NFnDesc* hashmap_getDesc; static B hashmap_hasName; static NFnDesc* hashmap_hasDesc; static B hashmap_setName; static NFnDesc* hashmap_setDesc; +static B hashmap_deleteName; static NFnDesc* hashmap_deleteDesc; static B hashmap_countName; static NFnDesc* hashmap_countDesc; // Hash object handling defined in search.c extern B hashmap_build(B keys, usz n); extern B hashmap_lookup(B* vars, B w, B x); extern void hashmap_set(B* vars, B w, B x); +extern void hashmap_delete(B* vars, B x); extern usz hashmap_count(B hash); #define VARS c(NS,nfn_objU(t))->sc->vars B hashmap_get_c1(B t, B x ) { return hashmap_lookup(VARS, bi_N, x); } B hashmap_get_c2(B t, B w, B x) { return hashmap_lookup(VARS, w, x); } B hashmap_set_c2(B t, B w, B x) { hashmap_set(VARS, w, x); return inc(nfn_objU(t)); } +B hashmap_delete_c1(B t, B x ) { hashmap_delete(VARS, x); return inc(nfn_objU(t)); } B hashmap_count_c1(B t, B x) { dec(x); return m_usz(hashmap_count(VARS[2])); } B hashmap_has_c1(B t, B x) { B l = hashmap_lookup(VARS, bi_noVar, x); @@ -729,12 +732,13 @@ B hashmap_has_c1(B t, B x) { } #undef VARS static NOINLINE void hashmap_init() { - hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get", "has", "set", "count"); + hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get", "has", "set", "delete", "count"); NSDesc* d = hashmap_ns->nsDesc; for (usz i = 0; i < 3; i++) d->expGIDs[i] = -1; hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); hashmap_hasName = m_c8vec_0("has"); gc_add(hashmap_hasName); hashmap_hasDesc = registerNFn(m_c8vec_0("(hashmap).Has"), hashmap_has_c1, c2_bad); hashmap_setName = m_c8vec_0("set"); gc_add(hashmap_setName); hashmap_setDesc = registerNFn(m_c8vec_0("(hashmap).Set"), c1_bad, hashmap_set_c2); + hashmap_deleteName = m_c8vec_0("delete"); gc_add(hashmap_deleteName); hashmap_deleteDesc = registerNFn(m_c8vec_0("(hashmap).Delete"), hashmap_delete_c1, c2_bad); hashmap_countName = m_c8vec_0("count"); gc_add(hashmap_countName); hashmap_countDesc = registerNFn(m_c8vec_0("(hashmap).Count"), hashmap_count_c1, c2_bad); } B hashMap_c2(B t, B w, B x) { @@ -744,9 +748,9 @@ B hashMap_c2(B t, B w, B x) { if (hashmap_ns==NULL) hashmap_init(); w = taga(toHArr(w)); x = taga(toHArr(x)); B h = hashmap_build(w, n); - B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N), m_nfn(hashmap_hasDesc, bi_N), m_nfn(hashmap_setDesc, bi_N), m_nfn(hashmap_countDesc, bi_N)); + B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N), m_nfn(hashmap_hasDesc, bi_N), m_nfn(hashmap_setDesc, bi_N), m_nfn(hashmap_deleteDesc, bi_N), m_nfn(hashmap_countDesc, bi_N)); Scope* sc = c(NS,ns)->sc; - for (usz i = 3; i < 7; i++) nfn_swapObj(sc->vars[i], incG(ns)); + for (usz i = 3; i < 8; i++) nfn_swapObj(sc->vars[i], incG(ns)); return ns; } From 53beab531be30e9e329a453183767ee8360b2d40 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 28 Nov 2023 11:09:32 -0500 Subject: [PATCH 07/11] =?UTF-8?q?=E2=80=A2HashMap=20key=20and=20value=20li?= =?UTF-8?q?st=20compaction=20and=20methods=20to=20return=20those=20lists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 46 +++++++++++++++++++++++++++++++++++++++++-- src/builtins/sysfn.c | 15 ++++++++++---- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 2cb5081d..2eac975f 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -617,14 +617,14 @@ static const u64 hmask = ~(u32)0; usz hashmap_count(B hash) { return c(HashMap, hash)->pop; } -HashMap* hashmap_alloc(usz pop, usz sh, u64 sz) { +static HashMap* hashmap_alloc(usz pop, usz sh, u64 sz) { HashMap* map = mm_alloc(fsizeof(HashMap,a,u64,sz), t_hashmap); map->pop = pop; map->sh = sh; map->sz = sz; memset64(map->a, empty, sz); return map; } -NOINLINE HashMap* hashmap_resize(HashMap* old_map) { +static NOINLINE HashMap* hashmap_resize(HashMap* old_map) { usz sh = old_map->sh - 1; if (sh < 32) thrM("•HashMap: hash size maximum exceeded"); u64 sz = hashmap_size(sh); @@ -639,6 +639,47 @@ NOINLINE HashMap* hashmap_resize(HashMap* old_map) { return map->a[sz-1]==empty ? map : hashmap_resize(map); } +static NOINLINE void hashmap_compact(B* vars) { + B k = vars[0]; B* keys = harr_ptr(k); + B v = vars[1]; B* vals = harr_ptr(v); + assert(reusable(k) && reusable(v)); + usz l0 = IA(k); usz l = l0; + while (l>0 && q_N(keys[l-1])) l--; + TALLOC(usz, ind_map, l+1); + ind_map[0] = -1; // Index by i+1 to maintain empty entries + usz j = 0; // Number of entries seen + for (usz i=0; i 0) { + HashMap* map = c(HashMap, vars[2]); + assert(j == map->pop); + u64 sz = map->sz; + u64* hp = map->a; + for (u64 i=0; ipop == IA(r)) return r; + if (!reusable(vars[0])) vars[0] = taga(cpyHArr(vars[0])); + if (!reusable(vars[1])) vars[1] = taga(cpyHArr(vars[1])); + hashmap_compact(vars); + return vars[which]; +} + // Expects already defined: u64* hp, u64 sh, B* keys #define HASHMAP_FIND(X, FOUND) \ u64 h = bqn_hash(X, wy_secret); \ @@ -718,6 +759,7 @@ void hashmap_delete(B* vars, B x) { if (!reusable(vb)) { vars[1]=vb=taga(cpyHArr(vb)); } usz p = --(map->pop); dec(keys[i]); keys[i]=bi_N; B* s = harr_ptr(vb)+i; dec(*s); *s=bi_N; + if (p==0 || p+32 < IA(kb)/2) hashmap_compact(vars); return; ) thrM("(hashmap).Delete: key not found"); diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 836e0b68..c43e3702 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -713,26 +713,31 @@ static B hashmap_hasName; static NFnDesc* hashmap_hasDesc; static B hashmap_setName; static NFnDesc* hashmap_setDesc; static B hashmap_deleteName; static NFnDesc* hashmap_deleteDesc; static B hashmap_countName; static NFnDesc* hashmap_countDesc; +static B hashmap_keysName; static NFnDesc* hashmap_keysDesc; +static B hashmap_valuesName; static NFnDesc* hashmap_valuesDesc; // Hash object handling defined in search.c extern B hashmap_build(B keys, usz n); extern B hashmap_lookup(B* vars, B w, B x); extern void hashmap_set(B* vars, B w, B x); extern void hashmap_delete(B* vars, B x); extern usz hashmap_count(B hash); +extern B hashmap_keys_or_vals(B* vars, usz which); #define VARS c(NS,nfn_objU(t))->sc->vars B hashmap_get_c1(B t, B x ) { return hashmap_lookup(VARS, bi_N, x); } B hashmap_get_c2(B t, B w, B x) { return hashmap_lookup(VARS, w, x); } B hashmap_set_c2(B t, B w, B x) { hashmap_set(VARS, w, x); return inc(nfn_objU(t)); } B hashmap_delete_c1(B t, B x ) { hashmap_delete(VARS, x); return inc(nfn_objU(t)); } -B hashmap_count_c1(B t, B x) { dec(x); return m_usz(hashmap_count(VARS[2])); } B hashmap_has_c1(B t, B x) { B l = hashmap_lookup(VARS, bi_noVar, x); if (l.u==bi_noVar.u) return m_f64(0); dec(l); return m_f64(1); } +B hashmap_count_c1(B t, B x) { dec(x); return m_usz(hashmap_count(VARS[2])); } +B hashmap_keys_c1 (B t, B x) { dec(x); return inc(hashmap_keys_or_vals(VARS, 0)); } +B hashmap_values_c1(B t, B x) { dec(x); return inc(hashmap_keys_or_vals(VARS, 1)); } #undef VARS static NOINLINE void hashmap_init() { - hashmap_ns = m_nnsDesc("keys", "vals", "hash", "get", "has", "set", "delete", "count"); + hashmap_ns = m_nnsDesc("keylist", "vallist", "hash", "get", "has", "set", "delete", "count", "keys", "values"); NSDesc* d = hashmap_ns->nsDesc; for (usz i = 0; i < 3; i++) d->expGIDs[i] = -1; hashmap_getName = m_c8vec_0("get"); gc_add(hashmap_getName); hashmap_getDesc = registerNFn(m_c8vec_0("(hashmap).Get"), hashmap_get_c1, hashmap_get_c2); @@ -740,6 +745,8 @@ static NOINLINE void hashmap_init() { hashmap_setName = m_c8vec_0("set"); gc_add(hashmap_setName); hashmap_setDesc = registerNFn(m_c8vec_0("(hashmap).Set"), c1_bad, hashmap_set_c2); hashmap_deleteName = m_c8vec_0("delete"); gc_add(hashmap_deleteName); hashmap_deleteDesc = registerNFn(m_c8vec_0("(hashmap).Delete"), hashmap_delete_c1, c2_bad); hashmap_countName = m_c8vec_0("count"); gc_add(hashmap_countName); hashmap_countDesc = registerNFn(m_c8vec_0("(hashmap).Count"), hashmap_count_c1, c2_bad); + hashmap_keysName = m_c8vec_0("keys"); gc_add(hashmap_keysName); hashmap_keysDesc = registerNFn(m_c8vec_0("(hashmap).Keys"), hashmap_keys_c1, c2_bad); + hashmap_valuesName = m_c8vec_0("values"); gc_add(hashmap_valuesName); hashmap_valuesDesc = registerNFn(m_c8vec_0("(hashmap).Values"), hashmap_values_c1, c2_bad); } B hashMap_c2(B t, B w, B x) { if (!isArr(w) || RNK(w)!=1 || !isArr(x) || RNK(x)!=1) thrF("•HashMap: Arguments must be lists (%H≡≢𝕨, %H≡≢𝕩)", w, x); @@ -748,9 +755,9 @@ B hashMap_c2(B t, B w, B x) { if (hashmap_ns==NULL) hashmap_init(); w = taga(toHArr(w)); x = taga(toHArr(x)); B h = hashmap_build(w, n); - B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N), m_nfn(hashmap_hasDesc, bi_N), m_nfn(hashmap_setDesc, bi_N), m_nfn(hashmap_deleteDesc, bi_N), m_nfn(hashmap_countDesc, bi_N)); + B ns = m_nns(hashmap_ns, w, x, h, m_nfn(hashmap_getDesc, bi_N), m_nfn(hashmap_hasDesc, bi_N), m_nfn(hashmap_setDesc, bi_N), m_nfn(hashmap_deleteDesc, bi_N), m_nfn(hashmap_countDesc, bi_N), m_nfn(hashmap_keysDesc, bi_N), m_nfn(hashmap_valuesDesc, bi_N)); Scope* sc = c(NS,ns)->sc; - for (usz i = 3; i < 8; i++) nfn_swapObj(sc->vars[i], incG(ns)); + for (usz i = 3; i < 10; i++) nfn_swapObj(sc->vars[i], incG(ns)); return ns; } From 51027cabe1a102e9063084106f9db23bacaebc80 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 28 Nov 2023 16:53:45 -0500 Subject: [PATCH 08/11] Fix GC-after-free with multiple hash resizes --- src/builtins/search.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 2eac975f..13d4ed47 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -728,7 +728,8 @@ void hashmap_set(B* vars, B w, B x) { HashMap* map = c(HashMap, vars[2]); u64* hp = map->a; u64 sh = map->sh; if (map->pop>>(64-3-sh)>7 || hp[map->sz-1]!=empty) { // keep load <= 7/8 - map=hashmap_resize(map); + vars[2] = bi_N; // hashmap_resize might free then alloc + map = hashmap_resize(map); vars[2] = tag(map, OBJ_TAG); hp=map->a; sh=map->sh; } From 72a9a958191746cedfa3a6a00d31082965808e09 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 28 Nov 2023 20:08:27 -0500 Subject: [PATCH 09/11] =?UTF-8?q?Make=20sure=20=E2=80=A2HashMap=20table=20?= =?UTF-8?q?leaves=20last=20entry=20empty,=20as=20a=20sentinel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 13d4ed47..81c97167 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -727,12 +727,6 @@ B hashmap_lookup(B* vars, B w, B x) { void hashmap_set(B* vars, B w, B x) { HashMap* map = c(HashMap, vars[2]); u64* hp = map->a; u64 sh = map->sh; - if (map->pop>>(64-3-sh)>7 || hp[map->sz-1]!=empty) { // keep load <= 7/8 - vars[2] = bi_N; // hashmap_resize might free then alloc - map = hashmap_resize(map); - vars[2] = tag(map, OBJ_TAG); - hp=map->a; sh=map->sh; - } usz i = IA(vars[0]); B* keys = harr_ptr(vars[0]); HASHMAP_INSERT( @@ -740,6 +734,12 @@ void hashmap_set(B* vars, B w, B x) { B* s = harr_ptr(vars[1])+i; dec(*s); dec(w); *s=x; return; ) map->pop++; + if (map->pop>>(64-3-sh)>7 || je==map->sz-1) { // keep load <= 7/8 + vars[2] = bi_N; // hashmap_resize might free then alloc + map = hashmap_resize(map); + vars[2] = tag(map, OBJ_TAG); + hp=map->a; sh=map->sh; + } vars[0] = vec_addN(vars[0], w); vars[1] = vec_addN(vars[1], x); } From a22af96b1731b03da1f580fa71f7d77dc3db50b0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 28 Nov 2023 20:25:17 -0500 Subject: [PATCH 10/11] =?UTF-8?q?More=20thorough=20=E2=80=A2HashMap=20test?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/cases/hash.bqn | 18 ++++++++++++++---- test/hashmap.bqn | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 test/hashmap.bqn diff --git a/test/cases/hash.bqn b/test/cases/hash.bqn index 7f571d49..b7a462e7 100644 --- a/test/cases/hash.bqn +++ b/test/cases/hash.bqn @@ -45,7 +45,17 @@ a←(•ParseFloat¨ "1.2"‿"-0"‿"0")∾-⊸⋈0÷0 ⋄ ! (0‿3‿3‿3⊏a) !"⊐: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊐1‿2 !"⊒: Rank of 𝕩 must be at least the cell rank of 𝕨 (2‿2‿2 ≡ ≢𝕨, ⟨2⟩ ≡ ≢𝕩)" % (2‿2‿2⥊1‿2)⊒1‿2 -⟨2⟩ %% ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fgh" -∞ %% ∞ ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fghi" -3 %% ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Count@ -"B" %% ("b" ("a"‿"b" •HashMap "A"‿"C").Set "B").Get "b" +# hashmap +("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fgh" %% ⟨2⟩ +∞ ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fghi" %% ∞ +("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Count"xy" %% 3 +("b" ("a"‿"b" •HashMap "A"‿"C").Set "B").Get "b" %% "B" +((•HashMap˜↕4).Set´ 1‿"one").Keys↕2 %% ↕4 +(((•HashMap˜↕4).Delete 1).Set´ 1‿"one").Keys@ %% 0‿2‿3‿1 +(((•HashMap˜↕4).Delete 1).Set´ 1‿"one").Values@ %% 0‿2‿3‿"one" +({𝕩.Set´"xy"}⍟4•HashMap˜↕0).Count@ %% 1 +!"•HashMap: Arguments must be lists (⟨⟩≡≢𝕨, ⟨3⟩≡≢𝕩)" % 'a' •HashMap "str" +!"•HashMap: 𝕨 and 𝕩 must have the same length (4≡≠𝕨, 3≡≠𝕩)" % "stri" •HashMap "str" +!"•HashMap: 𝕨 contained duplicate keys" % "strs" •HashMap "stri" +!"(hashmap).Get: key not found" % ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Get "fg" +!"(hashmap).Delete: key not found" % ("abc"‿"de"‿"fgh" •HashMap ⥊¨↕3).Delete 'a' diff --git a/test/hashmap.bqn b/test/hashmap.bqn new file mode 100644 index 00000000..f8c40904 --- /dev/null +++ b/test/hashmap.bqn @@ -0,0 +1,42 @@ +u ← ⌊100×(•UnixTime+1|100וMonoTime)@ +ro← •MakeRand •Show u +R ← ro.Range + +err ← 1 # Run operations that should error + +Test ← {iter 𝕊 n: + # Global set of keys that might be used + keys ← ⊒⊸(-⊸∾⍟(0<⊣)¨) (R⟜15 R¨ 1-˜1e6⌊∘⋆R⟜0) n + # Which keys are currently set, and count + c ← +´ mask ← 0 = n R 20+R 10 + # Value for key if set + vals ← ((R⟜≠⊏⊢)⟜⊢‿÷‿⥊‿↕ {𝕎𝕩}¨ R⟜10) n + + map ← keys •HashMap○(mask⊸/) vals + + # Single operations and checks + _er ← { err ? ! 0∘𝔽⎊1 𝕩⊑keys ; 1 } + Cnt ← {𝕊: ! c ≡ map.Count@ } + Key ← {𝕊: ! (mask/keys) ≡○∧ map.Keys@ } + Val ← {𝕊: ! (mask/vals) ≡○∧ map.Values@ } + Has ← { ! (𝕩⊑mask) ≡ map.Has 𝕩⊑keys } + Get ← { 𝕩⊑mask ? ! (𝕩⊑vals) ≡ map.Get 𝕩⊑keys ; map.Get _er 𝕩 } + Set ← { 𝕨 map.Set˜ 𝕩⊑keys ⋄ vals 𝕨˙⌾(𝕩⊸⊑)↩ ⋄ mask {c+↩¬𝕩⋄1}⌾(𝕩⊸⊑)↩ ⋄ @ } + Del ← { 𝕩⊑mask ? map.Delete 𝕩⊑keys ⋄ mask 0⌾(𝕩⊸⊑)↩ ⋄ c-↩1 ; map.Delete _er 𝕩 } + + Selection ← { (𝕨≤1)◶⟨R·⌊1.5⊸×,𝕨⟩⊸⌊⊸ro.Deal∘≠⊸⊏ 𝕩 } + RandVal ← R∘100 + ops ← ⟨Cnt, Has, Get, Del, (R∘n⊑vals˙)Set⊢, RandVal⊸Set⟩ + + # Larger sets: argument is number of ops (sometimes overridden) + FullCheck ← Key∘Val∘Cnt + Deletes ← { Del¨ 𝕩 Selection mask } + Restores ← { ⊏⟜vals⊸(Set¨) 𝕩 Selection ¬mask } + RandomOps ← (R⟜≠⊏⊢)⟜ops {𝕎𝕩}¨ R⟜n + RandomSame ← R∘≠⊸⊑∘ops {𝕎𝕩}¨ R⟜n + opSets ← FullCheck‿Deletes‿Restores‿RandomOps‿RandomSame + ((R⟜≠⊏⊢)⟜opSets {𝕎𝕩}¨ R⟜100) iter +} + +1e5 Test 1e2 +1e4 Test 1e4 From 609980f9508af9b0b0eb89e868bd0f7edee84d2e Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 29 Nov 2023 17:55:07 +0200 Subject: [PATCH 11/11] comment about HashMap entry format --- src/builtins/search.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 81c97167..609f00d8 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -609,7 +609,7 @@ typedef struct HashMap { u64 pop; // count of defined entries u64 sh; // shift to turn hash into index u64 sz; // count of allocated entries, a power of 2 - u64 a[]; + u64 a[]; // lower 32 bits: index into keys/vals; upper 32 bits: upper 32 bits of hash } HashMap; static u64 hashmap_size(usz sh) { return ((u64)1 << (64-sh)) + 32; } static const u64 empty = ~(u64)0;