diff --git a/src/h.h b/src/h.h index 38787929..625911fb 100644 --- a/src/h.h +++ b/src/h.h @@ -129,7 +129,7 @@ char* format_type(u8 u) { /*derv.c*/ F(fork,"(fork)") F(atop,"(atop)") F(md1d,"(derived 1-modifier)") F(md2d,"(derived 2-modifier)") \ /*sort.c*/ F(gradeUp,"⍋") \ /*sysfn.c*/ F(type,"•Type") F(decp,"•Decompose") F(primInd,"•PrimInd") F(glyph,"•Glyph") F(repr,"•Repr") F(fill,"•FillFn") \ - /*sysfn.c*/ F(grLen,"•GroupLen") F(grOrd,"•groupOrd") F(asrt,"!") F(sys,"•getsys") F(bqn,"•BQN") F(cmp,"•Cmp") F(internal,"•Internal") F(show,"•Show") F(out,"•Out") \ + /*sysfn.c*/ F(grLen,"•GroupLen") F(grOrd,"•groupOrd") F(asrt,"!") F(sys,"•getsys") F(bqn,"•BQN") F(cmp,"•Cmp") F(internal,"•Internal") F(show,"•Show") F(out,"•Out") F(hash,"•Hash") \ enum PrimFns { #define F(N,X) pf_##N, diff --git a/src/hash.c b/src/hash.c new file mode 100644 index 00000000..b04073d1 --- /dev/null +++ b/src/hash.c @@ -0,0 +1,65 @@ +#include "h.h" +#include "wyhash.h" + +B bqn_squeeze(B x) { + assert(isArr(x)); + u8 xe = TI(x).elType; + if (xe==el_i32 || xe==el_c32) return x; + usz ia = a(x)->ia; + if (ia==0) return x; + if (xe==el_f64) { + f64* xp = f64any_ptr(x); + for (usz i = 0; i < ia; i++) if (xp[i] != (f64)(i32)xp[i]) return x; + return tag(toI32Arr(x), ARR_TAG); + } + assert(xe==el_B); + BS2B xgetU = TI(x).getU; + B x0 = xgetU(x, 0); + if (isNum(x0)) { + for (usz i = 0; i < ia; i++) { + B c = xgetU(x, i); + if (!isNum(c)) return x; + if (!q_i32(c)) { + for (i++; i < ia; i++) if (!isNum(xgetU(x, i))) return x; + return tag(toF64Arr(x), ARR_TAG); + } + } + return tag(toI32Arr(x), ARR_TAG); + } else if (isC32(x0)) { + for (usz i = 1; i < ia; i++) { + B c = xgetU(x, i); + if (!isC32(c)) return x; + } + return tag(toC32Arr(x), ARR_TAG); + } else return x; +} + +u64 bqn_hash(B x, const u64 secret[4]) { // consumes (because otherwise squeezing is hard as in-place cannot be done because slices) + if (isAtm(x)) { + if (q_f64(x)) return wyhash64(secret[0], x.u); + if (isC32(x)) return wyhash64(secret[1], x.u); + assert(isVal(x)); + dec(x); + return wyhash64(secret[2], x.u); + } + x = bqn_squeeze(x); + u8 xe = TI(x).elType; + usz xia = a(x)->ia; + u64 shHash = wyhash(a(x)->sh, rnk(x)*sizeof(usz), 0, secret); + u64 r; + if (xe==el_i32) { + r = wyhash(i32any_ptr(x), xia*4, shHash, secret); + } else if (xe==el_c32) { + r = wyhash(c32any_ptr(x), xia*4, shHash, secret); + } else if (xe==el_f64) { + r = wyhash(f64any_ptr(x), xia*8, shHash, secret); + } else { + assert(xe==el_B); + u64 data[xia]; + BS2B xget = TI(x).get; + for (usz i = 0; i < xia; i++) data[i] = bqn_hash(xget(x, i), secret); + r = wyhash(data, xia*8, shHash, secret); + } + dec(x); + return r; +} diff --git a/src/i32arr.c b/src/i32arr.c index 38af6048..d066cfea 100644 --- a/src/i32arr.c +++ b/src/i32arr.c @@ -47,12 +47,18 @@ NOINLINE B m_cai32(usz ia, i32* a) { return r; } -I32Arr* toI32Arr(B x) { +f64* f64any_ptr(B x); +I32Arr* toI32Arr(B x) { // assumes it's possible if (v(x)->type==t_i32arr) return c(I32Arr,x); i32* rp; B r = m_i32arrc(&rp, x); usz ia = a(r)->ia; - BS2B xgetU = TI(x).getU; - for (usz i = 0; i < ia; i++) rp[i] = o2i(xgetU(x,i)); + if (TI(x).elType==el_f64) { + f64* fp = f64any_ptr(x); + for (usz i = 0; i < ia; i++) rp[i] = (i32)fp[i]; + } else { + BS2B xgetU = TI(x).getU; + for (usz i = 0; i < ia; i++) rp[i] = o2iu(xgetU(x,i)); + } dec(x); return c(I32Arr,r); } diff --git a/src/main.c b/src/main.c index 15faff5f..84635822 100644 --- a/src/main.c +++ b/src/main.c @@ -37,6 +37,7 @@ #include "c32arr.c" #include "f64arr.c" #include "fillarr.c" +#include "hash.c" #include "mut.c" #include "utf.c" #include "file.c" diff --git a/src/sysfn.c b/src/sysfn.c index 5b1d93ec..82e6df52 100644 --- a/src/sysfn.c +++ b/src/sysfn.c @@ -161,8 +161,24 @@ B cmp_c2(B t, B w, B x) { return r; } +B hash_c2(B t, B w, B x) { + u64 secret[4]; make_secret(o2i64(w), secret); + u64 rv = bqn_hash(x, secret); + i32* rp; B r = m_i32arrv(&rp, 4); + rp[0] = (u16)(rv>>48); rp[1] = (u16)(rv>>32); + rp[2] = (u16)(rv>>16); rp[3] = (u16)(rv ); + return r; +} +B hash_c1(B t, B x) { + u64 rv = bqn_hash(x, _wyp); + i32* rp; B r = m_i32arrv(&rp, 4); + rp[0] = (u16)(rv>>48); rp[1] = (u16)(rv>>32); + rp[2] = (u16)(rv>>16); rp[3] = (u16)(rv ); + return r; +} -#define F(A,M,D) M(type) M(decp) M(primInd) M(glyph) M(repr) A(fill) A(grLen) D(grOrd) A(asrt) M(out) M(show) M(sys) M(bqn) D(cmp) D(internal) + +#define F(A,M,D) M(type) M(decp) M(primInd) M(glyph) M(repr) A(fill) A(grLen) D(grOrd) A(asrt) M(out) M(show) M(sys) M(bqn) D(cmp) D(internal) A(hash) BI_FNS0(F); static inline void sysfn_init() { BI_FNS1(F) } #undef F @@ -188,6 +204,7 @@ B sys_c1(B t, B x) { else if (eqStr(c, U"bqn")) r.a[i] = inc(bi_bqn); else if (eqStr(c, U"cmp")) r.a[i] = inc(bi_cmp); else if (eqStr(c, U"timed")) r.a[i] = inc(bi_timed); + else if (eqStr(c, U"hash")) r.a[i] = inc(bi_hash); else if (eqStr(c, U"repr")) r.a[i] = inc(bi_repr); else if (eqStr(c, U"fchars")) r.a[i] = makeRel(bi_fchars); else if (eqStr(c, U"fbytes")) r.a[i] = makeRel(bi_fbytes); diff --git a/src/wyhash.h b/src/wyhash.h new file mode 100644 index 00000000..9f78658d --- /dev/null +++ b/src/wyhash.h @@ -0,0 +1,266 @@ +// This is free and unencumbered software released into the public domain under The Unlicense (http://unlicense.org/) +// main repo: https://github.com/wangyi-fudan/wyhash +// author: 王一 Wang Yi +// contributors: Reini Urban, Dietrich Epp, Joshua Haberman, Tommy Ettinger, Daniel Lemire, Otmar Ertl, cocowalla, leo-yuriev, Diego Barrios Romero, paulie-g, dumblob, Yann Collet, ivte-ms, hyb, James Z.M. Gao, easyaspi314 (Devin), TheOneric + +/* quick example: + string s="fjsakfdsjkf"; + uint64_t hash=wyhash(s.c_str(), s.size(), 0, _wyp); +*/ + +#ifndef wyhash_final_version_3 +#define wyhash_final_version_3 + +#ifndef WYHASH_CONDOM +//protections that produce different results: +//1: normal valid behavior +//2: extra protection against entropy loss (probability=2^-63), aka. "blind multiplication" +#define WYHASH_CONDOM 1 +#endif + +#ifndef WYHASH_32BIT_MUM +//0: normal version, slow on 32 bit systems +//1: faster on 32 bit systems but produces different results, incompatible with wy2u0k function +#define WYHASH_32BIT_MUM 0 +#endif + +//includes +#include +#include +#if defined(_MSC_VER) && defined(_M_X64) + #include + #pragma intrinsic(_umul128) +#endif + +//likely and unlikely macros +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) + #define _likely_(x) __builtin_expect(x,1) + #define _unlikely_(x) __builtin_expect(x,0) +#else + #define _likely_(x) (x) + #define _unlikely_(x) (x) +#endif + +//128bit multiply function +static inline uint64_t _wyrot(uint64_t x) { return (x>>32)|(x<<32); } +static inline void _wymum(uint64_t *A, uint64_t *B){ +#if(WYHASH_32BIT_MUM) + uint64_t hh=(*A>>32)*(*B>>32), hl=(*A>>32)*(uint32_t)*B, lh=(uint32_t)*A*(*B>>32), ll=(uint64_t)(uint32_t)*A*(uint32_t)*B; + #if(WYHASH_CONDOM>1) + *A^=_wyrot(hl)^hh; *B^=_wyrot(lh)^ll; + #else + *A=_wyrot(hl)^hh; *B=_wyrot(lh)^ll; + #endif +#elif defined(__SIZEOF_INT128__) + __uint128_t r=*A; r*=*B; + #if(WYHASH_CONDOM>1) + *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); + #else + *A=(uint64_t)r; *B=(uint64_t)(r>>64); + #endif +#elif defined(_MSC_VER) && defined(_M_X64) + #if(WYHASH_CONDOM>1) + uint64_t a, b; + a=_umul128(*A,*B,&b); + *A^=a; *B^=b; + #else + *A=_umul128(*A,*B,B); + #endif +#else + uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; + uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; + #if(WYHASH_CONDOM>1) + *A^=lo; *B^=hi; + #else + *A=lo; *B=hi; + #endif +#endif +} + +//multiply and xor mix function, aka MUM +static inline uint64_t _wymix(uint64_t A, uint64_t B){ _wymum(&A,&B); return A^B; } + +//endian macros +#ifndef WYHASH_LITTLE_ENDIAN + #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 1 + #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define WYHASH_LITTLE_ENDIAN 0 + #else + #warning could not determine endianness! Falling back to little endian. + #define WYHASH_LITTLE_ENDIAN 1 + #endif +#endif + +//read functions +#if (WYHASH_LITTLE_ENDIAN) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v;} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return v;} +#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return __builtin_bswap32(v);} +#elif defined(_MSC_VER) +static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);} +static inline uint64_t _wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return _byteswap_ulong(v);} +#else +static inline uint64_t _wyr8(const uint8_t *p) { + uint64_t v; memcpy(&v, p, 8); + return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); +} +static inline uint64_t _wyr4(const uint8_t *p) { + uint32_t v; memcpy(&v, p, 4); + return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); +} +#endif +static inline uint64_t _wyr3(const uint8_t *p, size_t k) { return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1];} +//wyhash main function +static inline uint64_t wyhash(const void *key, size_t len, uint64_t seed, const uint64_t *secret){ + const uint8_t *p=(const uint8_t *)key; seed^=*secret; uint64_t a, b; + if(_likely_(len<=16)){ + if(_likely_(len>=4)){ a=(_wyr4(p)<<32)|_wyr4(p+((len>>3)<<2)); b=(_wyr4(p+len-4)<<32)|_wyr4(p+len-4-((len>>3)<<2)); } + else if(_likely_(len>0)){ a=_wyr3(p,len); b=0;} + else a=b=0; + } + else{ + size_t i=len; + if(_unlikely_(i>48)){ + uint64_t see1=seed, see2=seed; + do{ + seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); + see1=_wymix(_wyr8(p+16)^secret[2],_wyr8(p+24)^see1); + see2=_wymix(_wyr8(p+32)^secret[3],_wyr8(p+40)^see2); + p+=48; i-=48; + }while(_likely_(i>48)); + seed^=see1^see2; + } + while(_unlikely_(i>16)){ seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed); i-=16; p+=16; } + a=_wyr8(p+i-16); b=_wyr8(p+i-8); + } + return _wymix(secret[1]^len,_wymix(a^secret[1],b^seed)); +} + +//the default secret parameters +static const uint64_t _wyp[4] = {0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x8ebc6af09c88c6e3ull, 0x589965cc75374cc3ull}; + +//a useful 64bit-64bit mix function to produce deterministic pseudo random numbers that can pass BigCrush and PractRand +static inline uint64_t wyhash64(uint64_t A, uint64_t B){ A^=0xa0761d6478bd642full; B^=0xe7037ed1a0b428dbull; _wymum(&A,&B); return _wymix(A^0xa0761d6478bd642full,B^0xe7037ed1a0b428dbull);} + +//The wyrand PRNG that pass BigCrush and PractRand +static inline uint64_t wyrand(uint64_t *seed){ *seed+=0xa0761d6478bd642full; return _wymix(*seed,*seed^0xe7037ed1a0b428dbull);} + +//convert any 64 bit pseudo random numbers to uniform distribution [0,1). It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>12)*_wynorm;} + +//convert any 64 bit pseudo random numbers to APPROXIMATE Gaussian distribution. It can be combined with wyrand, wyhash64 or wyhash. +static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0;} + +#if(!WYHASH_32BIT_MUM) +//fast range integer random number generation on [0,k) credit to Daniel Lemire. May not work when WYHASH_32BIT_MUM=1. It can be combined with wyrand, wyhash64 or wyhash. +static inline uint64_t wy2u0k(uint64_t r, uint64_t k){ _wymum(&r,&k); return k; } +#endif + +//make your own secret +static inline void make_secret(uint64_t seed, uint64_t *secret){ + uint8_t c[] = {15, 23, 27, 29, 30, 39, 43, 45, 46, 51, 53, 54, 57, 58, 60, 71, 75, 77, 78, 83, 85, 86, 89, 90, 92, 99, 101, 102, 105, 106, 108, 113, 114, 116, 120, 135, 139, 141, 142, 147, 149, 150, 153, 154, 156, 163, 165, 166, 169, 170, 172, 177, 178, 180, 184, 195, 197, 198, 201, 202, 204, 209, 210, 212, 216, 225, 226, 228, 232, 240 }; + for(size_t i=0;i<4;i++){ + uint8_t ok; + do{ + ok=1; secret[i]=0; + for(size_t j=0;j<64;j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<> 1) & 0x5555555555555555; + x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333); + x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f; + x = (x * 0x0101010101010101) >> 56; + if(x!=32){ ok=0; break; } +#endif + } + }while(!ok); + } +} + +/* This is world's fastest hash map: 2x faster than bytell_hash_map. + It does not store the keys, but only the hash/signature of keys. + First we use pos=hash1(key) to approximately locate the bucket. + Then we search signature=hash2(key) from pos linearly. + If we find a bucket with matched signature we report the bucket + Or if we meet a bucket whose signifure=0, we report a new position to insert + The signature collision probability is very low as we usually searched N~10 buckets. + By combining hash1 and hash2, we acturally have 128 bit anti-collision strength. + hash1 and hash2 can be the same function, resulting lower collision resistance but faster. + The signature is 64 bit, but can be modified to 32 bit if necessary for save space. + The above two can be activated by define WYHASHMAP_WEAK_SMALL_FAST + simple examples: + const size_t size=213432; + vector idx(size); // allocate the index of fixed size. idx MUST be zeroed. + vector value(size); // we only care about the index, user should maintain his own value vectors. + string key="dhskfhdsj" // the object to be inserted into idx + size_t pos=wyhashmap(idx.data(), idx.size(), key.c_str(), key.size(), 1); // get the position and insert + if(pos +*/