diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 21c83b77..d5e08f10 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -24,6 +24,9 @@ B scan_ne(B x, u64 p, u64 ia) { // consumes x u64* rp; B r=m_bitarrv(&rp,ia); #if SINGELI_X86_64 && __PCLMUL__ clmul_scan_ne(p, xp, rp, BIT_N(ia)); + #if USE_VALGRIND + if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]); + #endif #else for (usz i = 0; i < BIT_N(ia); i++) { u64 c = xp[i]; diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index b4217e7b..d6b3ec86 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -1245,7 +1245,10 @@ B reverse_c2(B t, B w, B x) { } #ifdef __BMI2__ -#include + #include + #if USE_VALGRIND + #define _pdep_u64 vg_pdep_u64 + #endif #endif extern B rt_transp; diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 1a0de4fd..90bcb768 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -63,90 +63,12 @@ #include #if USE_VALGRIND - #define DBG_VG_SLASH 0 - u64 loadMask(u64* p, u64 unk, u64 exp, u64 i, u64 pos) { - // #if DBG_VG_SLASH - // if (pos==0) printf("index %2ld, got %016lx\n", i, p[i]); - // #endif - if (pos==0) return ~(p[i]^exp); - u64 res = loadMask(p, unk, exp, i, pos<<1); - if (unk&pos) res&= loadMask(p, unk, exp, i|pos, pos<<1); - return res; - } - NOINLINE u64 vg_load64(u64* p, u64 i) { - u64 unk = ~vg_getDefined_u64(i); - u64 res = p[vg_withDefined_u64(i, ~0ULL)]; // result value will always be the proper indexing operation - - i32 undefCount = POPC(unk); - if (undefCount>0) { - if (undefCount>8) err("too many unknown bits in index of vg_load64"); - res = vg_withDefined_u64(res, loadMask(p, unk, res, i & ~unk, 1)); - } - #if DBG_VG_SLASH - vg_printDefined_u64("idx", i); - vg_printDefined_u64("res", res); - #endif - return res; - } - NOINLINE u64 vg_pext_u64(u64 src, u64 mask) { - u64 maskD = vg_getDefined_u64(mask); - u64 r = vg_undef_u64(0); - i32 ri = 0; - u64 undefMask = 0; - for (i32 i = 0; i < 64; i++) { - u64 c = 1ull<>i)&1) { - r|= (c&1) << i; - c>>= 1; - } - } - #if DBG_VG_SLASH - printf("pdep:\n"); - vg_printDefined_u64("src", src); - vg_printDefined_u64("msk", mask); - vg_printDefined_u64("res", r); - vg_printDefined_u64("exp", _pdep_u64(src, mask)); - #endif - return r; - } - NOINLINE u64 rand_popc64(u64 x) { - u64 def = vg_getDefined_u64(x); - if (def==~0ULL) return POPC(x); - i32 min = POPC(x & def); - i32 diff = POPC(~def); - i32 res = min + vgRand64Range(diff); - #if DBG_VG_SLASH - printf("popc:\n"); - vg_printDefined_u64("x", x); - printf("popc in %d-%d; res: %d\n", min, min+diff, res); - #endif - return res; - } #define _pext_u32 vg_pext_u64 #define _pext_u64 vg_pext_u64 #define _pdep_u32 vg_pdep_u64 #define _pdep_u64 vg_pdep_u64 #else - #define vg_load64(p, i) p[i] + #define vg_loadLUT64(p, i) p[i] #define rand_popc64(X) POPC(X) #endif diff --git a/src/core/stuff.c b/src/core/stuff.c index af8e91f0..72abc33e 100644 --- a/src/core/stuff.c +++ b/src/core/stuff.c @@ -686,81 +686,7 @@ NOINLINE void print_allocStats() { #if USE_VALGRIND - static void printBitDef(u8 val, u8 def) { - printf("%s", def&1? val&1?"1":"0" : val&1?"¹":"⁰"); - } - void vg_printDump_p(char* name, void* data, u64 len) { - u8 vbits[len]; - int r = VALGRIND_GET_VBITS(data, vbits, len); - - if(name!=NULL) printf("%s:\n", name); - if (r!=1) printf("(failed to get vbits)\n"); - - for (u64 i = 0; i < len; i++) { - if (i!=0) printf(i&7? " " : "\n"); - u8 cv = ~vbits[i]; - u8 cd = ((u8*)data)[i]; - VALGRIND_SET_VBITS(&cd, &(u8[]){0}, 1); - for (i32 j = 7; j >= 0; j--) { - printBitDef(cd>>j, cv>>j); - } - } - putchar('\n'); - } - void vg_printDefined_u64(char* name, u64 x) { - if(name!=NULL) printf("%s: ", name); - u64 d = vg_getDefined_u64(x); - u64 xv = x; - VALGRIND_MAKE_MEM_DEFINED(&xv, 8); - - for (i32 i = 63; i >= 0; i--) printBitDef(xv>>i, d>>i); - printf("\n"); - } - u64 vg_rand(u64 x) { // randomize undefined bits in x, and return a value with all bits defined - u64 d = vg_getDefined_u64(x); - if (~d == 0) return x; - return (x & d) | (vgRand64() & ~d); - } - B vg_validateResult(B x) { - if (!isArr(x)) return x; - void* data; - u64 len; - u8 xe = TI(x,elType); - u64 ia = IA(x); - if (xe!=el_B) { - data = tyany_ptr(x); - if (xe==el_bit) { - i32 left = ia&63; - len = (ia>>6)*8; - if (left) { - u64 last = ((u64*)data)[len/8]; - u64 exp = (1ULL<= 0; j--) { + printBitDef(cd>>j, cv>>j); + } + } + putchar('\n'); +} + +void vg_printDefined_u64(char* name, u64 x) { + if(name!=NULL) printf("%s: ", name); + u64 d = vg_getDefined_u64(x); + u64 xv = x; + VALGRIND_MAKE_MEM_DEFINED(&xv, 8); + + for (i32 i = 63; i >= 0; i--) printBitDef(xv>>i, d>>i); + printf("\n"); +} + +u64 vg_rand(u64 x) { // randomize undefined bits in x, and return a value with all bits defined + u64 d = vg_getDefined_u64(x); + if (~d == 0) return x; + return (x & d) | (vgRand64() & ~d); +} + +B vg_validateResult(B x) { + if (!isArr(x)) return x; + void* data; + u64 len; + u8 xe = TI(x,elType); + u64 ia = IA(x); + if (xe!=el_B) { + data = tyany_ptr(x); + if (xe==el_bit) { + i32 left = ia&63; + len = (ia>>6)*8; + if (left) { + u64 last = ((u64*)data)[len/8]; + u64 exp = (1ULL<0) { + if (undefCount>8) err("too many unknown bits in index of vg_loadLUT64"); + res = vg_withDefined_u64(res, loadMask(p, unk, res, i & ~unk, 1)); + } + #if DBG_VG_OVERRIDES + vg_printDefined_u64("idx", i); + vg_printDefined_u64("res", res); + #endif + return res; +} + +NOINLINE u64 vg_pext_u64(u64 src, u64 mask) { + u64 maskD = vg_getDefined_u64(mask); + u64 r = vg_undef_u64(0); + i32 ri = 0; + u64 undefMask = 0; + for (i32 i = 0; i < 64; i++) { + u64 c = 1ull<>i)&1) { + r|= (c&1) << i; + c>>= 1; + } + } + #if DBG_VG_OVERRIDES + printf("pdep:\n"); + vg_printDefined_u64("src", src); + vg_printDefined_u64("msk", mask); + vg_printDefined_u64("res", r); + vg_printDefined_u64("exp", _pdep_u64(src, mask)); + #endif + return r; +} + +NOINLINE u64 rand_popc64(u64 x) { + u64 def = vg_getDefined_u64(x); + if (def==~0ULL) return POPC(x); + i32 min = POPC(x & def); + i32 diff = POPC(~def); + i32 res = min + vgRand64Range(diff); + #if DBG_VG_OVERRIDES + printf("popc:\n"); + vg_printDefined_u64("x", x); + printf("popc in %d-%d; res: %d\n", min, min+diff, res); + #endif + return res; +} \ No newline at end of file diff --git a/src/utils/valgrind.h b/src/utils/valgrind.h index 4bbd954d..fb2e4342 100644 --- a/src/utils/valgrind.h +++ b/src/utils/valgrind.h @@ -28,6 +28,11 @@ u64 vgRand64Range(u64 range); u64 vgRand64(void); u64 vg_rand(u64 x); // randomize undefined bits in x, and return the value with all bits defined +u64 vg_loadLUT64(u64* p, u64 i); +u64 vg_pext_u64(u64 src, u64 mask); +u64 vg_pdep_u64(u64 src, u64 mask); +u64 rand_popc64(u64 x); + void vg_printDefined_u64(char* name, u64 x); void vg_printDump_p(char* name, void* data, u64 len); #define vg_printDump_v(X) ({ AUTO x_ = (X); vg_printDump_p(#X, &x_, sizeof(x_)); x_; })