Skip radix moves if the top bytes are all the same

This commit is contained in:
Marshall Lochbaum 2022-10-22 22:16:55 -04:00
parent b9d5f10d4a
commit 7cbd23c832
2 changed files with 39 additions and 22 deletions

View File

@ -13,12 +13,14 @@
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
#define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0); #define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0);
#define RADIX_SUM_1_u32 avx2_scan_pluswrap_u32(c0,c0, 256,0);
#define RADIX_SUM_2_u8 avx2_scan_pluswrap_u8 (c0,c0,2*256,0); #define RADIX_SUM_2_u8 avx2_scan_pluswrap_u8 (c0,c0,2*256,0);
#define RADIX_SUM_2_u32 avx2_scan_pluswrap_u32(c0,c0,2*256,0); #define RADIX_SUM_2_u32 avx2_scan_pluswrap_u32(c0,c0,2*256,0);
#define RADIX_SUM_4_u8 avx2_scan_pluswrap_u8 (c0,c0,4*256,0); #define RADIX_SUM_4_u8 avx2_scan_pluswrap_u8 (c0,c0,4*256,0);
#define RADIX_SUM_4_u32 avx2_scan_pluswrap_u32(c0,c0,4*256,0); #define RADIX_SUM_4_u32 avx2_scan_pluswrap_u32(c0,c0,4*256,0);
#else #else
#define RADIX_SUM_1_u8 RDX_SUM_1(u8) #define RADIX_SUM_1_u8 RDX_SUM_1(u8)
#define RADIX_SUM_1_u32 RDX_SUM_1(u32)
#define RADIX_SUM_2_u8 RDX_SUM_2(u8) #define RADIX_SUM_2_u8 RDX_SUM_2(u8)
#define RADIX_SUM_2_u32 RDX_SUM_2(u32) #define RADIX_SUM_2_u32 RDX_SUM_2(u32)
#define RADIX_SUM_4_u8 RDX_SUM_4(u8) #define RADIX_SUM_4_u8 RDX_SUM_4(u8)
@ -34,3 +36,5 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v
#define RADIX_SUM_2_usz RDX_SUM_2(usz) #define RADIX_SUM_2_usz RDX_SUM_2(usz)
#define RADIX_SUM_4_usz RDX_SUM_4(usz) #define RADIX_SUM_4_usz RDX_SUM_4(usz)
#endif #endif
u8 radix_offsets_2_u32(usz* c0, u32* v0, usz n); // selfsearch.c

View File

@ -7,26 +7,39 @@ B not_c1(B t, B x);
B shape_c1(B t, B x); B shape_c1(B t, B x);
#include "radix.h" #include "radix.h"
#define RADIX_LOOKUP_i32(INIT, SETTAB) \ u8 radix_offsets_2_u32(usz* c0, u32* v0, usz n) {
/* Count keys */ \ usz rx = 256;
for (usz j=0; j<2*rx; j++) c0[j] = 0; \ usz* c1 = c0 + rx;
c1[0] = -n; \ // Count keys
for (usz i=0; i<n; i++) { u32 v=v0[i]; (c0+1)[(u8)(v>>24)]++; (c1+1)[(u8)(v>>16)]++; } \ for (usz j=0; j<2*rx+1; j++) c0[j] = 0;
/* Inclusive prefix sum; note c offsets above */ \ for (usz i=0; i<n; i++) { u32 v=v0[i]; (c0+1)[(u8)(v>>16)]++; (c1+1)[(u8)(v>>24)]++; }
RADIX_SUM_2_u32; \ u32 v=v0[0];
/* Radix moves */ \ // Inclusive prefix sum; note c offsets above
for (usz i=0; i<n; i++) { u32 v=v0[i]; u8 k=k0[i]=(u8)(v>>24); usz c=c0[k]++; v1[c]=v; } \ if ((c1+1)[(u8)(v>>24)] < n) { c1[0]-=n; RADIX_SUM_2_u32; return 2; }
for (usz i=0; i<n; i++) { u32 v=v1[i]; u8 k=k1[i]=(u8)(v>>16); usz c=c1[k]++; v2[c]=v; } \ if ((c0+1)[(u8)(v>>16)] < n) { RADIX_SUM_1_u32; return 1; }
/* Table lookup */ \ return 0;
u32 tv=v2[0]>>16; v2[n]=~v2[n-1]; \ }
for (usz l=0, i=0; l<n; ) { \ #define RADIX_LOOKUP_32(INIT, SETTAB) \
for (; ; l++) { u32 v=v2[l], t0=tv; tv=v>>16; if (tv!=t0) break; tab[(u16)v]=INIT; } \ /* Radix moves */ \
for (; i<l; i++) { u32 j=(u16)v2[i]; r2[i]=tab[j]; tab[j]SETTAB; } \ u8 bytes = radix_offsets_2_u32(c0, v0, n); \
} \ if (bytes==0) { \
/* Radix unmoves */ \ for (usz i=0; i<n; i++) { tab[(u16)v0[i]]=INIT; } \
*--c0 = *--c1 = 0; /* Move back one to account for increments in radix step */ \ for (usz i=0; i<n; i++) { u32 j=(u16)v0[i]; r0[i]=tab[j]; tab[j]SETTAB; } \
for (usz i=0; i<n; i++) { r1[i]=r2[c1[k1[i]]++]; } \ } else { \
for (usz i=0; i<n; i++) { r0[i]=r1[c0[k0[i]]++]; } \ if (bytes==1) { v1=v2; r1=r2; } \
for (usz i=0; i<n; i++) { u32 v=v0[i]; u8 k=k0[i]=(u8)(v>>16); usz c=c0[k]++; v1[c]=v; } \
if (bytes==2) for (usz i=0; i<n; i++) { u32 v=v1[i]; u8 k=k1[i]=(u8)(v>>24); usz c=c1[k]++; v2[c]=v; } \
/* Table lookup */ \
u32 tv=v2[0]>>16; v2[n]=~v2[n-1]; \
for (usz l=0, i=0; l<n; ) { \
for (; ; l++) { u32 v=v2[l], t0=tv; tv=v>>16; if (tv!=t0) break; tab[(u16)v]=INIT; } \
for (; i<l; i++) { u32 j=(u16)v2[i]; r2[i]=tab[j]; tab[j]SETTAB; } \
} \
/* Radix unmoves */ \
*--c0 = *--c1 = 0; /* Move back one to account for increments in radix step */ \
if (bytes==2) for (usz i=0; i<n; i++) { r1[i]=r2[c1[k1[i]]++]; } \
for (usz i=0; i<n; i++) { r0[i]=r1[c0[k0[i]]++]; } \
} \
decG(x); TFREE(alloc); decG(x); TFREE(alloc);
B memberOf_c1(B t, B x) { B memberOf_c1(B t, B x) {
@ -85,7 +98,7 @@ B memberOf_c1(B t, B x) {
u8 *r1 = (u8 *)(k1+n); // n [+..] r1 n ## u8 *r1 = (u8 *)(k1+n); // n [+..] r1 n ##
u8 *tab= (u8 *)(r1); // tn [+] tab tn ##### u8 *tab= (u8 *)(r1); // tn [+] tab tn #####
RADIX_LOOKUP_i32(1, =0) RADIX_LOOKUP_32(1, =0)
return num_squeeze(r); return num_squeeze(r);
} }
#undef BRUTE #undef BRUTE
@ -150,7 +163,7 @@ B count_c1(B t, B x) {
u32 *r1 = (u32*)v1; // n [+..] r1 4*n ######## u32 *r1 = (u32*)v1; // n [+..] r1 4*n ########
u32 *tab= (u32*)v1; // tn [+] tab 4*tn ########### u32 *tab= (u32*)v1; // tn [+] tab 4*tn ###########
RADIX_LOOKUP_i32(0, ++) RADIX_LOOKUP_32(0, ++)
return num_squeeze(r); return num_squeeze(r);
} }
#undef BRUTE #undef BRUTE