From 7cbd23c832f8d08cfebd321e240b679389cc9675 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 22 Oct 2022 22:16:55 -0400 Subject: [PATCH] Skip radix moves if the top bytes are all the same --- src/builtins/radix.h | 4 +++ src/builtins/selfsearch.c | 57 ++++++++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/builtins/radix.h b/src/builtins/radix.h index 29b93b08..f83414c1 100644 --- a/src/builtins/radix.h +++ b/src/builtins/radix.h @@ -13,12 +13,14 @@ extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); #define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0); +#define RADIX_SUM_1_u32 avx2_scan_pluswrap_u32(c0,c0, 256,0); #define RADIX_SUM_2_u8 avx2_scan_pluswrap_u8 (c0,c0,2*256,0); #define RADIX_SUM_2_u32 avx2_scan_pluswrap_u32(c0,c0,2*256,0); #define RADIX_SUM_4_u8 avx2_scan_pluswrap_u8 (c0,c0,4*256,0); #define RADIX_SUM_4_u32 avx2_scan_pluswrap_u32(c0,c0,4*256,0); #else #define RADIX_SUM_1_u8 RDX_SUM_1(u8) +#define RADIX_SUM_1_u32 RDX_SUM_1(u32) #define RADIX_SUM_2_u8 RDX_SUM_2(u8) #define RADIX_SUM_2_u32 RDX_SUM_2(u32) #define RADIX_SUM_4_u8 RDX_SUM_4(u8) @@ -34,3 +36,5 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v #define RADIX_SUM_2_usz RDX_SUM_2(usz) #define RADIX_SUM_4_usz RDX_SUM_4(usz) #endif + +u8 radix_offsets_2_u32(usz* c0, u32* v0, usz n); // selfsearch.c diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 53e4f8e9..e985ffd5 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -7,26 +7,39 @@ B not_c1(B t, B x); B shape_c1(B t, B x); #include "radix.h" -#define RADIX_LOOKUP_i32(INIT, SETTAB) \ - /* Count keys */ \ - for (usz j=0; j<2*rx; j++) c0[j] = 0; \ - c1[0] = -n; \ - for (usz i=0; i>24)]++; (c1+1)[(u8)(v>>16)]++; } \ - /* Inclusive prefix sum; note c offsets above */ \ - RADIX_SUM_2_u32; \ - /* Radix moves */ \ - for (usz i=0; i>24); usz c=c0[k]++; v1[c]=v; } \ - for (usz i=0; i>16); usz c=c1[k]++; v2[c]=v; } \ - /* Table lookup */ \ - u32 tv=v2[0]>>16; v2[n]=~v2[n-1]; \ - for (usz l=0, i=0; l>16; if (tv!=t0) break; tab[(u16)v]=INIT; } \ - for (; i>16)]++; (c1+1)[(u8)(v>>24)]++; } + u32 v=v0[0]; + // Inclusive prefix sum; note c offsets above + if ((c1+1)[(u8)(v>>24)] < n) { c1[0]-=n; RADIX_SUM_2_u32; return 2; } + if ((c0+1)[(u8)(v>>16)] < n) { RADIX_SUM_1_u32; return 1; } + return 0; +} +#define RADIX_LOOKUP_32(INIT, SETTAB) \ + /* Radix moves */ \ + u8 bytes = radix_offsets_2_u32(c0, v0, n); \ + if (bytes==0) { \ + for (usz i=0; i>16); usz c=c0[k]++; v1[c]=v; } \ + if (bytes==2) for (usz i=0; i>24); usz c=c1[k]++; v2[c]=v; } \ + /* Table lookup */ \ + u32 tv=v2[0]>>16; v2[n]=~v2[n-1]; \ + for (usz l=0, i=0; l>16; if (tv!=t0) break; tab[(u16)v]=INIT; } \ + for (; i