Port replicate-by-scan code from C to Singeli
This commit is contained in:
parent
771496a25c
commit
e50ae40f8b
@ -661,7 +661,7 @@ cachedBin‿linkerCache ← {
|
||||
"xag"‿"src/builtins/scan.c"‿"scan", "xa."‿"src/builtins/fold.c"‿"fold",
|
||||
"xag"‿"src/builtins/sort.c"‿"bins"
|
||||
|
||||
"x.."‿"src/builtins/select.c"‿"select", "xa."‿"src/builtins/slash.c"‿"constrep",
|
||||
"x.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/slash.c"‿"constrep",
|
||||
"xag"‿"src/builtins/slash.c"‿"slash", "xa."‿"src/builtins/slash.c"‿"count"
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
@ -86,11 +86,12 @@
|
||||
#endif
|
||||
|
||||
#if SINGELI
|
||||
extern void (*const si_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const si_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||
extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
extern void (*const si_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2);
|
||||
#define SINGELI_FILE slash
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
#if SINGELI_AVX2 || SINGELI_NEON
|
||||
#define SINGELI_FILE constrep
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
@ -100,20 +101,6 @@
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
#if SINGELI
|
||||
extern void (*const si_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const si_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||
extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define ALIAS(I,U) static void si_scan_pluswrap_##I(I* a, I* b, u64 c, I d) { si_scan_pluswrap_##U((U*)a, (U*)b, c, d); }
|
||||
ALIAS(i8,u8) ALIAS(i16,u16) ALIAS(i32,u32)
|
||||
#undef ALIAS
|
||||
#define si_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i<e; i++) js=rp[i]+=js;
|
||||
#define PLUS_SCAN(T) si_scan_pluswrap_##T(rp+k,rp+k,e-k,js); js=rp[e-1];
|
||||
extern void (*const si_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2);
|
||||
#else
|
||||
#define PLUS_SCAN(T) for (usz i=k; i<e; i++) js=rp[i]+=js;
|
||||
#endif
|
||||
|
||||
// Dense Where, still significantly worse than SIMD
|
||||
// Assumes modifiable DST
|
||||
#define WHERE_DENSE(SRC, DST, LEN, OFF) do { \
|
||||
@ -537,32 +524,6 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
return r;
|
||||
}
|
||||
|
||||
// Replicate using plus/max/xor-scan
|
||||
#define SCAN_CORE(WV, UPD, SET, SCAN) \
|
||||
usz b = 1<<10; \
|
||||
for (usz k=0, j=0, ij=WV; ; ) { \
|
||||
usz e = b<s-k? k+b : s; \
|
||||
for (usz i=k; i<e; i++) rp[i]=0; \
|
||||
SET; \
|
||||
while (ij<e) { j++; UPD; ij+=WV; } \
|
||||
SCAN; \
|
||||
if (e==s) {break;} k=e; \
|
||||
}
|
||||
#define SUM_CORE(T, WV, PREP, INC) \
|
||||
SCAN_CORE(WV, PREP; rp[ij]+=INC, , PLUS_SCAN(T))
|
||||
|
||||
#if SINGELI_AVX2
|
||||
#define IND_BY_SCAN \
|
||||
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, si_scan_max_i32(rp+k,rp+k,e-k))
|
||||
#else
|
||||
#define IND_BY_SCAN usz js=0; SUM_CORE(i32, xp[j], , 1)
|
||||
#endif
|
||||
|
||||
#define REP_BY_SCAN(T, WV) \
|
||||
T* xp = xv; T* rp = rv; \
|
||||
T js=xp[0], px=js; \
|
||||
SUM_CORE(T, WV, T sx=px, (px=xp[j])-sx)
|
||||
|
||||
#define BOOL_REP_XOR_SCAN(WV) \
|
||||
usz b = 1<<12; \
|
||||
u64 xx=xp[0], xs=xx>>63, js=-(xx&1); xx^=xx<<1; \
|
||||
@ -622,14 +583,13 @@ B slash_c1(B t, B x) {
|
||||
for (u64 j = 0; j < c; j++) *rp++ = i;
|
||||
}
|
||||
} else {
|
||||
#if SINGELI
|
||||
if (s/32 <= xia) { // Sparse case: type of x matters
|
||||
#define SPARSE_IND(T) T* xp = T##any_ptr(x); IND_BY_SCAN
|
||||
i32* rp; r = m_i32arrv(&rp, s);
|
||||
if (xe == el_i8 ) { SPARSE_IND(i8 ); }
|
||||
else if (xe == el_i16) { SPARSE_IND(i16); }
|
||||
else { SPARSE_IND(i32); }
|
||||
#undef SPARSE_IND
|
||||
} else { // Dense case: only result type matters
|
||||
si_indices_scan_i32[elwByteLog(xe)](tyany_ptr(x), rp, s);
|
||||
} else
|
||||
#endif
|
||||
{ // Dense case: only result type matters
|
||||
#define DENSE_IND(T) \
|
||||
T* rp; r = m_##T##arrv(&rp, s); \
|
||||
for (u64 i = 0; i < xia; i++) { \
|
||||
@ -748,17 +708,12 @@ B slash_c2(B t, B w, B x) {
|
||||
void* rv = m_tyarrlv(&r, xk, s, xt);
|
||||
if (rsh) { Arr* ra=a(r); SPRNK(ra,xr); ra->sh = rsh; ra->ia = s*arr_csz(x); }
|
||||
void* xv = tyany_ptr(x);
|
||||
#if SINGELI
|
||||
if ((xk<3? s/64 : s/32) <= wia) { // Sparse case: use both types
|
||||
#define CASE(L,XT) case L: { REP_BY_SCAN(XT, wp[j]) break; }
|
||||
#define SPARSE_REP(WT) \
|
||||
WT* wp = WT##any_ptr(w); \
|
||||
switch (xk) { default: UD; CASE(0,u8) CASE(1,u16) CASE(2,u32) CASE(3,u64) }
|
||||
if (we == el_i8 ) { SPARSE_REP(i8 ); }
|
||||
else if (we == el_i16) { SPARSE_REP(i16); }
|
||||
else { SPARSE_REP(i32); }
|
||||
#undef SPARSE_REP
|
||||
#undef CASE
|
||||
} else { // Dense case: only type of x matters
|
||||
si_replicate_scan[4*elwByteLog(we) + xk](tyany_ptr(w), xv, rv, s);
|
||||
} else
|
||||
#endif
|
||||
{ // Dense case: only type of x matters
|
||||
#define CASE(L,T) case L: { \
|
||||
T* xp = xv; T* rp = rv; \
|
||||
for (usz i = 0; i < wia; i++) { \
|
||||
@ -845,13 +800,18 @@ B slash_c2(B t, B w, B x) {
|
||||
u8 xk = xl-3;
|
||||
void* rv = m_tyarrv(&r, 1<<xk, s, xt);
|
||||
void* xv = tyany_ptr(x);
|
||||
#if SINGELI_AVX2 || SINGELI_NEON
|
||||
simd_constrep[xk](wv, xv, rv, xlen);
|
||||
#if SINGELI
|
||||
si_constrep[xk](wv, xv, rv, xlen);
|
||||
#else
|
||||
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }
|
||||
#define CASE(L,T) case L: { \
|
||||
T* xp = xv; T* rp = rv; \
|
||||
for (usz i = 0; i < xlen; i++) { \
|
||||
for (i64 j = 0; j < wv; j++) *rp++ = xp[i]; \
|
||||
} \
|
||||
} break;
|
||||
switch (xk) { default: UD; CASE(0,u8) CASE(1,u16) CASE(2,u32) CASE(3,u64) }
|
||||
#endif
|
||||
#undef CASE
|
||||
#endif
|
||||
}
|
||||
|
||||
atmW_maybesh:;
|
||||
|
||||
@ -1,8 +1,72 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'} | hasarch{'AARCH64'}) {
|
||||
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
def ind_types = tup{i8, i16, i32}
|
||||
def dat_types = tup{...ind_types, u64}
|
||||
|
||||
# Indices and Replicate using plus- or max-scan
|
||||
def scan_core{upd, set, scan, rp:pT, wp:W, s:(usz)} = {
|
||||
def getw{j} = if (isptr{W}) cast_i{usz,load{wp,j}} else wp
|
||||
b:usz = 1<<10
|
||||
k:usz = 0; j:usz = 0; ij:=getw{j}
|
||||
while (1) {
|
||||
e := tern{b<s-k, k+b, s}
|
||||
@for (rp over i from k to e) rp = 0
|
||||
if (set) store{rp, k, cast_i{eltype{pT},j}}
|
||||
while (ij<e) { ++j; upd{rp, j, ij}; ij+=getw{j} }
|
||||
scan{rp+k, e-k}
|
||||
if (e==s) return{}
|
||||
k = e
|
||||
}
|
||||
}
|
||||
def indrep_by_sum{T, rp:*T, wp, s:(usz), js, inc} = {
|
||||
def scan{ptr, len} = @for (ptr over len) js=ptr+=js
|
||||
def scan{ptr, len & width{T}<=32} = {
|
||||
def scanfn = merge{'si_scan_pluswrap_u',fmtnat{width{T}}}
|
||||
p := *ty_u{eltype{type{ptr}}}~~ptr
|
||||
emit{void, scanfn, p, p, len, js}; js=load{ptr,len-1}
|
||||
}
|
||||
def upd{rp, j, ij} = store{rp, ij, load{rp,ij}+inc{j}}
|
||||
scan_core{upd, 0, scan, rp, wp, s}
|
||||
}
|
||||
|
||||
fn ind_by_scan_i32{W}(xv:*void, rp:*i32, s:usz) : void = {
|
||||
xp := *W~~xv
|
||||
if (hasarch{'X86_64'} & ~hasarch{'SSE4.1'}) { # no min instruction
|
||||
js:i32 = 0
|
||||
indrep_by_sum{i32, rp, xp, s, js, {j}=>1}
|
||||
} else {
|
||||
scan_core{
|
||||
{rp,j,ij} => store{rp,ij,cast_i{i32,j}}, 1,
|
||||
{ptr,len} => emit{void, 'si_scan_max_i32', ptr,ptr,len},
|
||||
rp, xp, s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def rep_by_scan{T, wp, xv:*void, rv:*void, s} = {
|
||||
xp := *T~~xv; js := *xp; px := js
|
||||
def inc{j} = {sx:=px; px=load{xp,j}; px-sx}
|
||||
indrep_by_sum{T, *T~~rv, wp, s, js, inc}
|
||||
}
|
||||
fn rep_by_scan{W, T}(wp:*void, xv:*void, rv:*void, s:usz) : void = {
|
||||
rep_by_scan{T, *W~~wp, xv, rv, s}
|
||||
}
|
||||
|
||||
exportT{'si_indices_scan_i32', each{ind_by_scan_i32, ind_types}}
|
||||
exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}}
|
||||
|
||||
|
||||
# Constant replicate
|
||||
if (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) {
|
||||
|
||||
fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}}
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
def incl{a,b} = slice{iota{b+1},a}
|
||||
|
||||
# 1+˝∨`⌾⌽0=div|⌜range
|
||||
@ -220,6 +284,6 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}}
|
||||
}
|
||||
|
||||
}
|
||||
exportT{'si_constrep', each{rep_const, dat_types}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user