Add generic-architecture versions of Singeli scans
This commit is contained in:
parent
ec14d45c50
commit
00c454d37a
@ -650,7 +650,7 @@ cachedBin‿linkerCache ← {
|
||||
"xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold",
|
||||
"xag"‿"src/builtins/sort.c"‿"bins"
|
||||
|
||||
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
||||
"2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan",
|
||||
"2a."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq",
|
||||
"xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
||||
⟩
|
||||
|
||||
@ -78,22 +78,22 @@
|
||||
}
|
||||
|
||||
#if SINGELI_AVX2
|
||||
extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
extern void (*const si_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const si_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2);
|
||||
extern void (*const si_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
extern void (*const si_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
#define COUNT_THRESHOLD 32
|
||||
#define WRITE_SPARSE_i8 \
|
||||
for (usz i=0; i<n; i++) rp[i]=j; \
|
||||
while (ij<n) { rp[ij]=GRADE_UD(++j,--j); ij+=c0o[j]; } \
|
||||
GRADE_UD(avx2_scan_max_i8,avx2_scan_min_i8)(rp,rp,n);
|
||||
GRADE_UD(si_scan_max_i8,si_scan_min_i8)(rp,rp,n);
|
||||
#define WRITE_SPARSE_i16 \
|
||||
usz b = 1<<10; \
|
||||
for (usz k=0; ; ) { \
|
||||
usz e = b<n-k? k+b : n; \
|
||||
for (usz i=k; i<e; i++) rp[i]=j; \
|
||||
while (ij<e) { rp[ij]=GRADE_UD(++j,--j); ij+=c0o[j]; } \
|
||||
GRADE_UD(avx2_scan_max_i16,avx2_scan_min_i16)(rp+k,rp+k,e-k); \
|
||||
GRADE_UD(si_scan_max_i16,si_scan_min_i16)(rp+k,rp+k,e-k); \
|
||||
if (e==n) {break;} k=e; \
|
||||
}
|
||||
#define WRITE_SPARSE(T) WRITE_SPARSE_##T
|
||||
|
||||
@ -12,14 +12,14 @@
|
||||
#define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); }
|
||||
|
||||
#if SINGELI_AVX2
|
||||
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define RADIX_SUM_1_u8 avx2_scan_pluswrap_u8 (c0,c0, 256,0);
|
||||
#define RADIX_SUM_1_u32 avx2_scan_pluswrap_u32(c0,c0, 256,0);
|
||||
#define RADIX_SUM_2_u8 avx2_scan_pluswrap_u8 (c0,c0,2*256,0);
|
||||
#define RADIX_SUM_2_u32 avx2_scan_pluswrap_u32(c0,c0,2*256,0);
|
||||
#define RADIX_SUM_4_u8 avx2_scan_pluswrap_u8 (c0,c0,4*256,0);
|
||||
#define RADIX_SUM_4_u32 avx2_scan_pluswrap_u32(c0,c0,4*256,0);
|
||||
extern void (*const si_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define RADIX_SUM_1_u8 si_scan_pluswrap_u8 (c0,c0, 256,0);
|
||||
#define RADIX_SUM_1_u32 si_scan_pluswrap_u32(c0,c0, 256,0);
|
||||
#define RADIX_SUM_2_u8 si_scan_pluswrap_u8 (c0,c0,2*256,0);
|
||||
#define RADIX_SUM_2_u32 si_scan_pluswrap_u32(c0,c0,2*256,0);
|
||||
#define RADIX_SUM_4_u8 si_scan_pluswrap_u8 (c0,c0,4*256,0);
|
||||
#define RADIX_SUM_4_u32 si_scan_pluswrap_u32(c0,c0,4*256,0);
|
||||
#else
|
||||
#define RADIX_SUM_1_u8 RDX_SUM_1(u8)
|
||||
#define RADIX_SUM_1_u32 RDX_SUM_1(u32)
|
||||
@ -30,9 +30,9 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v
|
||||
#endif
|
||||
|
||||
#if SINGELI_AVX2 && !USZ_64
|
||||
#define RADIX_SUM_1_usz avx2_scan_pluswrap_u32(c0,c0, 256,0);
|
||||
#define RADIX_SUM_2_usz avx2_scan_pluswrap_u32(c0,c0,2*256,0);
|
||||
#define RADIX_SUM_4_usz avx2_scan_pluswrap_u32(c0,c0,4*256,0);
|
||||
#define RADIX_SUM_1_usz si_scan_pluswrap_u32(c0,c0, 256,0);
|
||||
#define RADIX_SUM_2_usz si_scan_pluswrap_u32(c0,c0,2*256,0);
|
||||
#define RADIX_SUM_4_usz si_scan_pluswrap_u32(c0,c0,4*256,0);
|
||||
#else
|
||||
#define RADIX_SUM_1_usz RDX_SUM_1(usz)
|
||||
#define RADIX_SUM_2_usz RDX_SUM_2(usz)
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
static u64 vg_rand(u64 x) { return x; }
|
||||
#endif
|
||||
|
||||
#if SINGELI_AVX2
|
||||
#if SINGELI
|
||||
#define SINGELI_FILE scan
|
||||
#include "../utils/includeSingeli.h"
|
||||
#if __PCLMUL__
|
||||
@ -78,8 +78,8 @@ B scan_add_bool(B x, u64 ia) { // consumes x
|
||||
} else {
|
||||
void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re));
|
||||
#define SUM_BITWISE(T) { T c=0; for (usz i=0; i<ia; i++) { c+= bitp_get(xp,i); ((T*)rp)[i]=c; } }
|
||||
#if SINGELI_AVX2
|
||||
#define SUM(W,T) avx2_bcs##W(xp, rp, ia);
|
||||
#if SINGELI
|
||||
#define SUM(W,T) si_bcs##W(xp, rp, ia);
|
||||
#else
|
||||
#define SUM(W,T) SUM_BITWISE(T)
|
||||
#endif
|
||||
@ -96,8 +96,8 @@ B scan_add_bool(B x, u64 ia) { // consumes x
|
||||
}
|
||||
|
||||
// min/max-scan
|
||||
#if SINGELI_AVX2
|
||||
#define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I);
|
||||
#if SINGELI
|
||||
#define MINMAX_SCAN(T,NAME,C,I) si_scan_##NAME##_init_##T(xp, rp, ia, I);
|
||||
#else
|
||||
#define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i<ia; i++) { if (xp[i] C c)c=xp[i]; rp[i]=c; }
|
||||
#endif
|
||||
@ -155,16 +155,16 @@ static B scan_lt(B x, u64 p, usz ia) {
|
||||
static B scan_plus(f64 r0, B x, u8 xe, usz ia) {
|
||||
assert(xe!=el_bit && elNum(xe));
|
||||
B r; void* rp = m_tyarrv(&r, xe==el_f64? sizeof(f64) : sizeof(i32), ia, xe==el_f64? t_f64arr : t_i32arr);
|
||||
#if SINGELI_AVX2
|
||||
#if SINGELI
|
||||
switch(xe) { default:UD;
|
||||
case el_i8: { if (!q_fi32(r0) || simd_scan_plus_i8_i32 (i8any_ptr(x), r0, rp, ia)!=ia) goto cs_i8_f64; decG(x); return r; }
|
||||
case el_i16: { if (!q_fi32(r0) || simd_scan_plus_i16_i32(i16any_ptr(x), r0, rp, ia)!=ia) goto cs_i16_f64; decG(x); return r; }
|
||||
case el_i32: { if (!q_fi32(r0) || simd_scan_plus_i32_i32(i32any_ptr(x), r0, rp, ia)!=ia) goto cs_i32_f64; decG(x); return r; }
|
||||
case el_i8: { if (!q_fi32(r0) || si_scan_plus_i8_i32 (i8any_ptr(x), r0, rp, ia)!=ia) goto cs_i8_f64; decG(x); return r; }
|
||||
case el_i16: { if (!q_fi32(r0) || si_scan_plus_i16_i32(i16any_ptr(x), r0, rp, ia)!=ia) goto cs_i16_f64; decG(x); return r; }
|
||||
case el_i32: { if (!q_fi32(r0) || si_scan_plus_i32_i32(i32any_ptr(x), r0, rp, ia)!=ia) goto cs_i32_f64; decG(x); return r; }
|
||||
case el_f64: { f64* xp=f64any_ptr(x); f64 c=r0; for (usz i=0; i<ia; i++) { c+= xp[i]; ((f64*)rp)[i]=c; } decG(x); return r; }
|
||||
}
|
||||
cs_i8_f64: { x=taga(cpyI16Arr(x)); goto cs_i16_f64; }
|
||||
cs_i16_f64: { decG(r); f64* rp; r = m_f64arrv(&rp, ia); simd_scan_plus_i16_f64(i16any_ptr(x), r0, rp, ia); decG(x); return r; }
|
||||
cs_i32_f64: { decG(r); f64* rp; r = m_f64arrv(&rp, ia); simd_scan_plus_i32_f64(i32any_ptr(x), r0, rp, ia); decG(x); return r; }
|
||||
cs_i16_f64: { decG(r); f64* rp; r = m_f64arrv(&rp, ia); si_scan_plus_i16_f64(i16any_ptr(x), r0, rp, ia); decG(x); return r; }
|
||||
cs_i32_f64: { decG(r); f64* rp; r = m_f64arrv(&rp, ia); si_scan_plus_i32_f64(i32any_ptr(x), r0, rp, ia); decG(x); return r; }
|
||||
#else
|
||||
if (xe==el_i8 && q_fi32(r0)) { i8* xp=i8any_ptr (x); i32 c=r0; for (usz i=0; i<ia; i++) { if (addOn(c,xp[i])) goto base; ((i32*)rp)[i]=c; } decG(x); return r; }
|
||||
if (xe==el_i16 && q_fi32(r0)) { i16* xp=i16any_ptr(x); i32 c=r0; for (usz i=0; i<ia; i++) { if (addOn(c,xp[i])) goto base; ((i32*)rp)[i]=c; } decG(x); return r; }
|
||||
|
||||
@ -98,13 +98,18 @@
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE count
|
||||
#include "../utils/includeSingeli.h"
|
||||
|
||||
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define avx2_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i<e; i++) js=rp[i]+=js;
|
||||
#define PLUS_SCAN(T) avx2_scan_pluswrap_##T(rp+k,rp+k,e-k,js); js=rp[e-1];
|
||||
extern void (*const avx2_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2);
|
||||
#endif
|
||||
|
||||
#if SINGELI
|
||||
extern void (*const si_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const si_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||
extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
#define ALIAS(I,U) static void si_scan_pluswrap_##I(I* a, I* b, u64 c, I d) { si_scan_pluswrap_##U((U*)a, (U*)b, c, d); }
|
||||
ALIAS(i8,u8) ALIAS(i16,u16) ALIAS(i32,u32)
|
||||
#undef ALIAS
|
||||
#define si_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i<e; i++) js=rp[i]+=js;
|
||||
#define PLUS_SCAN(T) si_scan_pluswrap_##T(rp+k,rp+k,e-k,js); js=rp[e-1];
|
||||
extern void (*const si_scan_max_i32)(int32_t* v0,int32_t* v1,uint64_t v2);
|
||||
#else
|
||||
#define PLUS_SCAN(T) for (usz i=k; i<e; i++) js=rp[i]+=js;
|
||||
#endif
|
||||
@ -548,7 +553,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
|
||||
#if SINGELI_AVX2
|
||||
#define IND_BY_SCAN \
|
||||
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, avx2_scan_max_i32(rp+k,rp+k,e-k))
|
||||
SCAN_CORE(xp[j], rp[ij]=j, rp[k]=j, si_scan_max_i32(rp+k,rp+k,e-k))
|
||||
#else
|
||||
#define IND_BY_SCAN usz js=0; SUM_CORE(i32, xp[j], , 1)
|
||||
#endif
|
||||
|
||||
@ -5,6 +5,11 @@ include './avx2'
|
||||
include './mask'
|
||||
include './f64'
|
||||
|
||||
# Initialized scan, generic implementation
|
||||
fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = {
|
||||
@for (x, r over len) r = m = op{m, x}
|
||||
}
|
||||
|
||||
def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}}
|
||||
def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}}
|
||||
|
||||
@ -45,7 +50,8 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
|
||||
}
|
||||
|
||||
# Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈
|
||||
fn avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
def scan_idem = scan_scal
|
||||
fn scan_idem{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
# Within each lane, scan using shifts by powers of 2. First k elements
|
||||
# when shifting by k don't need to change, so leave them alone.
|
||||
def w = width{T}
|
||||
@ -61,23 +67,22 @@ fn avx2_scan_idem{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
|
||||
scan_post{T, init, x, r, len, op, pre}
|
||||
}
|
||||
fn avx2_scan_idem{T==f64, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
fn scan_idem{T==f64, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
def sh{s, a} = op{a, shuf{[4]u64, a, s}}
|
||||
scan_post{T, init, x, r, len, op, {a}=>sh{4b1110,sh{4b2200,a}}}
|
||||
}
|
||||
export{'avx2_scan_min_init_i8', avx2_scan_idem{i8 , min}}; export{'avx2_scan_max_init_i8', avx2_scan_idem{i8 , max}}
|
||||
export{'avx2_scan_min_init_i16', avx2_scan_idem{i16, min}}; export{'avx2_scan_max_init_i16', avx2_scan_idem{i16, max}}
|
||||
export{'avx2_scan_min_init_i32', avx2_scan_idem{i32, min}}; export{'avx2_scan_max_init_i32', avx2_scan_idem{i32, max}}
|
||||
export{'avx2_scan_min_init_f64', avx2_scan_idem{f64, min}}; export{'avx2_scan_max_init_f64', avx2_scan_idem{f64, max}}
|
||||
|
||||
fn avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
|
||||
def m = 1 << (width{T}-1)
|
||||
def id = (if (same{op,min}) m-1; else -m)
|
||||
avx2_scan_idem{T, op}(x, r, len, id)
|
||||
export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}}
|
||||
export{'si_scan_min_init_i16', scan_idem{i16, min}}; export{'si_scan_max_init_i16', scan_idem{i16, max}}
|
||||
export{'si_scan_min_init_i32', scan_idem{i32, min}}; export{'si_scan_max_init_i32', scan_idem{i32, max}}
|
||||
export{'si_scan_min_init_f64', scan_idem{f64, min}}; export{'si_scan_max_init_f64', scan_idem{f64, max}}
|
||||
|
||||
fn scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
|
||||
scan_idem{T, op}(x, r, len, (if (same{op,min}) maxvalue else minvalue){T})
|
||||
}
|
||||
export{'avx2_scan_min_i8', avx2_scan_idem_id{i8 , min}}; export{'avx2_scan_max_i8', avx2_scan_idem_id{i8 , max}}
|
||||
export{'avx2_scan_min_i16', avx2_scan_idem_id{i16, min}}; export{'avx2_scan_max_i16', avx2_scan_idem_id{i16, max}}
|
||||
export{'avx2_scan_min_i32', avx2_scan_idem_id{i32, min}}; export{'avx2_scan_max_i32', avx2_scan_idem_id{i32, max}}
|
||||
export{'si_scan_min_i8', scan_idem_id{i8 , min}}; export{'si_scan_max_i8', scan_idem_id{i8 , max}}
|
||||
export{'si_scan_min_i16', scan_idem_id{i16, min}}; export{'si_scan_max_i16', scan_idem_id{i16, max}}
|
||||
export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', scan_idem_id{i32, max}}
|
||||
|
||||
# Assumes identity is 0
|
||||
def scan_assoc{op, a:T} = {
|
||||
@ -92,17 +97,22 @@ def scan_assoc{op, a:T} = {
|
||||
def scan_plus = scan_assoc{+, .}
|
||||
|
||||
# Associative scan
|
||||
fn avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
def scan_assoc_0 = scan_scal
|
||||
fn scan_assoc_0{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
# Prefix op on entire AVX register
|
||||
|
||||
scan_post{T, init, x, r, len, op, scan_plus}
|
||||
}
|
||||
export{'avx2_scan_pluswrap_u8', avx2_scan_assoc_0{u8 , +}}
|
||||
export{'avx2_scan_pluswrap_u16', avx2_scan_assoc_0{u16, +}}
|
||||
export{'avx2_scan_pluswrap_u32', avx2_scan_assoc_0{u32, +}}
|
||||
export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}}
|
||||
export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}}
|
||||
export{'si_scan_pluswrap_u32', scan_assoc_0{u32, +}}
|
||||
|
||||
# Boolean cumulative sum
|
||||
fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
fn bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
def bitp_get{arr, n} = (load{arr, n>>6} >> (n&63)) & 1
|
||||
c:T = 0
|
||||
@for (r over i to l) { c+= cast_i{T, bitp_get{x,i}}; r = c }
|
||||
}
|
||||
fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
|
||||
def U = ty_u{T}
|
||||
def w = width{T}
|
||||
def vl= 256 / w
|
||||
@ -157,9 +167,9 @@ fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = {
|
||||
step{load{xv, e}, e, st}
|
||||
}
|
||||
}
|
||||
export{'avx2_bcs8', avx2_bcs{i8}}
|
||||
export{'avx2_bcs16', avx2_bcs{i16}}
|
||||
export{'avx2_bcs32', avx2_bcs{i32}}
|
||||
export{'si_bcs8', bcs{i8}}
|
||||
export{'si_bcs16', bcs{i16}}
|
||||
export{'si_bcs32', bcs{i32}}
|
||||
|
||||
|
||||
|
||||
@ -190,7 +200,23 @@ def maxabsval{T & issigned{T}} = -minvalue{T}
|
||||
def maxsafeint{T & issigned{T}} = maxvalue{T}
|
||||
def maxsafeint{T==f64} = 1<<53
|
||||
|
||||
def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = {
|
||||
fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
|
||||
i:u64 = 0
|
||||
if (hasarch{'AVX2'}) simd_plus_scan_part{X,R}{x, c, r, len, i}
|
||||
@forUnroll{1,1} (js from i to len) {
|
||||
def vs = eachx{load, x, js}
|
||||
each{{j, v} => {
|
||||
def {b,n} = addChk{c, promote{R, v}}
|
||||
if (rare{b}) return{j}
|
||||
store{r, j, n}
|
||||
c = n
|
||||
}, js, vs}
|
||||
}
|
||||
len
|
||||
}
|
||||
# Sum as many vector registers as possible; modifies c and i
|
||||
def simd_plus_scan_part{X, R}{x:*X, c:(R), r:*R, len:u64, i:u64} = {
|
||||
def b = max{width{R}/2, width{X}}
|
||||
def bulk = arch_defvw/b
|
||||
|
||||
def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size
|
||||
@ -203,7 +229,6 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = {
|
||||
|
||||
if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} }
|
||||
|
||||
i:u64 = 0
|
||||
cv:= [arch_defvw/width{R}]R ** c
|
||||
|
||||
if (R==f64 and c != floor{c}) goto{'end'}
|
||||
@ -237,24 +262,13 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = {
|
||||
|
||||
setlabel{'end'}
|
||||
c = extract{cv, 0}
|
||||
|
||||
@forUnroll{1,1} (js from i to len) {
|
||||
def vs = eachx{load, x, js}
|
||||
each{{j, v} => {
|
||||
def {b,n} = addChk{c, promote{R, v}}
|
||||
if (rare{b}) return{j}
|
||||
store{r, j, n}
|
||||
c = n
|
||||
}, js, vs}
|
||||
}
|
||||
len
|
||||
}
|
||||
fn simd_plus_scanG{X, b, R}(x:*X, c:R, r:*R, len:u64) : void = simd_plus_scan{X,b,R}{x, c, r, len}
|
||||
fn simd_plus_scanC{X, b, R}(x:*X, c:R, r:*R, len:u64) : u64 = simd_plus_scan{X,b,R}{x, c, r, len}
|
||||
def plus_scanG{X, R} = plus_scan{X, R, void}
|
||||
def plus_scanC{X, R} = plus_scan{X, R, u64}
|
||||
|
||||
export{'simd_scan_plus_i8_i32', simd_plus_scanC{i8, 16, i32}}
|
||||
export{'simd_scan_plus_i16_i32', simd_plus_scanC{i16, 16, i32}}
|
||||
export{'simd_scan_plus_i32_i32', simd_plus_scanC{i32, 32, i32}}
|
||||
export{'si_scan_plus_i8_i32', plus_scanC{i8, i32}}
|
||||
export{'si_scan_plus_i16_i32', plus_scanC{i16, i32}}
|
||||
export{'si_scan_plus_i32_i32', plus_scanC{i32, i32}}
|
||||
|
||||
export{'simd_scan_plus_i16_f64', simd_plus_scanG{i16, 32, f64}}
|
||||
export{'simd_scan_plus_i32_f64', simd_plus_scanG{i32, 32, f64}}
|
||||
export{'si_scan_plus_i16_f64', plus_scanG{i16, f64}}
|
||||
export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user