diff --git a/src/builtins/sfns.c b/src/builtins/sfns.c index 30994217..9a6804a1 100644 --- a/src/builtins/sfns.c +++ b/src/builtins/sfns.c @@ -436,6 +436,19 @@ B select_c2(B t, B w, B x) { return c2(rt_select, w, x); } + +#ifdef __BMI2__ + #include + u64 c16lut[] = {0x0000000000000000, 0x000000000000ffff, 0x00000000ffff0000, 0x00000000ffffffff, 0x0000ffff00000000, 0x0000ffff0000ffff, 0x0000ffffffff0000, 0x0000ffffffffffff, 0xffff000000000000, 0xffff00000000ffff, 0xffff0000ffff0000, 0xffff0000ffffffff, 0xffffffff00000000, 0xffffffff0000ffff, 0xffffffffffff0000, 0xffffffffffffffff}; + + #if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/slash.c" + #pragma GCC diagnostic pop + #endif +#endif + extern B rt_slash; B slash_c1(B t, B x) { if (RARE(isAtm(x)) || RARE(rnk(x)!=1)) thrF("/: Argument must have rank 1 (%H ≡ ≢𝕩)", x); @@ -453,61 +466,60 @@ B slash_c1(B t, B x) { dec(x); return r; } - i32* rp; B r = m_i32arrv(&rp, s); + B r; u8 xe = TI(x,elType); - if (xe==el_bit) { + #if SINGELI && defined(__BMI2__) + if (xia<=32768 && xe==el_bit) { u64* xp = bitarr_ptr(x); - while (xia>0 && !bitp_get(xp,xia-1)) xia--; - for (u64 i = 0; i < xia; i++) { - *rp = i; - rp+= bitp_get(xp, i); - } - } else if (xe==el_i8) { - i8* xp = i8any_ptr(x); - while (xia>0 && !xp[xia-1]) xia--; - for (u64 i = 0; i < xia; i++) { - i32 c = xp[i]; - if (LIKELY(c==0 || c==1)) { + if (xia<=128) { i8* rp; r = m_i8arrv (&rp, s+8); a(r)->ia-= 8; bmipopc_1slash8 (xp, rp, xia); } + else { i16* rp; r = m_i16arrv(&rp, s+16); a(r)->ia-= 16; bmipopc_1slash16(xp, rp, xia); } + } else + #endif + { + i32* rp; r = m_i32arrv(&rp, s); + if (xe==el_bit) { + u64* xp = bitarr_ptr(x); + while (xia>0 && !bitp_get(xp,xia-1)) xia--; + for (u64 i = 0; i < xia; i++) { *rp = i; - rp+= c; - } else { - for (i32 j = 0; j < c; j++) *rp++ = i; + rp+= bitp_get(xp, i); } - } - } else if (xe==el_i32) { - i32* xp = i32any_ptr(x); - while (xia>0 && !xp[xia-1]) xia--; - for (u64 i = 0; i < xia; i++) { - i32 c = xp[i]; - if (LIKELY(c==0 || c==1)) { - *rp = i; - rp+= c; - } else { - for (i32 j = 0; j < c; j++) *rp++ = i; + } else if (xe==el_i8) { + i8* xp = i8any_ptr(x); + while (xia>0 && !xp[xia-1]) xia--; + for (u64 i = 0; i < xia; i++) { + i32 c = xp[i]; + if (LIKELY(c==0 || c==1)) { + *rp = i; + rp+= c; + } else { + for (i32 j = 0; j < c; j++) *rp++ = i; + } + } + } else if (xe==el_i32) { + i32* xp = i32any_ptr(x); + while (xia>0 && !xp[xia-1]) xia--; + for (u64 i = 0; i < xia; i++) { + i32 c = xp[i]; + if (LIKELY(c==0 || c==1)) { + *rp = i; + rp+= c; + } else { + for (i32 j = 0; j < c; j++) *rp++ = i; + } + } + } else { + SLOW1("/𝕩", x); + SGetU(x) + for (u64 i = 0; i < xia; i++) { + usz c = o2s(GetU(x, i)); + for (u64 j = 0; j < c; j++) *rp++ = i; } - } - } else { - SLOW1("/𝕩", x); - SGetU(x) - for (u64 i = 0; i < xia; i++) { - usz c = o2s(GetU(x, i)); - for (u64 j = 0; j < c; j++) *rp++ = i; } } dec(x); return r; } -#ifdef __BMI2__ - #include - u64 c16lut[] = {0x0000000000000000, 0x000000000000ffff, 0x00000000ffff0000, 0x00000000ffffffff, 0x0000ffff00000000, 0x0000ffff0000ffff, 0x0000ffffffff0000, 0x0000ffffffffffff, 0xffff000000000000, 0xffff00000000ffff, 0xffff0000ffff0000, 0xffff0000ffffffff, 0xffffffff00000000, 0xffffffff0000ffff, 0xffffffffffff0000, 0xffffffffffffffff}; - - #if SINGELI - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-variable" - #include "../singeli/gen/slash.c" - #pragma GCC diagnostic pop - #endif -#endif B slash_c2(B t, B w, B x) { if (isArr(x) && rnk(x)==1 && isArr(w) && rnk(w)==1 && depth(w)==1) { @@ -552,8 +564,8 @@ B slash_c2(B t, B w, B x) { #endif #if SINGELI && defined(__BMI2__) - case el_i8: { i8* xp = i8any_ptr (x); i8* rp; r = m_i8arrv (&rp, wsum+8); a(r)->ia-= 8; comp8 (wp, xp, rp, wia); break; } - case el_i16: { i16* xp = i16any_ptr(x); i16* rp; r = m_i16arrv(&rp, wsum+16); a(r)->ia-= 16; comp16(wp, xp, rp, wia); break; } + case el_i8: { i8* xp = i8any_ptr (x); i8* rp; r = m_i8arrv (&rp, wsum+8); a(r)->ia-= 8; bmipopc_2slash8 (wp, xp, rp, wia); break; } + case el_i16: { i16* xp = i16any_ptr(x); i16* rp; r = m_i16arrv(&rp, wsum+16); a(r)->ia-= 16; bmipopc_2slash16(wp, xp, rp, wia); break; } #else case el_i8: { i8* xp = i8any_ptr (x); i8* rp; r = m_i8arrv (&rp,wsum); for (usz i=0; i>4} -# } -# } -# 'comp16'=comp16 - -comp16(w:*u64, x:*i16, r:*i16, l:u64) : void = { - xv:= reinterpret{*u64, x} +def comp16{w:*u64, X, r:*i16, l:u64} = { @for(w in reinterpret{*u8,w} over i to cdiv{l,8}) { def step{w} = { pc:= popc{w} - store{reinterpret{*u64,r}, 0, pext{load{xv,0}, load{emit{*u64,'','c16lut'}, w}}} + store{reinterpret{*u64,r}, 0, pext{promote{u64,X{}}, load{emit{*u64,'','c16lut'}, w}}} r+= pc - xv+= 1 } step{w&15} step{w>>4} } } -'comp16'=comp16 \ No newline at end of file + +slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = { + xv:= reinterpret{*u64, x} + F{w, {} => {c:=load{xv,0}; xv+=1; c}, r, l} +} + +slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = { + x:u64 = iota + F{w, {} => {c:=x; x+= add; c}, r, l} +} + +'bmipopc_2slash8' = slash2{comp8, i8} +'bmipopc_2slash16' = slash2{comp16, i16} +'bmipopc_1slash8' = slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808} +'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004} \ No newline at end of file