Merge pull request #48 from mlochbaum/singelirep
Fast constant Replicate
This commit is contained in:
commit
d1783e4706
@ -1 +1 @@
|
|||||||
Subproject commit 4a4cee084d0e0f13593ed1feb678aa85bde02002
|
Subproject commit 88901dc87defdb281a1496e402a70afa96b02020
|
||||||
@ -9,9 +9,10 @@ srcfile ← •file.At srcfileR
|
|||||||
resfile ← •file.At resfileR
|
resfile ← •file.At resfileR
|
||||||
tmpfile ← (•file.At buildDir) •file.At filename∾".d.orig"
|
tmpfile ← (•file.At buildDir) •file.At filename∾".d.orig"
|
||||||
depfile ← (•file.At buildDir) •file.At filename∾".d"
|
depfile ← (•file.At buildDir) •file.At filename∾".d"
|
||||||
|
prefix ← "si_"∾ (∧`'.'⊸≠)⊸/ •file.Name srcfile
|
||||||
|
|
||||||
# invoke singeli, which creates the result file
|
# invoke singeli, which creates the result file
|
||||||
(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ srcfile⟩) •Import siDir •file.At "singeli"
|
(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ "-n" ⋄ prefix ⋄ srcfile⟩) •Import siDir •file.At "singeli"
|
||||||
|
|
||||||
# output dependency file
|
# output dependency file
|
||||||
deps ← siDir⊸•file.At¨ •file.Lines tmpfile
|
deps ← siDir⊸•file.At¨ •file.Lines tmpfile
|
||||||
|
|||||||
2
makefile
2
makefile
@ -224,7 +224,7 @@ preSingeliBin:
|
|||||||
@${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c
|
@${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c
|
||||||
|
|
||||||
|
|
||||||
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c bits.c}
|
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c neq.c slash.c constrep.c bits.c}
|
||||||
@echo $(postmsg)
|
@echo $(postmsg)
|
||||||
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
||||||
@echo $< | cut -c 17- | sed 's/^/ /'
|
@echo $< | cut -c 17- | sed 's/^/ /'
|
||||||
|
|||||||
@ -61,7 +61,7 @@ B tbl_c2(Md1D* d, B w, B x) { B f = d->f;
|
|||||||
|
|
||||||
BBB2B fc2 = c2fn(f);
|
BBB2B fc2 = c2fn(f);
|
||||||
if (!EACH_FILLS && isFun(f) && isPervasiveDy(f) && TI(w,arrD1)) {
|
if (!EACH_FILLS && isFun(f) && isPervasiveDy(f) && TI(w,arrD1)) {
|
||||||
if (TI(x,arrD1) && xia<80 && wia>130) {
|
if (TI(x,arrD1) && wia>130 && xia<2560>>arrTypeBitsLog(TY(x))) {
|
||||||
Arr* wd = arr_shVec(TI(w,slice)(incG(w), 0, wia));
|
Arr* wd = arr_shVec(TI(w,slice)(incG(w), 0, wia));
|
||||||
r = fc2(f, slash_c2(f, m_i32(xia), taga(wd)), shape_c2(f, m_f64(ria), incG(x)));
|
r = fc2(f, slash_c2(f, m_i32(xia), taga(wd)), shape_c2(f, m_f64(ria), incG(x)));
|
||||||
} else if (xia>7) {
|
} else if (xia>7) {
|
||||||
@ -108,8 +108,17 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f;
|
|||||||
return homFil2(f, eachd(f, w, x), wf, xf);
|
return homFil2(f, eachd(f, w, x), wf, xf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if SINGELI && __PCLMUL__
|
||||||
|
#pragma GCC diagnostic push
|
||||||
|
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||||
|
#include "../singeli/gen/neq.c"
|
||||||
|
#pragma GCC diagnostic pop
|
||||||
|
#endif
|
||||||
B scan_ne(u64 p, u64* xp, u64 ia) {
|
B scan_ne(u64 p, u64* xp, u64 ia) {
|
||||||
u64* rp; B r=m_bitarrv(&rp,ia);
|
u64* rp; B r=m_bitarrv(&rp,ia);
|
||||||
|
#if SINGELI && __PCLMUL__
|
||||||
|
clmul_scan_ne(p, xp, rp, BIT_N(ia));
|
||||||
|
#else
|
||||||
for (usz i = 0; i < BIT_N(ia); i++) {
|
for (usz i = 0; i < BIT_N(ia); i++) {
|
||||||
u64 c = xp[i];
|
u64 c = xp[i];
|
||||||
u64 r = c ^ (c<<1);
|
u64 r = c ^ (c<<1);
|
||||||
@ -118,6 +127,7 @@ B scan_ne(u64 p, u64* xp, u64 ia) {
|
|||||||
rp[i] = r;
|
rp[i] = r;
|
||||||
p = -(r>>63); // repeat sign bit
|
p = -(r>>63); // repeat sign bit
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -105,6 +105,11 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if SINGELI
|
#if SINGELI
|
||||||
|
#pragma GCC diagnostic push
|
||||||
|
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||||
|
#include "../singeli/gen/constrep.c"
|
||||||
|
#pragma GCC diagnostic pop
|
||||||
|
|
||||||
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||||
extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||||
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||||
@ -713,6 +718,34 @@ B slash_c2(B t, B w, B x) {
|
|||||||
if (xl == 0) {
|
if (xl == 0) {
|
||||||
u64* xp = bitarr_ptr(x);
|
u64* xp = bitarr_ptr(x);
|
||||||
u64* rp; r = m_bitarrv(&rp, s);
|
u64* rp; r = m_bitarrv(&rp, s);
|
||||||
|
#if __BMI2__
|
||||||
|
if (wv <= 52) {
|
||||||
|
u64 m = (u64)-1 / (((u64)1<<wv)-1); // TODO table lookup
|
||||||
|
u64 xw = 0;
|
||||||
|
if (m & 1) { // Power of two
|
||||||
|
for (usz i=-1, j=0; j<BIT_N(s); j++) {
|
||||||
|
xw >>= (64/wv);
|
||||||
|
if ((j&(wv-1))==0) xw = xp[++i];
|
||||||
|
u64 rw = _pdep_u64(xw, m);
|
||||||
|
rp[j] = (rw<<wv)-rw;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
usz d = POPC(m); // == 64/wv
|
||||||
|
usz q = CTZ(m); // == 64%wv
|
||||||
|
m = m<<(wv-q) | 1;
|
||||||
|
u64 mt = (u64)1<<(d+1); // Bit d+1 may be needed, isn't pdep-ed
|
||||||
|
usz tsh = d*wv-(d+1);
|
||||||
|
for (usz xi=0, o=0, j=0; j<BIT_N(s); j++) {
|
||||||
|
xw = *(u64*)((u8*)xp+xi/8) >> (xi%8);
|
||||||
|
u64 ex = (xw&mt)<<tsh;
|
||||||
|
u64 rw = _pdep_u64(xw, m);
|
||||||
|
rp[j] = ((rw-ex)<<(wv-o))-(rw>>o|(xw&1));
|
||||||
|
o += q;
|
||||||
|
bool oo = o>=wv; xi+=d+oo; o-=wv&-oo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else
|
||||||
|
#endif
|
||||||
if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) }
|
if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) }
|
||||||
else { BOOL_REP_OVER(wv, xlen) }
|
else { BOOL_REP_OVER(wv, xlen) }
|
||||||
goto decX_ret;
|
goto decX_ret;
|
||||||
@ -720,7 +753,11 @@ B slash_c2(B t, B w, B x) {
|
|||||||
u8 xk = xl-3;
|
u8 xk = xl-3;
|
||||||
void* rv = m_tyarrv(&r, 1<<xk, s, xt);
|
void* rv = m_tyarrv(&r, 1<<xk, s, xt);
|
||||||
void* xv = tyany_ptr(x);
|
void* xv = tyany_ptr(x);
|
||||||
|
#if SINGELI
|
||||||
|
#define CASE(L,T) case L: constrep_##T(wv, xv, rv, xlen); break;
|
||||||
|
#else
|
||||||
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }
|
#define CASE(L,T) case L: { REP_BY_SCAN(T, wv) break; }
|
||||||
|
#endif
|
||||||
switch (xk) { default: UD; CASE(0,u8) CASE(1,u16) CASE(2,u32) CASE(3,u64) }
|
switch (xk) { default: UD; CASE(0,u8) CASE(1,u16) CASE(2,u32) CASE(3,u64) }
|
||||||
#undef CASE
|
#undef CASE
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
include 'skin/c'
|
include 'skin/c'
|
||||||
include 'arch/c'
|
include 'arch/c'
|
||||||
|
include 'util/kind'
|
||||||
|
|
||||||
oper ~~ reinterpret infix right 55
|
oper ~~ reinterpret infix right 55
|
||||||
def Size = u64
|
def Size = u64
|
||||||
@ -12,21 +13,20 @@ def cdiv{a,b} = (a+b-1)/b
|
|||||||
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
||||||
def lb{n==1} = 0
|
def lb{n==1} = 0
|
||||||
|
|
||||||
def tail{n,x} = x - (x>>n << n) # get the n least significant bits
|
def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
|
||||||
def bit {k,x} = tail{1,x>>k} << k # get the k-th bit
|
def bit {k,x} = x & (1<<k) # get the k-th bit
|
||||||
|
|
||||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||||
def assert{x & x==0} = assert{'failed assertion'}
|
def assert{x & x==0} = assert{'failed assertion'}
|
||||||
def assert{x & x==1} = 1
|
def assert{x & x==1} = 1
|
||||||
def assert{x:u1} = emit{void, 'si_assert', x}
|
def assert{x:u1} = { if (not x) emit{void, '__builtin_unreachable'} }
|
||||||
|
|
||||||
|
|
||||||
# various checks
|
# various checks
|
||||||
def knum{x} = match{kind{x},'number'}
|
def isreg = kreg
|
||||||
def isreg{x} = match{kind{x},'register'}
|
def isconst = kcon
|
||||||
def isconst{x} = match{kind{x},'constant'}
|
def istype = ktyp
|
||||||
def istype{x} = match{kind{x},'type'}
|
def istup = ktup
|
||||||
def istup{x} = match{kind{x},'tuple'}
|
|
||||||
|
|
||||||
def isunsigned{T} = isint{T} & ~issigned{T}
|
def isunsigned{T} = isint{T} & ~issigned{T}
|
||||||
|
|
||||||
@ -44,10 +44,8 @@ def anyInt{x & isreg{x}|isconst{x}} = isint{x}
|
|||||||
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
||||||
def trunc{T, x & knum{x}} = cast{T, x}
|
def trunc{T, x & knum{x}} = cast{T, x}
|
||||||
|
|
||||||
def ty_s{w== 8} = i8; def ty_u{w== 8} = u8
|
def ty_s{w} = primtype{'i', w}
|
||||||
def ty_s{w==16} = i16; def ty_u{w==16} = u16
|
def ty_u{w} = primtype{'u', w}
|
||||||
def ty_s{w==32} = i32; def ty_u{w==32} = u32
|
|
||||||
def ty_s{w==64} = i64; def ty_u{w==64} = u64
|
|
||||||
|
|
||||||
def ty_u{T & isprim{T}} = ty_u{width{T}}
|
def ty_u{T & isprim{T}} = ty_u{width{T}}
|
||||||
def ty_s{T & isprim{T}} = ty_s{width{T}}
|
def ty_s{T & isprim{T}} = ty_s{width{T}}
|
||||||
|
|||||||
185
src/singeli/src/constrep.singeli
Normal file
185
src/singeli/src/constrep.singeli
Normal file
@ -0,0 +1,185 @@
|
|||||||
|
include './base'
|
||||||
|
include './sse3'
|
||||||
|
include './avx'
|
||||||
|
include './avx2'
|
||||||
|
include './mask'
|
||||||
|
include 'util/tup'
|
||||||
|
def incl{a,b} = slice{iota{b+1},a}
|
||||||
|
|
||||||
|
# 1+˝∨`⌾⌽0=div|⌜range
|
||||||
|
def makefact{divisor, range} = {
|
||||||
|
def t = table{{a,b}=>0==b%a, divisor, range}
|
||||||
|
fold{+, 1, reverse{scan{|, reverse{t}}}}
|
||||||
|
}
|
||||||
|
def basic_rep = incl{2, 7}
|
||||||
|
def fact_size = 128
|
||||||
|
def fact_inds = slice{iota{fact_size},8}
|
||||||
|
def fact_tab = makefact{basic_rep, fact_inds}
|
||||||
|
factors:*u8 = fact_tab
|
||||||
|
|
||||||
|
def get_shufs{step, wv, len} = {
|
||||||
|
def i = iota{len*step}
|
||||||
|
split{step, (i - i%wv)/wv}
|
||||||
|
}
|
||||||
|
def get_shuf_data{wv, len} = get_shufs{32, wv, len}
|
||||||
|
def get_shuf_data{wv} = get_shuf_data{wv, wv}
|
||||||
|
def rep_iter_from_sh{sh} = {
|
||||||
|
def l = tuplen{sh}
|
||||||
|
def h = l >> 1
|
||||||
|
{x, gen} => {
|
||||||
|
def fs{v, s} = gen{sel{[16]i8, v, s}}
|
||||||
|
a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}}
|
||||||
|
if (l%2) fs{x, tupsel{h, sh}}
|
||||||
|
b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def get_rep_iter{V, wv==2}{x, gen} = {
|
||||||
|
def s = shuf{[4]u64, x, 4b3120}
|
||||||
|
each{{q}=>gen{V~~q}, unpackQ{s, s}}
|
||||||
|
}
|
||||||
|
def get_rep_iter{V==[4]u64, wv} = {
|
||||||
|
def step = 4
|
||||||
|
def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} }
|
||||||
|
def sh = each{base4, get_shufs{step, wv, wv}}
|
||||||
|
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
|
||||||
|
}
|
||||||
|
def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = {
|
||||||
|
def double{x} = {
|
||||||
|
s:=shuf{[4]u64, x, 4b3120}; s+=s
|
||||||
|
each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}}
|
||||||
|
}
|
||||||
|
def doubles{n,tup} = slice{join{each{double,tup}}, 0, n}
|
||||||
|
def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}}
|
||||||
|
def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up
|
||||||
|
def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} }
|
||||||
|
def ext{e} = {
|
||||||
|
def m = tlen{2*e}; def n = tlen{e} # m<n
|
||||||
|
if (elbytes<=e) set{slice{iota{n},m}}
|
||||||
|
else slice{sh,0,n} = doubles{n,slice{sh,0,m}}
|
||||||
|
}
|
||||||
|
set{iota{tlen{8}}}; ext{4}; ext{2}; ext{1}
|
||||||
|
sh
|
||||||
|
}
|
||||||
|
|
||||||
|
def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
|
||||||
|
def step = vcount{V}
|
||||||
|
nv := n / step
|
||||||
|
j:u64 = 0
|
||||||
|
def write{v} = { store{rv, j, v}; ++j }
|
||||||
|
@for (xv over nv) onreps{xv, write}
|
||||||
|
if (nv*step < n) {
|
||||||
|
nr := n * wv
|
||||||
|
e := nr / step
|
||||||
|
s := broadcast{V, 0}
|
||||||
|
def end = makelabel{}
|
||||||
|
onreps{load{xv,nv}, {v} => {
|
||||||
|
s = v
|
||||||
|
if (j == e) goto{end}
|
||||||
|
write{s}
|
||||||
|
}}
|
||||||
|
setlabel{end}
|
||||||
|
q := nr & (step-1)
|
||||||
|
if (q) maskstoreF{rv, maskOf{V, q}, e, s}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
||||||
|
|
||||||
|
def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially
|
||||||
|
rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}}
|
||||||
|
rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}}
|
||||||
|
rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||||
|
def V = [32]i8
|
||||||
|
def st = read_shuf_vecs{wv, elbytes, sh}
|
||||||
|
rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
|
||||||
|
}
|
||||||
|
|
||||||
|
def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
|
||||||
|
rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}}
|
||||||
|
rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
|
||||||
|
rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||||
|
def h = 4
|
||||||
|
def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}}
|
||||||
|
def V = [32]i8
|
||||||
|
def step = vcount{V} # Bytes written
|
||||||
|
def wvb = wv * elbytes
|
||||||
|
def hs = (h*step) / wvb # Actual step size in argument elements
|
||||||
|
re := r + n*wvb - h*step
|
||||||
|
i:u64 = 0
|
||||||
|
while (r <= re) {
|
||||||
|
a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010}
|
||||||
|
@unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}}
|
||||||
|
i += hs*elbytes
|
||||||
|
r += hs*wvb
|
||||||
|
}
|
||||||
|
re += (h-1)*step
|
||||||
|
a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010}
|
||||||
|
s := broadcast{V, 0}
|
||||||
|
def end = makelabel{}
|
||||||
|
@unroll (j to h) {
|
||||||
|
s = sel{[16]i8, a, tupsel{j,sh}}
|
||||||
|
if (r > re) goto{end}
|
||||||
|
store{*V~~r, 0, s}
|
||||||
|
r += step
|
||||||
|
}
|
||||||
|
setlabel{end}
|
||||||
|
q := (re+step) - r
|
||||||
|
if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s}
|
||||||
|
}
|
||||||
|
|
||||||
|
rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||||
|
if (wv > tupsel{-1,rcsh_vals}) {
|
||||||
|
return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)}
|
||||||
|
}
|
||||||
|
n *= elbytes
|
||||||
|
ri := wv - tupsel{0,rcsh_vals}
|
||||||
|
sh := *[32]i8~~rcsh_data + load{rcsh_offs,ri}
|
||||||
|
def try{k} = { if (wv==k) rcsh_sub{k}(elbytes, x, r, n, sh) }
|
||||||
|
each{try, rcsh_vals}
|
||||||
|
}
|
||||||
|
|
||||||
|
def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = {
|
||||||
|
assert{kv > 0}
|
||||||
|
def V = [256/width{T}]T
|
||||||
|
@for (x over n) {
|
||||||
|
v := broadcast{V, x}
|
||||||
|
@loop (j to kv) store{*V~~r, j, v}
|
||||||
|
r += wv
|
||||||
|
store{*V~~r, -1, v}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n}
|
||||||
|
rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n}
|
||||||
|
|
||||||
|
rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||||
|
assert{wv>=2}
|
||||||
|
if (wv>=8 and wv<=fact_size) {
|
||||||
|
fa := promote{u64, load{factors,wv-8}}
|
||||||
|
if (fa > 1) {
|
||||||
|
fi := wv / fa
|
||||||
|
def t = *void~~(*T~~r + (promote{u64,wv}-fi)*n)
|
||||||
|
rep_const{T}(fi,x,t,n)
|
||||||
|
rep_const{T}(fa,t,r,fi*n)
|
||||||
|
return{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def wT = width{T}
|
||||||
|
def vn = 256/wT
|
||||||
|
def V = [vn]T
|
||||||
|
def max_shuffle = 2*vn
|
||||||
|
if (wv <= max_shuffle) {
|
||||||
|
def specialize{k} = {
|
||||||
|
if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}}
|
||||||
|
}
|
||||||
|
specialize{2}
|
||||||
|
rep_const_shuffle_any(wv, wT/8, *i8~~x, *i8~~r, n)
|
||||||
|
} else {
|
||||||
|
kv := wv / vn
|
||||||
|
@unroll (k from (max_shuffle/vn) to 4) {
|
||||||
|
if (kv == k) return{rep_const_broadcast{T, k}(wv, *T~~x, *T~~r, n)}
|
||||||
|
}
|
||||||
|
rep_const_broadcast{T}(kv, wv, *T~~x, *T~~r, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
'constrep_u8' = rep_const{i8 }; 'constrep_u16' = rep_const{i16}
|
||||||
|
'constrep_u32' = rep_const{i32}; 'constrep_u64' = rep_const{u64}
|
||||||
34
src/singeli/src/neq.singeli
Normal file
34
src/singeli/src/neq.singeli
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
include './base'
|
||||||
|
include './sse3'
|
||||||
|
|
||||||
|
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
|
||||||
|
def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b}
|
||||||
|
|
||||||
|
def loadl{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si64', a+n}
|
||||||
|
def storel{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si64', a+n, v}
|
||||||
|
|
||||||
|
clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
|
||||||
|
def V = [2]u64
|
||||||
|
m := broadcast{V, mark}
|
||||||
|
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
|
||||||
|
p := clmul{a, m, i}
|
||||||
|
t := shr{[16]u8, p, 8}
|
||||||
|
s := p ^ carry
|
||||||
|
carry = s ^ t
|
||||||
|
s
|
||||||
|
}
|
||||||
|
xv := *V ~~ x
|
||||||
|
rv := *V ~~ r
|
||||||
|
e := words/2;
|
||||||
|
c := broadcast{V, init}
|
||||||
|
@for (rv, xv over e) {
|
||||||
|
rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})}
|
||||||
|
}
|
||||||
|
if (words & 1) {
|
||||||
|
storel{rv,e, clmul{loadl{xv,e}, m, 0} ^ c}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = {
|
||||||
|
clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1))
|
||||||
|
}
|
||||||
|
'clmul_scan_ne' = clmul_scan_ne_bit
|
||||||
@ -97,6 +97,9 @@ def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b}
|
|||||||
def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
|
def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
|
||||||
# no 64-bit arithmetic shift :/
|
# no 64-bit arithmetic shift :/
|
||||||
|
|
||||||
|
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
|
||||||
|
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
|
||||||
|
|
||||||
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
||||||
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user