diff --git a/makefile b/makefile index e4f40c3e..ac53541b 100644 --- a/makefile +++ b/makefile @@ -198,7 +198,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o nescan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< @@ -224,7 +224,7 @@ preSingeliBin: @${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c constrep.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c neq.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/md1.c b/src/builtins/md1.c index be50fdc9..60992716 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -108,8 +108,14 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; return homFil2(f, eachd(f, w, x), wf, xf); } +#if SINGELI +extern void (*const clmul_scan_ne)(uint64_t v0,uint64_t* v1,uint64_t* v2,uint64_t v3); +#endif B scan_ne(u64 p, u64* xp, u64 ia) { u64* rp; B r=m_bitarrv(&rp,ia); +#if SINGELI + clmul_scan_ne(p, xp, rp, BIT_N(ia)); +#else for (usz i = 0; i < BIT_N(ia); i++) { u64 c = xp[i]; u64 r = c ^ (c<<1); @@ -118,6 +124,7 @@ B scan_ne(u64 p, u64* xp, u64 ia) { rp[i] = r; p = -(r>>63); // repeat sign bit } +#endif return r; } diff --git a/src/builtins/nescan.c b/src/builtins/nescan.c new file mode 100644 index 00000000..b0d60751 --- /dev/null +++ b/src/builtins/nescan.c @@ -0,0 +1,6 @@ +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/neq.c" + #pragma GCC diagnostic pop +#endif diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli new file mode 100644 index 00000000..86c60f18 --- /dev/null +++ b/src/singeli/src/neq.singeli @@ -0,0 +1,34 @@ +include './base' +include './sse3' + +def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} +def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b} + +def loadl{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si64', a+n} +def storel{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si64', a+n, v} + +clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { + def V = [2]u64 + m := broadcast{V, mark} + def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total + p := clmul{a, m, i} + t := shr{[16]u8, p, 8} + s := p ^ carry + carry = s ^ t + s + } + xv := *V ~~ x + rv := *V ~~ r + e := words/2; + c := broadcast{V, init} + @for (rv, xv over e) { + rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})} + } + if (words & 1) { + storel{rv,e, clmul{loadl{xv,e}, m, 0} ^ c} + } +} +clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = { + clmul_scan_ne_any(x, r, init, ia, -(u64~~1)) +} +'clmul_scan_ne' = clmul_scan_ne_bit diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 0b4182b2..39d95049 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -97,6 +97,9 @@ def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b} def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} # no 64-bit arithmetic shift :/ +def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} +def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} + def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b} @@ -174,4 +177,4 @@ def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b} # SSE4.2 -def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b} \ No newline at end of file +def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}