merge dyarith.singeli and dyarith2.singeli

This commit is contained in:
dzaima 2022-08-29 19:16:35 +03:00
parent 777c5cc1a5
commit 78e5799147
4 changed files with 107 additions and 188 deletions

View File

@ -216,7 +216,7 @@ preSingeliBin:
@${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith2.c copy.c equal.c squeeze.c scan.c slash.c}
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c}
@echo $(postmsg)
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
@echo $< | cut -c 17- | sed 's/^/ /'
@ -224,9 +224,9 @@ src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
ifeq (${i_singeli}, 1)
# arithmetic singeli generator
src/builtins/arithd2.c: src/singeli/c/dyarith2.c
src/singeli/c/dyarith2.c: src/singeli/gen/arTables.c
src/singeli/src/dyarith2.singeli: src/singeli/gen/arDefs.singeli
src/builtins/arithd2.c: src/singeli/c/dyarith.c
src/singeli/c/dyarith.c: src/singeli/gen/arTables.c
src/singeli/src/dyarith.singeli: src/singeli/gen/arDefs.singeli
src/singeli/gen/arDefs.singeli: genArithTables
src/singeli/gen/arTables.c: genArithTables

View File

@ -165,7 +165,7 @@ NOINLINE B do_dyArith(DyTable* table, B w, B x) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#include "../gen/dyarith2.c"
#include "../gen/dyarith.c"
#pragma GCC diagnostic pop
static void rootAAu_f64_f64_f64(u8* r, u8* w, u8* x, u64 len) { for (u64 i = 0; i < len; i++) ((f64*)r)[i] = pow(((f64*)x)[i], 1.0/((f64*)w)[i]); }

View File

@ -8,38 +8,42 @@ include './bitops'
include './mask'
def fmt_op{X== __add}= '__add'
def fmt_op{X== __sub}= '__sub'
def fmt_op{X== __mul}= '__mul'
def fmt_op{X== __div}= '__div'
def fmt_op{X== __or}= '__or'
def fmt_op{X== __and}= '__and'
def fmt_op{X==bqn_or}= 'bqn_or'
def fmt_op{X== min}= 'min'
def fmt_op{X== max}= 'max'
def rootty{T & isprim{T}} = T
def rootty{T & isvec{T}} = eltype{T}
def ty_sc{O, R} = R # keep floats as-is
def ty_sc{O, R & issigned{O} & isunsigned{rootty{R}}} = ty_s{R}
def ty_sc{O, R & isunsigned{O} & issigned{rootty{R}}} = ty_u{R}
def bqn_or{a, b} = (a+b)-(a*b)
# old scalar code
# def ty_dbl{T & isprim{T}} = apply{tern{issigned{T}, ty_s, ty_u}, tup{width{T}*2}}
# def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}})
# def dcast_i{x:T} = promote{ty_dbl{T}, x}
# def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isprim{T}} = {
# r:= F{dcast_i{w}, dcast_i{x}}
# tup{r, r!=promote{type{r}, trunc{T, r}}}
# }
# + & -
def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}}
def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub}} = anyneg{M{(w^x) & (w^r)}}
def arithChk1{F, M, w:T, x:T, r:T & match{F,__add} & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub} & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M}
def arithChk1{F==__add, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M}
def arithChk2{F, M, w:T, x:T, i & issigned{rootty{T}}} = {
def arithChk2{F, M, w:T, x:T & (match{F,__add} | match{F,__sub})} = {
r:= F{w,x}
tup{r, arithChk1{F, M, w, x, r}}
}
# ×/∧
def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
rl:= __mul {w,x}
rh:= __mulhi{w,x}
tup{rl, anyne{rh, rl>>15, M}}
}
def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
# ×
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
def wp = unpackQ{w, T ~~ (broadcast{T,0}>w)}
def xp = unpackQ{x, T ~~ (broadcast{T,0}>x)}
def rp = each{__mul, wp, xp}
@ -50,7 +54,12 @@ def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}}
}
}
def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
rl:= __mul {w,x}
rh:= __mulhi{w,x}
tup{rl, anyne{rh, rl>>15, M}}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
max:= [8]f32 ~~ broadcast{[8]u32, 0x4efffffe}
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
f32mul:= cf32{w} * cf32{x}
@ -68,37 +77,36 @@ def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} =
}
# f64
def arithChk3{F, M, w:T, x:T, i} = {
def r2 = arithChk2{F, M, w, x, i}
if (rare{tupsel{1,r2}}) return{i}
tupsel{0,r2}
def runner{u, F} = {
def c = ~u
def run{F, OO, M, w, x} = { show{'todo', fmt_op{F}, c, w, x}; emit{void,'__builtin_abort'}; w }
def run{F, OO, M, w:T, x:T & c} = {
def r2 = arithChk2{F, M, w, x}
if (rare{tupsel{1,r2}}) OO{}
tupsel{0,r2}
}
def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation
def run{F, OO, M, w:VU, x:VS & isunsigned{eltype{VU}} & issigned{eltype{VS}}} = { # 'a'+3, 'a'-3
top:= broadcast{VU, 1<<(width{eltype{VU}}-1)}
top ^ VU~~run{F, OO, M, VS~~(w^top), x}
}
# def run{F==__add, OO, M, w:VS, x:VU & issigned{eltype{VS}} & isunsigned{eltype{VU}}} = run{F, OO, M, x, w} # 3+'a' → 'a'+3
run
}
def arithChk3{F, M, w:T, x:T, i & f64==rootty{T}} = F{w,x}
# old atom F array & array F atom
def arithAny{VT, F, W, X, r, len} = {
def bulk = vcount{VT}
maskedLoop{bulk, len, {i, M} => {
rv:= arithChk3{F, M, W{i}, X{i}, i*bulk}
storeBatch{r, i, rv, M}
}}
def run = runner{eltype{VT}==f64, F}
maskedLoop{bulk, len, {i, M} => storeBatch{r, i, run{F, {} => return{i}, M, W{i}, X{i}}, M}}
len
}
def arithAA{VT, F, w, x, r, len} = {
arithAny{VT, F, {i}=>load{*VT~~w, i}, {i}=>load{*VT~~x, i}, r, len}
}
def arithAS{VT, F, w, x, r, len} = {
xv:= broadcast{VT, x}
arithAny{VT, F, {i}=>load{*VT~~w, i}, {i}=>xv, r, len}
}
def arithSA{VT, F, w, x, r, len} = {
wv:= broadcast{VT, w}
arithAny{VT, F, {i}=>wv, {i}=>load{*VT~~x, i}, r, len}
}
# cast a guaranteed float to a more specific type; return{0} if not possible
def cast_fB{T, x:(u64) & f64==T} = from_B{f64, x}
def cast_fB{T, x:(u64) & issigned{T} & T<i64} = {
@ -107,8 +115,8 @@ def cast_fB{T, x:(u64) & issigned{T} & T<i64} = {
if (rare{f!=fext{r}}) return{cast{Size,0}}
r
}
arithAA{F,VT}(w: *u8, x: *u8, r: *u8, len: Size) : Size = { def c{x} = *eltype{VT} ~~ x; arithAA{VT, F, c{w}, c{x}, c{r}, len} }
def arithAS{VT, F, w, x, r, len} = { xv:= broadcast{VT, x}; arithAny{VT, F, {i}=>load{*VT~~w, i}, {i}=>xv, r, len} }
def arithSA{VT, F, w, x, r, len} = { wv:= broadcast{VT, w}; arithAny{VT, F, {i}=>wv, {i}=>load{*VT~~x, i}, r, len} }
arithAS{F,VT}(w: *u8, x: u64, r: *u8, len: Size) : Size = { def T=eltype{VT}; arithAS{VT, F, *T ~~ w, cast_fB{T, x}, *T~~r, len} }
arithSA{F,VT}(w: u64, x: *u8, r: *u8, len: Size) : Size = { def T=eltype{VT}; arithSA{VT, F, cast_fB{T, w}, *T ~~ x, *T~~r, len} }
@ -123,4 +131,49 @@ arithSA{F,VT}(w: u64, x: *u8, r: *u8, len: Size) : Size = { def T=eltype{VT}; ar
'avx2_mulAS_i8' = arithAS{__mul,[32]i8 }; 'avx2_mulSA_i8' = arithSA{__mul,[32]i8 }
'avx2_mulAS_i16' = arithAS{__mul,[16]i16}; 'avx2_mulSA_i16' = arithSA{__mul,[16]i16}
'avx2_mulAS_i32' = arithAS{__mul,[ 8]i32}; 'avx2_mulSA_i32' = arithSA{__mul,[ 8]i32}
'avx2_mulAS_f64' = arithAS{__mul,[ 4]f64}; 'avx2_mulSA_f64' = arithSA{__mul,[ 4]f64}
'avx2_mulAS_f64' = arithAS{__mul,[ 4]f64}; 'avx2_mulSA_f64' = arithSA{__mul,[ 4]f64}
# new array F atom
def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
# show{fmt_op{F}, mode, W, X, R}
if (R==u1) {
def bulk = vw/64;
def TY = [bulk]u64
maskedLoop{bulk, cdiv{len, 64}, {i, M} => {
cw:= loadBatch{*u64~~w, i, TY}
cx:= loadBatch{*u64~~x, i, TY}
storeBatch{*u64~~r, i, F{cw, cx}, M}
}}
} else {
def bulk = vw / max{max{width{W}, width{X}}, width{R}}
def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}}
def TY = [bulk]R
def run = runner{match{overflow, 0}, F}
maskedLoop{bulk, len, {i, M} => {
cw:= loadBatch{*W~~w, i, ty_sc{W, TY}}
cx:= loadBatch{*X~~x, i, ty_sc{X, TY}}
storeBatch{*R~~r, i, TY~~run{F, {} => overflow{i}, M, cw, cx}, M}
}}
}
}
arithAAc{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : u64 = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
if (mode==1) len
else 0
}
arithAAu{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : void = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
}
def arithAA{mode, F, W, X, R} = {
def vw = 256
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
else arithAAu{vw, mode, F, W, X, R}
}
include './../gen/arDefs'

View File

@ -1,134 +0,0 @@
# include './base'
# include './f64'
# include './cbqnDefs'
# include './sse3'
# include './avx'
# include './avx2'
# include './bitops'
# include './mask'
include './dyarith'
def rootty{T & isprim{T}} = T
def rootty{T & isvec{T}} = eltype{T}
def ty_sc{O, R} = R # keep floats as-is
def ty_sc{O, R & issigned{O} & isunsigned{rootty{R}}} = ty_s{R}
def ty_sc{O, R & isunsigned{O} & issigned{rootty{R}}} = ty_u{R}
def bqn_or{a, b} = (a+b)-(a*b)
def fmt_op{X== __add}= '__add'
def fmt_op{X== __sub}= '__sub'
def fmt_op{X== __mul}= '__mul'
def fmt_op{X== __div}= '__div'
def fmt_op{X== __or}= '__or'
def fmt_op{X== __and}= '__and'
def fmt_op{X==bqn_or}= 'bqn_or'
def fmt_op{X== min}= 'min'
def fmt_op{X== max}= 'max'
def arithChk1{F==__add, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M}
def arithChk2{F, M, w:T, x:T & (match{F,__add} | match{F,__sub})} = {
r:= F{w,x}
tup{r, arithChk1{F, M, w, x, r}}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
rl:= __mul {w,x}
rh:= __mulhi{w,x}
tup{rl, anyne{rh, rl>>15, M}}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = {
def wp = unpackQ{w, T ~~ (broadcast{T,0}>w)}
def xp = unpackQ{x, T ~~ (broadcast{T,0}>x)}
def rp = each{__mul, wp, xp}
def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp}
if (M{0}) { # masked check
tup{packQ{rp}, any{M{packQ{bad}}}}
} else { # unmasked check; can do check in a simpler way
tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}}
}
}
def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = {
max:= [8]f32 ~~ broadcast{[8]u32, 0x4efffffe}
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
f32mul:= cf32{w} * cf32{x}
tup{w*x, any{M{abs{f32mul} >= max}}}
# TODO fallback to the below if the above fails
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half
# def wp = unpackQ{w, broadcast{T, 0}}
# def xp = unpackQ{x, broadcast{T, 0}}
# def rp = each{__mul32, wp, xp}
# def T2 = to_el{i64, T}
# def bad = each{{v} => {
# (((T2~~v) + broadcast{T2,0x80000000}) ^ broadcast{T2, cast{i64,1}<<63}) > broadcast{T2, cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}}
# }, rp}
# tup{packQQ{each{{v} => v&broadcast{T2, 0xFFFFFFFF}, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
}
def runner{u, F} = {
def c = ~u
def run{F, OO, M, w, x} = { show{'todo', fmt_op{F}, c, w, x}; emit{void,'__builtin_abort'}; w }
def run{F, OO, M, w:T, x:T & c} = {
def r2 = arithChk2{F, M, w, x}
if (rare{tupsel{1,r2}}) OO{}
tupsel{0,r2}
}
def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation
def run{F, OO, M, w:VU, x:VS & isunsigned{eltype{VU}} & issigned{eltype{VS}}} = { # 'a'+3, 'a'-3
top:= broadcast{VU, 1<<(width{eltype{VU}}-1)}
top ^ VU~~run{F, OO, M, VS~~(w^top), x}
}
# def run{F==__add, OO, M, w:VS, x:VU & issigned{eltype{VS}} & isunsigned{eltype{VU}}} = run{F, OO, M, x, w} # 3+'a' → 'a'+3
run
}
def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
# show{fmt_op{F}, mode, W, X, R}
if (R==u1) {
def bulk = vw/64;
def TY = [bulk]u64
maskedLoop{bulk, cdiv{len, 64}, {i, M} => {
cw:= loadBatch{*u64~~w, i, TY}
cx:= loadBatch{*u64~~x, i, TY}
storeBatch{*u64~~r, i, F{cw, cx}, M}
}}
} else {
def bulk = vw / max{max{width{W}, width{X}}, width{R}}
def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}}
def TY = [bulk]R
def run = runner{match{overflow, 0}, F}
maskedLoop{bulk, len, {i, M} => {
cw:= loadBatch{*W~~w, i, ty_sc{W, TY}}
cx:= loadBatch{*X~~x, i, ty_sc{X, TY}}
storeBatch{*R~~r, i, TY~~run{F, {} => overflow{i}, M, cw, cx}, M}
}}
}
}
arithAAc{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : u64 = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
if (mode==1) len
else 0
}
arithAAu{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : void = {
arithAAimpl{vw, mode, F, W, X, R, w, x, r, len}
}
def arithAA{mode, F, W, X, R} = {
def vw = 256
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
else arithAAu{vw, mode, F, W, X, R}
}
include './../gen/arDefs'