diff --git a/makefile b/makefile index c0861829..7c94e89f 100644 --- a/makefile +++ b/makefile @@ -216,7 +216,7 @@ preSingeliBin: @${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith2.c copy.c equal.c squeeze.c scan.c slash.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' @@ -224,9 +224,9 @@ src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin ifeq (${i_singeli}, 1) # arithmetic singeli generator -src/builtins/arithd2.c: src/singeli/c/dyarith2.c -src/singeli/c/dyarith2.c: src/singeli/gen/arTables.c -src/singeli/src/dyarith2.singeli: src/singeli/gen/arDefs.singeli +src/builtins/arithd2.c: src/singeli/c/dyarith.c +src/singeli/c/dyarith.c: src/singeli/gen/arTables.c +src/singeli/src/dyarith.singeli: src/singeli/gen/arDefs.singeli src/singeli/gen/arDefs.singeli: genArithTables src/singeli/gen/arTables.c: genArithTables diff --git a/src/singeli/c/arithdDispatch.c b/src/singeli/c/arithdDispatch.c index aae5d882..a1a9430e 100644 --- a/src/singeli/c/arithdDispatch.c +++ b/src/singeli/c/arithdDispatch.c @@ -165,7 +165,7 @@ NOINLINE B do_dyArith(DyTable* table, B w, B x) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" -#include "../gen/dyarith2.c" +#include "../gen/dyarith.c" #pragma GCC diagnostic pop static void rootAAu_f64_f64_f64(u8* r, u8* w, u8* x, u64 len) { for (u64 i = 0; i < len; i++) ((f64*)r)[i] = pow(((f64*)x)[i], 1.0/((f64*)w)[i]); } diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 4b115114..8ad7f284 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -8,38 +8,42 @@ include './bitops' include './mask' + +def fmt_op{X== __add}= '__add' +def fmt_op{X== __sub}= '__sub' +def fmt_op{X== __mul}= '__mul' +def fmt_op{X== __div}= '__div' +def fmt_op{X== __or}= '__or' +def fmt_op{X== __and}= '__and' +def fmt_op{X==bqn_or}= 'bqn_or' +def fmt_op{X== min}= 'min' +def fmt_op{X== max}= 'max' + def rootty{T & isprim{T}} = T def rootty{T & isvec{T}} = eltype{T} +def ty_sc{O, R} = R # keep floats as-is +def ty_sc{O, R & issigned{O} & isunsigned{rootty{R}}} = ty_s{R} +def ty_sc{O, R & isunsigned{O} & issigned{rootty{R}}} = ty_u{R} + +def bqn_or{a, b} = (a+b)-(a*b) -# old scalar code -# def ty_dbl{T & isprim{T}} = apply{tern{issigned{T}, ty_s, ty_u}, tup{width{T}*2}} -# def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}}) -# def dcast_i{x:T} = promote{ty_dbl{T}, x} -# def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isprim{T}} = { -# r:= F{dcast_i{w}, dcast_i{x}} -# tup{r, r!=promote{type{r}, trunc{T, r}}} -# } # + & - -def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}} -def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub}} = anyneg{M{(w^x) & (w^r)}} -def arithChk1{F, M, w:T, x:T, r:T & match{F,__add} & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M} -def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub} & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M} +def arithChk1{F==__add, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^r) & (x^r)}} +def arithChk1{F==__sub, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^x) & (w^r)}} +def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M} +def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M} -def arithChk2{F, M, w:T, x:T, i & issigned{rootty{T}}} = { + + +def arithChk2{F, M, w:T, x:T & (match{F,__add} | match{F,__sub})} = { r:= F{w,x} tup{r, arithChk1{F, M, w, x, r}} } -# ×/∧ - -def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i16==eltype{T}} = { - rl:= __mul {w,x} - rh:= __mulhi{w,x} - tup{rl, anyne{rh, rl>>15, M}} -} -def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i8==eltype{T}} = { +# × +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = { def wp = unpackQ{w, T ~~ (broadcast{T,0}>w)} def xp = unpackQ{x, T ~~ (broadcast{T,0}>x)} def rp = each{__mul, wp, xp} @@ -50,7 +54,12 @@ def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i8==eltype{T}} = { tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}} } } -def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} = { +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = { + rl:= __mul {w,x} + rh:= __mulhi{w,x} + tup{rl, anyne{rh, rl>>15, M}} +} +def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { max:= [8]f32 ~~ broadcast{[8]u32, 0x4efffffe} def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} f32mul:= cf32{w} * cf32{x} @@ -68,37 +77,36 @@ def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} = } -# f64 -def arithChk3{F, M, w:T, x:T, i} = { - def r2 = arithChk2{F, M, w, x, i} - if (rare{tupsel{1,r2}}) return{i} - tupsel{0,r2} + +def runner{u, F} = { + def c = ~u + + def run{F, OO, M, w, x} = { show{'todo', fmt_op{F}, c, w, x}; emit{void,'__builtin_abort'}; w } + + def run{F, OO, M, w:T, x:T & c} = { + def r2 = arithChk2{F, M, w, x} + if (rare{tupsel{1,r2}}) OO{} + tupsel{0,r2} + } + + def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation + + def run{F, OO, M, w:VU, x:VS & isunsigned{eltype{VU}} & issigned{eltype{VS}}} = { # 'a'+3, 'a'-3 + top:= broadcast{VU, 1<<(width{eltype{VU}}-1)} + top ^ VU~~run{F, OO, M, VS~~(w^top), x} + } + # def run{F==__add, OO, M, w:VS, x:VU & issigned{eltype{VS}} & isunsigned{eltype{VU}}} = run{F, OO, M, x, w} # 3+'a' → 'a'+3 + run } -def arithChk3{F, M, w:T, x:T, i & f64==rootty{T}} = F{w,x} +# old atom F array & array F atom def arithAny{VT, F, W, X, r, len} = { def bulk = vcount{VT} - maskedLoop{bulk, len, {i, M} => { - rv:= arithChk3{F, M, W{i}, X{i}, i*bulk} - storeBatch{r, i, rv, M} - }} + def run = runner{eltype{VT}==f64, F} + maskedLoop{bulk, len, {i, M} => storeBatch{r, i, run{F, {} => return{i}, M, W{i}, X{i}}, M}} len } - -def arithAA{VT, F, w, x, r, len} = { - arithAny{VT, F, {i}=>load{*VT~~w, i}, {i}=>load{*VT~~x, i}, r, len} -} -def arithAS{VT, F, w, x, r, len} = { - xv:= broadcast{VT, x} - arithAny{VT, F, {i}=>load{*VT~~w, i}, {i}=>xv, r, len} -} -def arithSA{VT, F, w, x, r, len} = { - wv:= broadcast{VT, w} - arithAny{VT, F, {i}=>wv, {i}=>load{*VT~~x, i}, r, len} -} - - # cast a guaranteed float to a more specific type; return{0} if not possible def cast_fB{T, x:(u64) & f64==T} = from_B{f64, x} def cast_fB{T, x:(u64) & issigned{T} & Tload{*VT~~w, i}, {i}=>xv, r, len} } +def arithSA{VT, F, w, x, r, len} = { wv:= broadcast{VT, w}; arithAny{VT, F, {i}=>wv, {i}=>load{*VT~~x, i}, r, len} } arithAS{F,VT}(w: *u8, x: u64, r: *u8, len: Size) : Size = { def T=eltype{VT}; arithAS{VT, F, *T ~~ w, cast_fB{T, x}, *T~~r, len} } arithSA{F,VT}(w: u64, x: *u8, r: *u8, len: Size) : Size = { def T=eltype{VT}; arithSA{VT, F, cast_fB{T, w}, *T ~~ x, *T~~r, len} } @@ -123,4 +131,49 @@ arithSA{F,VT}(w: u64, x: *u8, r: *u8, len: Size) : Size = { def T=eltype{VT}; ar 'avx2_mulAS_i8' = arithAS{__mul,[32]i8 }; 'avx2_mulSA_i8' = arithSA{__mul,[32]i8 } 'avx2_mulAS_i16' = arithAS{__mul,[16]i16}; 'avx2_mulSA_i16' = arithSA{__mul,[16]i16} 'avx2_mulAS_i32' = arithAS{__mul,[ 8]i32}; 'avx2_mulSA_i32' = arithSA{__mul,[ 8]i32} -'avx2_mulAS_f64' = arithAS{__mul,[ 4]f64}; 'avx2_mulSA_f64' = arithSA{__mul,[ 4]f64} \ No newline at end of file +'avx2_mulAS_f64' = arithAS{__mul,[ 4]f64}; 'avx2_mulSA_f64' = arithSA{__mul,[ 4]f64} + + + +# new array F atom +def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = { + # show{fmt_op{F}, mode, W, X, R} + if (R==u1) { + def bulk = vw/64; + def TY = [bulk]u64 + maskedLoop{bulk, cdiv{len, 64}, {i, M} => { + cw:= loadBatch{*u64~~w, i, TY} + cx:= loadBatch{*u64~~x, i, TY} + storeBatch{*u64~~r, i, F{cw, cx}, M} + }} + } else { + def bulk = vw / max{max{width{W}, width{X}}, width{R}} + def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}} + def TY = [bulk]R + + def run = runner{match{overflow, 0}, F} + + maskedLoop{bulk, len, {i, M} => { + cw:= loadBatch{*W~~w, i, ty_sc{W, TY}} + cx:= loadBatch{*X~~x, i, ty_sc{X, TY}} + storeBatch{*R~~r, i, TY~~run{F, {} => overflow{i}, M, cw, cx}, M} + }} + } +} + +arithAAc{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : u64 = { + arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} + if (mode==1) len + else 0 +} +arithAAu{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : void = { + arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} +} + +def arithAA{mode, F, W, X, R} = { + def vw = 256 + if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R} + else arithAAu{vw, mode, F, W, X, R} +} + +include './../gen/arDefs' \ No newline at end of file diff --git a/src/singeli/src/dyarith2.singeli b/src/singeli/src/dyarith2.singeli deleted file mode 100644 index 87944810..00000000 --- a/src/singeli/src/dyarith2.singeli +++ /dev/null @@ -1,134 +0,0 @@ -# include './base' -# include './f64' -# include './cbqnDefs' -# include './sse3' -# include './avx' -# include './avx2' -# include './bitops' -# include './mask' -include './dyarith' - - -def rootty{T & isprim{T}} = T -def rootty{T & isvec{T}} = eltype{T} - -def ty_sc{O, R} = R # keep floats as-is -def ty_sc{O, R & issigned{O} & isunsigned{rootty{R}}} = ty_s{R} -def ty_sc{O, R & isunsigned{O} & issigned{rootty{R}}} = ty_u{R} - -def bqn_or{a, b} = (a+b)-(a*b) - -def fmt_op{X== __add}= '__add' -def fmt_op{X== __sub}= '__sub' -def fmt_op{X== __mul}= '__mul' -def fmt_op{X== __div}= '__div' -def fmt_op{X== __or}= '__or' -def fmt_op{X== __and}= '__and' -def fmt_op{X==bqn_or}= 'bqn_or' -def fmt_op{X== min}= 'min' -def fmt_op{X== max}= 'max' - -def arithChk1{F==__add, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^r) & (x^r)}} -def arithChk1{F==__sub, M, w:T, x:T, r:T & issigned{rootty{T}}} = anyneg{M{(w^x) & (w^r)}} -def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__adds{w,x}, r, M} -def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & width{eltype{T}}<=16} = anyne{__subs{w,x}, r, M} - -def arithChk2{F, M, w:T, x:T & (match{F,__add} | match{F,__sub})} = { - r:= F{w,x} - tup{r, arithChk1{F, M, w, x, r}} -} -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i16==eltype{T}} = { - rl:= __mul {w,x} - rh:= __mulhi{w,x} - tup{rl, anyne{rh, rl>>15, M}} -} -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i8==eltype{T}} = { - def wp = unpackQ{w, T ~~ (broadcast{T,0}>w)} - def xp = unpackQ{x, T ~~ (broadcast{T,0}>x)} - def rp = each{__mul, wp, xp} - def bad = each{{v} => [16]i16 ~~ ((v<<8)>>8 != v), rp} - if (M{0}) { # masked check - tup{packQ{rp}, any{M{packQ{bad}}}} - } else { # unmasked check; can do check in a simpler way - tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}} - } -} -def arithChk2{F, M, w:T, x:T & match{F,__mul} & isvec{T} & i32==eltype{T}} = { - max:= [8]f32 ~~ broadcast{[8]u32, 0x4efffffe} - def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} - f32mul:= cf32{w} * cf32{x} - tup{w*x, any{M{abs{f32mul} >= max}}} - # TODO fallback to the below if the above fails - # TODO don't do this, but instead shuffle one half, do math, unshuffle that half - # def wp = unpackQ{w, broadcast{T, 0}} - # def xp = unpackQ{x, broadcast{T, 0}} - # def rp = each{__mul32, wp, xp} - # def T2 = to_el{i64, T} - # def bad = each{{v} => { - # (((T2~~v) + broadcast{T2,0x80000000}) ^ broadcast{T2, cast{i64,1}<<63}) > broadcast{T2, cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}} - # }, rp} - # tup{packQQ{each{{v} => v&broadcast{T2, 0xFFFFFFFF}, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M -} - -def runner{u, F} = { - def c = ~u - - def run{F, OO, M, w, x} = { show{'todo', fmt_op{F}, c, w, x}; emit{void,'__builtin_abort'}; w } - - def run{F, OO, M, w:T, x:T & c} = { - def r2 = arithChk2{F, M, w, x} - if (rare{tupsel{1,r2}}) OO{} - tupsel{0,r2} - } - - def run{F, OO, M, w, x & u} = F{w, x} # trivial base implementation - - def run{F, OO, M, w:VU, x:VS & isunsigned{eltype{VU}} & issigned{eltype{VS}}} = { # 'a'+3, 'a'-3 - top:= broadcast{VU, 1<<(width{eltype{VU}}-1)} - top ^ VU~~run{F, OO, M, VS~~(w^top), x} - } - # def run{F==__add, OO, M, w:VS, x:VU & issigned{eltype{VS}} & isunsigned{eltype{VU}}} = run{F, OO, M, x, w} # 3+'a' → 'a'+3 - run -} - -def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = { - # show{fmt_op{F}, mode, W, X, R} - if (R==u1) { - def bulk = vw/64; - def TY = [bulk]u64 - maskedLoop{bulk, cdiv{len, 64}, {i, M} => { - cw:= loadBatch{*u64~~w, i, TY} - cx:= loadBatch{*u64~~x, i, TY} - storeBatch{*u64~~r, i, F{cw, cx}, M} - }} - } else { - def bulk = vw / max{max{width{W}, width{X}}, width{R}} - def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}} - def TY = [bulk]R - - def run = runner{match{overflow, 0}, F} - - maskedLoop{bulk, len, {i, M} => { - cw:= loadBatch{*W~~w, i, ty_sc{W, TY}} - cx:= loadBatch{*X~~x, i, ty_sc{X, TY}} - storeBatch{*R~~r, i, TY~~run{F, {} => overflow{i}, M, cw, cx}, M} - }} - } -} - -arithAAc{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : u64 = { - arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} - if (mode==1) len - else 0 -} -arithAAu{vw, mode, F, W, X, R}(r:*u8, w:*u8, x:*u8, len:u64) : void = { - arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} -} - -def arithAA{mode, F, W, X, R} = { - def vw = 256 - if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R} - else arithAAu{vw, mode, F, W, X, R} -} - -include './../gen/arDefs' \ No newline at end of file