diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 2bf1d6ae..da2419e4 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -11,6 +11,7 @@ config SLOW_PDEP = 0 def same = is oper ~~ reinterpret infix right 55 oper ** broadcast infix right 55 +def extend_each{G, ...args}{...fs} = each{{f, ...args} => (if (length{args}>0) G{...args} else G){f}, fs, ...args} def isreg = kreg def isconst = kcon @@ -184,6 +185,25 @@ def { def blend_bit{f:T, t:T, m:M if width{T}==width{M}} = T ~~ ((M~~t & m) | (M~~f &~ m)) def blend_hom{f:T, t:T, m:M} = blend_bit{f, t, m} +local def anyall{is_all} = { def extend _{me} = { + def me{...vs if length{vs}>1 and not same{try_same_type{vs,'!'},'!'}} = { + def {a,b} = split{length{vs}/2, vs} + me{...each{tern{is_all,&,|}, a, b}} + } + # def me{v:[k]E, vl if any_num{vl}} = { + # def {n,m:M} = hom_to_int_ext{v} + # if (is_all) { + # all:M = maxvalue{M} + # (m | (all << (vl*n))) == all + # } else { + # (m << (width{M} - vl*n)) != 0 + # } + # } + def me{...vs, v:[k]_, k} = me{...vs, v} +}} +extend (extend_each{anyall{1}}){all_hom, all_top} +extend (extend_each{anyall{0}}){any_hom, any_top} + def hom_to_int_ext{a:T} = tup{1, hom_to_int{a}} # tup{n,mask}; mask with each bit repeated n times def ctz_ext{{n,v}} = ctz{v}/n # ctz for a result of homMaskX def hom_to_int{...vs if length{vs}>1} = { @@ -199,8 +219,8 @@ def hom_to_int{...vs if length{vs}>1} = { def hom_to_int{x if ktup{x}} = hom_to_int{...x} def first_hom{x:[_]_} = ctz_ext{hom_to_int_ext{x}} def popc_hom{x:[_]_} = { - def {n,v} = hom_to_int_ext{x} - popc{v} / n + def {n,m} = hom_to_int_ext{x} + popc{m} / n } def try_first_hom{x:[_]_} = { def ex = hom_to_int_ext{x} @@ -219,7 +239,6 @@ def store{ptr:*E, val:V=[k]E, k} = store{*V~~ptr, 0, val} local def extend accept_vec_ptr{op} = { def op{ptr:*[k]E, m:M, val:[k]E} = op{*E~~ptr, m, val} } -def extend_each{G}{...fs} = each{G, fs} extend (extend_each{accept_vec_ptr}){ store_masked_hom,store_blended_hom, store_masked_top,store_blended_top, diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 9c952eb9..f9901158 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -103,7 +103,7 @@ def runner{u, R, F} = { } # any_hom, any_top already give masked vals; anyne doesn't, and and_some assumes no masking -def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} } +def runChecks_any{F, vals} = { F{...each{select{.,1}, vals}} } def runChecks{'any_hom', vals, M} = runChecks_any{any_hom, vals} def runChecks{'any_top', vals, M} = runChecks_any{any_top, vals} def runChecks{'none', vals, M} = 0 @@ -112,7 +112,7 @@ def runChecks{'anyne', vals, M} = { def i{vals} = { def {_,xs,ys} = flip{vals} assert{M{0} == 0} - ~all_hom{tree_fold{&, each{==, xs, ys}}} + ~all_hom{M, ...each{==, xs, ys}} } def i{{{_,x,y}}} = anyne{x, y, M} i{vals} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 3e5c0f10..5ff8abd2 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -15,7 +15,7 @@ def mask_of_first{T,n if w128{T}} = mask_of_impl{T, n, 128} def mask_of_first{T,n if w64{T}} = mask_of_impl{T, n, 64} def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y} -def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M{x!=y}} +def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M, x!=y} def anyne{x:T, y:T, M if M{0}==0 and any_int{x}} = x!=y def anyne{x:T, y:T, M if M{0}==1 and any_int{x}} = M{x^y} != 0 def anyne_bit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} @@ -63,9 +63,15 @@ def load_widen{ptr:*E0, i, [k]E1} = { def load_widen {ptr:*E, {...ns}, T } = each{load_widen {ptr, ., T }, ns} def store_narrow{ptr:*E, {...ns}, xs, M} = each{store_narrow{ptr, ., ., M}, ns, xs} -# TODO also similar any_hom & use those more -def all_hom{M, x:T if kgen{M}} = ~any_hom{M{~x}} # TODO better -def all_hom{(mask_none), ...xs} = all_hom{...xs} +local def anyall{is_all}{inv} = { def extend _{me} = { + def me{M if kgen{M}, ...vs} = me{...vs, ...(if (M{0}) tup{M{'count'}} else tup{})} + def me{v:[k]E, vl if any_num{vl} and not same{vl,k}} = { + if (is_all) ~inv{}{mask_first{vl}{~v}} + else me{mask_first{vl}{v}} + } +}} +extend (extend_each{anyall{1}, tup{{}=>any_hom, {}=>any_top}}){all_hom, all_top} +extend (extend_each{anyall{0}, tup{{}=>all_hom, {}=>all_top}}){any_hom, any_top} # "harmless" pointer cast that'll only cast void* local def cast_h{T,p} = assert{0, 'expected pointer with element',T,'or void but got ',p} diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 78a40f17..78ae793c 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { def bulk = arch_defvw/width{T} def VT = [bulk]T @for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { - if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk} + if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk} sr{each{abs, cx}} } len diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 33a26f49..55be386c 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -278,7 +278,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = { def cx0 = tup{load{*[bulk]X~~(x+i)}} def cx = if(wd) widenFull{WE,cx0} else cx0 if (~wd) { # within-vector overflow check; widening gives range space for this to not happen - if (rare{any_hom{tree_fold{|, each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}}) goto{'end'} + if (rare{any_hom{...each{{c:T} => absu{c} >= (ty_u{T}**maxFastE), cx}}}) goto{'end'} } def s0 = each{scan_plus, cx} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 745be650..a6c9e8a4 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -27,7 +27,7 @@ def search{E, x, n:(u64), OP} = { } @for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { eq:= each{OP, x} - if (any_hom{M{tree_fold{|, eq}}}) { + if (any_hom{M, ...eq}) { find_first{ {i,c} => any_hom{c}, {i,c} => tup{i,c}, @@ -54,7 +54,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { def bulk = arch_defvw/width{E} def VT = [bulk]E @for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { - if (any_hom{M{x!=x}}) return{1} + if (any_hom{M, x!=x}) return{1} sr{x + VT**0} } 0 @@ -325,7 +325,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { def minA = acc{2, VT**min1} def maxA = acc{2, VT**min1} @for_mu{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) { - if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} + if (E==f64 and any_hom{M, ...each{{c} => ~isI64{c}, cx}}) return{0} minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 967cf22d..d16a29d1 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -25,7 +25,7 @@ if_inline (hasarch{'AVX2'}) { def wrapChk{cw0:VI, xlf, M} = { cw:= cw0 + (xlf & VI~~(cw0= ty_u{xlf}}}) return{0} + if (any_hom{M, ty_u{cw} >= ty_u{xlf}}) return{0} cw } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 090da910..c5c0f57d 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -18,7 +18,7 @@ def any_nonC32{M, x:[_](u64)} = any_hom{M{~in_range_len{x, cbqn_c32Tag{}<<48, 1< def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { def H = re_el{u32, T} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} - any_top{M{T~~ne}} + any_top{M, T~~ne} } def any_nonC32{(mask_none), x:[k](u64), y:[k](u64)} = { def T32 = [k*2]u32 @@ -103,7 +103,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { @for_mu{bulk, 4}(xs in xb, M in 'm' over len) { minc:= tree_fold{min, xs} maxc:= tree_fold{max, xs} - if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff} + if (M{0}==0 and any_hom{M, (minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}) return{0xffff_ffff} acc{M, minc, maxc} } } else { @@ -132,7 +132,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { @for_mu{bulk, 2}(xs in xb, M in 'm' over is to len) { if (hasarch{'X86_64'}) { def ns = each{narrow{i32,.}, xs} - if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}} + if (any_hom{M, ...each{{ns,x} => widen{XV,ns}!=x, ns, xs}}) case_B{select{is, 0}} acc{M, ns} } else { assert{hasarch{'AARCH64'}} @@ -141,7 +141,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { def int32 = if (k==2) pack{...int, 0} else pack{...int, [bulk]i64**0, 0} def wd = each{{G} => cvt{f64,G{int32}}, slice{tup{widen{[bulk]i64,.}, widen_upper}, 0, k}} - if (~all_hom{M, tree_fold{&, each{==, wd, xs}}}) case_B{select{is, 0}} + if (~all_hom{M, ...each{==, wd, xs}}) case_B{select{is, 0}} acc{M, tup{int32}} } }