use __min & __max directly instead of custom min/max
might make sense to bring min/max back later as aliases, but the current situation of min!=__min was stupid
This commit is contained in:
parent
8beff254e2
commit
ee1668c7ea
@ -24,7 +24,7 @@ def mu_extra{...args} = {
|
||||
def get_all{prop} = each{{c}=>c{prop}, args}
|
||||
tup{
|
||||
{} => get_all{'from_unr'},
|
||||
fold{min, get_all{'flush_min'}},
|
||||
fold{__min, get_all{'flush_min'}},
|
||||
{} => get_all{'flush'},
|
||||
}
|
||||
}
|
||||
@ -128,7 +128,7 @@ def sum_accumulator{DE, unr, VT=[_]SE if int_els{DE,SE} and DE>SE and hasarch{'A
|
||||
1/0
|
||||
} else {
|
||||
def p{G} = __floor{G{ME} / G{SE} / 2} # divided by 2 because addpwa adds two elements per iter
|
||||
if (issigned{SE}) min{p{maxvalue}, p{minvalue}} else p{maxvalue}
|
||||
if (issigned{SE}) __min{p{maxvalue}, p{minvalue}} else p{maxvalue}
|
||||
}
|
||||
def me{'flush'} = if (not exact) {
|
||||
into_scal{acc_tup}
|
||||
@ -210,7 +210,7 @@ def sum_accumulator{DE==i64, unr, VT=[k]SE==i32 if hasarch{'X86_64'}} = { # TODO
|
||||
extend finish{me}
|
||||
}
|
||||
|
||||
def assoc_accumulator{F if is{F,min} or is{F,max}, unr if knum{unr}, VT=([16]i8), ident if hasarch{'X86_64'} and not hasarch{'SSE4.1'}} = {
|
||||
def assoc_accumulator{F if is{F,__min} or is{F,__max}, unr if knum{unr}, VT=([16]i8), ident if hasarch{'X86_64'} and not hasarch{'SSE4.1'}} = {
|
||||
def adda{M, a, c:([16]i8)} = mask_ident{F,ident}{M, a, ty_u{c} ^ [16]u8**128}
|
||||
def {acc, ..._} = acc_impl{adda, F, unr, [16]u8, u8, (ident%256) ^ 128}
|
||||
def me{...} = acc
|
||||
@ -271,8 +271,8 @@ def count_accumulator{DE, unr, VT=[k]SE if isunsigned{DE} and int_els{DE,SE}} =
|
||||
# implicit identity values
|
||||
local def of_e{[_]E, G} = G{E}
|
||||
local def of_e{E if isprim{E}, G} = G{E}
|
||||
def assoc_accumulator{F==min, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) E~~1/0 else maxvalue{E}}}
|
||||
def assoc_accumulator{F==max, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) -E~~1/0 else minvalue{E}}}
|
||||
def assoc_accumulator{F==__min, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) E~~1/0 else maxvalue{E}}}
|
||||
def assoc_accumulator{F==__max, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) -E~~1/0 else minvalue{E}}}
|
||||
def assoc_accumulator{F==__add, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, 0}
|
||||
|
||||
def any_accumulator{unr, T} = bool_accumulator{|, unr, T, 0}
|
||||
|
||||
@ -12,7 +12,7 @@ local {
|
||||
}
|
||||
|
||||
local def re_mask{[l]_, sub} = {
|
||||
def w = max{32,l}
|
||||
def w = __max{32,l}
|
||||
sub{fmtnat{l}, fmtnat{w}, ty_u{w}}
|
||||
}
|
||||
def reinterpret{M=[l==width{T}](u1), a:T} = {
|
||||
|
||||
@ -52,7 +52,7 @@ def tree_fold{F, x} = {
|
||||
def tree_fold{F, {x}} = x
|
||||
|
||||
def eachx{F, ...args} = {
|
||||
def l = tree_fold{max, each{{x} => if(istup{x}) length{x} else 0, args}}
|
||||
def l = tree_fold{__max, each{{x} => if(istup{x}) length{x} else 0, args}}
|
||||
each{F, ...each{{x} => if (istup{x}) x else l**x, args}}
|
||||
}
|
||||
|
||||
@ -214,7 +214,7 @@ def ctz_ext{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||
def hom_to_int{...vs if length{vs}>1} = {
|
||||
def n = length{vs}
|
||||
def [k]_ = one_type{vs}
|
||||
def RT = ty_u{max{8,k*n}}
|
||||
def RT = ty_u{__max{8,k*n}}
|
||||
def sl{...a} = promote{RT, hom_to_int{...slice{vs,...a}}}
|
||||
def h = n/2
|
||||
def lo = sl{0,h}
|
||||
@ -301,16 +301,14 @@ def vshl = vec_merge_shift_right
|
||||
def floor = __floor
|
||||
def ceil = __ceil
|
||||
def abs = __abs
|
||||
def min = __min
|
||||
def max = __max
|
||||
def adds = __adds
|
||||
def subs = __subs
|
||||
|
||||
|
||||
|
||||
# more arith
|
||||
def min{a, b if any_num{a} and any_num{b}} = tern{a<b, a, b}
|
||||
def max{a, b if any_num{a} and any_num{b}} = tern{a>b, a, b}
|
||||
def __min{a, b if any_num{a} and any_num{b}} = tern{a<b, a, b}
|
||||
def __max{a, b if any_num{a} and any_num{b}} = tern{a>b, a, b}
|
||||
def cdiv{a,b} = (a+b-1)/b # ceiling divide
|
||||
def cdiv{a,b if knum{a} and knum{b}} = ceil{a/b}
|
||||
def popc{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_popcountll', x}
|
||||
@ -417,7 +415,7 @@ def for_blocks{blsz}{vars,begin,end,iter} = {
|
||||
iter{tup{i0,i1}, vars}
|
||||
if (likely{i1 == end}) goto{el}
|
||||
i0 = i1
|
||||
i1 = min{i1+blsz, ux~~end}
|
||||
i1 = __min{i1+blsz, ux~~end}
|
||||
}
|
||||
setlabel{el}
|
||||
}
|
||||
|
||||
@ -27,7 +27,7 @@ def dup_half{vec,half} = shuf{[4]u64, vec, 2*half+tup{0,1,0,1}}
|
||||
include './scan_common'
|
||||
fn max_scan{T, up}(x:*T, len:u64) : void = {
|
||||
if (hasarch{'X86_64'}) {
|
||||
def op = max
|
||||
def op = __max
|
||||
def pre = make_scan_idem{T, op, up}
|
||||
def vl = (if (hasarch{'AVX2'}) 256 else 128)/width{T}
|
||||
def V = [vl]T
|
||||
@ -250,7 +250,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} =
|
||||
bits := each{{j} => U**(lowbits << j), iota{lstep}}
|
||||
# Unroll sizes up to a full lane, handling extra lanes conditionally
|
||||
# in the largest one
|
||||
@unroll (klog from 2 to min{maxstep,lstep}+1) {
|
||||
@unroll (klog from 2 to __min{maxstep,lstep}+1) {
|
||||
def last = klog==lstep
|
||||
def this = if (not last) log==klog else log>=klog
|
||||
if (this) @for_vec_overlap{vl} (j to xn) {
|
||||
|
||||
@ -121,8 +121,8 @@ def table{aa, F, w, G_bit, G_vec} = {
|
||||
...each{{E}=>{
|
||||
def E2 = (if (aa and isunsigned{E} and (eqne{F} or E==u32)) {ty_s{E}} else E)
|
||||
def bulk = w/width{E2}
|
||||
def unr = if (hasarch{'X86_64'} and width{E2}==16) 2 else max{8/bulk,1}
|
||||
G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, unr}}
|
||||
def unr = if (hasarch{'X86_64'} and width{E2}==16) 2 else __max{8/bulk,1}
|
||||
G_vec{[bulk]E2, __max{tern{hasarch{'AARCH64'},2,1}, unr}}
|
||||
}, tup{i8, i16, i32, f64, u8, u16, u32}}
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,7 +17,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
||||
def vw = arch_defvw
|
||||
assert{l!=0}
|
||||
|
||||
def bulk = vw/max{width{X}, width{R}}
|
||||
def bulk = vw/__max{width{X}, width{R}}
|
||||
xp:= *tern{X==u1, u64, X} ~~ x
|
||||
rp:= *tern{R==u1, u64, R} ~~ r
|
||||
def XV = [bulk]X
|
||||
@ -45,7 +45,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
||||
def XU = ty_u{XV}
|
||||
def op{x} = (XU~~x) == XU~~XV**1
|
||||
|
||||
def unr = if (hasarch{'X86_64'} and width{X}==16) 2 else max{8/bulk, 1}
|
||||
def unr = if (hasarch{'X86_64'} and width{X}==16) 2 else __max{8/bulk, 1}
|
||||
def bulk2 = bulk*unr
|
||||
xi:ux = 0
|
||||
@for_nz (i to cdiv{l,bulk2}) {
|
||||
|
||||
@ -44,12 +44,12 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
|
||||
}
|
||||
# Now start analysis
|
||||
@for (xv, xp in *V~~(x-1) over _ from a to b) {
|
||||
jv = min{jv, xv}; mv = max{mv, xv}
|
||||
jv = __min{jv, xv}; mv = __max{mv, xv}
|
||||
dc -= xp != xv
|
||||
}
|
||||
@for (x over _ from rv to r) { if (x<min_allowed) return{x}; if (x>mx) mx=x }
|
||||
jt := vfold{min, jv}
|
||||
mt := vfold{max, mv}
|
||||
jt := vfold{__min, jv}
|
||||
mt := vfold{__max, mv}
|
||||
if (jt < min_allowed) return{jt}
|
||||
if (mt > mx) mx = mt
|
||||
|
||||
@ -196,7 +196,7 @@ fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = {
|
||||
d:usz = block
|
||||
d2 := undefined{usz}
|
||||
while ((d2=d+d) <= rem and bxi{d2}) d = d2
|
||||
l := min{(rem &~ (block-1)) - d, d}
|
||||
l := __min{(rem &~ (block-1)) - d, d}
|
||||
# Target is in [d,d+l); shrink l
|
||||
while (l > block) {
|
||||
h := (l/2) &~ (block-1)
|
||||
@ -209,7 +209,7 @@ fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = {
|
||||
i += d; xo += d; xi = load{xo}
|
||||
}
|
||||
# Count the next block normally
|
||||
rem = min{rem, usz~~block} # TODO get rid of the need of the usz~~ here
|
||||
rem = __min{rem, usz~~block} # TODO get rid of the need of the usz~~ here
|
||||
count_adapt_runs{xo, r, rem}
|
||||
rxi := load{r, xi}
|
||||
if (rxi >= block) {
|
||||
|
||||
@ -142,7 +142,7 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
|
||||
sr{each{&, cw, cx}}
|
||||
}
|
||||
} else {
|
||||
def bulk = vw / max{max{width{W}, width{X}}, width{R}}
|
||||
def bulk = vw / __max{__max{width{W}, width{X}}, width{R}}
|
||||
def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}}
|
||||
def TY = [bulk]R
|
||||
|
||||
@ -174,7 +174,7 @@ def arithAA{mode, F, W, X, R} = {
|
||||
fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 = {
|
||||
# show{F, swap, mode, W, X, R}
|
||||
assert{len>0}
|
||||
def bulk = vw / max{width{W}, width{R}}
|
||||
def bulk = vw / __max{width{W}, width{R}}
|
||||
def TY = [bulk]R
|
||||
def overflow = tern{mode==1, {i}=>return{1}, {i}=>return{i}}
|
||||
|
||||
|
||||
@ -54,8 +54,8 @@ fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
||||
extract{r, 0}
|
||||
}
|
||||
|
||||
export{'si_fold_min_f64', fold_idem{f64,min}}
|
||||
export{'si_fold_max_f64', fold_idem{f64,max}}
|
||||
export{'si_fold_min_f64', fold_idem{f64,__min}}
|
||||
export{'si_fold_max_f64', fold_idem{f64,__max}}
|
||||
|
||||
fn fold_assoc_0{T, op}(x:*T, len:u64) : T = {
|
||||
a:T = 0
|
||||
@ -236,7 +236,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
||||
fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = {
|
||||
n <<= el
|
||||
def vl = arch_defvw / 8
|
||||
def thr = min{vl+2, 20}
|
||||
def thr = __min{vl+2, 20}
|
||||
if ((not has_simd) or n < vl or l > usz~~thr>>el or l<<el >= thr) return{0}
|
||||
nv := n / vl
|
||||
if (has_simd and (l & (l-1)) == 0) {
|
||||
|
||||
@ -228,12 +228,12 @@ Make ← {𝕊ns:
|
||||
sa⇐⟨⟩ ⋄ overflowEq⇐1
|
||||
}
|
||||
Make {
|
||||
commutative⇐1 ⋄ repr⇐"⌊" ⋄ cbqn⇐"floor" ⋄ si⇐"min" ⋄ id⇐"min"
|
||||
commutative⇐1 ⋄ repr⇐"⌊" ⋄ cbqn⇐"floor" ⋄ si⇐"__min" ⋄ id⇐"min"
|
||||
aa_a ⇐ bitand∾u_numDiag ⋄ aa_b ⇐ ⟨⟩
|
||||
sa⇐⟨"iiiu"⟩ ⋄ overflowEq⇐0
|
||||
}
|
||||
Make {
|
||||
commutative⇐1 ⋄ repr⇐"⌈" ⋄ cbqn⇐"ceil" ⋄ si⇐"max" ⋄ id⇐"max"
|
||||
commutative⇐1 ⋄ repr⇐"⌈" ⋄ cbqn⇐"ceil" ⋄ si⇐"__max" ⋄ id⇐"max"
|
||||
aa_a ⇐ bitor∾u_numDiag ⋄ aa_b ⇐ ⟨⟩
|
||||
sa⇐⟨"iiiu"⟩ ⋄ overflowEq⇐0
|
||||
}
|
||||
|
||||
@ -8,7 +8,7 @@ def __eq{a:V=[_]_, b if knum{b}} = a == V**b
|
||||
|
||||
def group_statistics{T} = {
|
||||
def usz_accumulator = count_accumulator{usz, ...}
|
||||
def max_accumulator = assoc_accumulator{max, ..., -1}
|
||||
def max_accumulator = assoc_accumulator{__max, ..., -1}
|
||||
|
||||
def {types, acc_gen, ops} = each{tup,
|
||||
tup{u8, any_accumulator, {_,w} => w < -1}, # bad
|
||||
|
||||
@ -70,7 +70,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
|
||||
# Iterate over existing elements in blocks of e<=32
|
||||
# Branchlessly skips over h0
|
||||
while (i < s) {
|
||||
e := min{s-i, usz~~32}
|
||||
e := __min{s-i, usz~~32}
|
||||
b:u32 = 0
|
||||
@for_backwards (h in hash+i over e) b = 2*b + promote{u32, h!=h0}
|
||||
while (b!=0) {
|
||||
@ -78,7 +78,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
|
||||
h := hash->j
|
||||
hash <-{j} h0
|
||||
k0 := h>>sh
|
||||
if (ordered) { k = max{k0, k+1} } # k0 can't be less than k
|
||||
if (ordered) { k = __max{k0, k+1} } # k0 can't be less than k
|
||||
else { k = k0; while (hash->k!=h0) ++k }
|
||||
cc += cast_i{ux, k-k0}
|
||||
hash <-{k} h
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
def __shl{(u16)}{a:T, b} = T~~(re_el{u16,a}<<b) # for x86's lack of u8 shift
|
||||
def __shr{(u16)}{a:T, b} = T~~(re_el{u16,a}>>b)
|
||||
def broadcast{[(n*2)]E, x:[n]E} = pair{x, x}
|
||||
def pow2_up{v, least} = max{least, 1<<ceil_log2{v}} # least ⌈ ⌈⌾(2⊸⋆⁼) v
|
||||
def pow2_up{v, least} = __max{least, 1<<ceil_log2{v}} # least ⌈ ⌈⌾(2⊸⋆⁼) v
|
||||
|
||||
# make a LUT of at least nt elements in tab, to be indexed by [ni_real≥ni]u8
|
||||
# E must be unsigned
|
||||
@ -177,9 +177,9 @@ def lut_gen{mode, E==u8, nt, ni if hasarch{'AARCH64'} and nt<=16*4 and ni<=16} =
|
||||
# TODO E==u8 nt==128 via tbx4(tbl4,d,i-64)
|
||||
|
||||
def lut_gen{mode, E, nt, ni if mode=='i' and hasarch{'AVX2'} and (E==u16 or E==u64)} = zip_halves{mode, E, nt, ni}
|
||||
# def lut_gen{mode, E, nt, ni if mode=='c' and hasarch{'AVX2'} and E==u16} = widen_inds{mode, E, nt, max{ni,16}, 2}
|
||||
# def lut_gen{mode, E, nt, ni if mode=='c' and hasarch{'AVX2'} and E==u16} = widen_inds{mode, E, nt, __max{ni,16}, 2}
|
||||
# def lut_gen{mode, E, nt, ni if mode=='c' and hasarch{'AVX2'} and E==u64} = widen_inds{mode, E, nt, ni, 2}
|
||||
# def lut_gen{mode, E, nt, ni if mode=='c' and E==u64} = zip_halves{mode, E, nt, ni} # widen_inds{mode, E, nt, max{ni,16}, 2}
|
||||
# def lut_gen{mode, E, nt, ni if mode=='c' and E==u64} = zip_halves{mode, E, nt, ni} # widen_inds{mode, E, nt, __max{ni,16}, 2}
|
||||
|
||||
def lut_gen{mode, E==u64, nt, ni if nt>16 and hasarch{'AVX2'}} = 0
|
||||
|
||||
|
||||
@ -145,7 +145,7 @@ def for_mu{bulk, unr, extra}{vars,begin==0,end,iter} = {
|
||||
def done = makelabel{}
|
||||
cs:u64 = 0
|
||||
while (1) {
|
||||
def ce = min{m, cs + flush_max-1}
|
||||
def ce = __min{m, cs + flush_max-1}
|
||||
@for (i from cs to ce) ml_exec{tup{i}, iter, vars, bulk, mask_none}
|
||||
if (ce == m) goto{done}
|
||||
cs = ce
|
||||
@ -168,7 +168,7 @@ def for_mu{bulk, unr, extra}{vars,begin==0,end,iter} = {
|
||||
def done = makelabel{}
|
||||
def unr_end = m-unr + 1
|
||||
while (1) {
|
||||
def ce = min{unr_end, i + (flush_max-unr)*unr}
|
||||
def ce = __min{unr_end, i + (flush_max-unr)*unr}
|
||||
while (i < ce) unr_iter{}
|
||||
if (ce == unr_end) goto{done}
|
||||
flush{}
|
||||
|
||||
@ -62,9 +62,9 @@ def fold_add {a:T=[_]E if nvec{T}} = emit{E, ntyp{'vaddv', T}, a}
|
||||
def fold_addw{a:T=[_]E if nveci{T}} = emit{w_d{E}, ntyp{'vaddlv', T}, a}
|
||||
def fold_min {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vminv', T}, a}
|
||||
def fold_max {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vmaxv', T}, a}
|
||||
def vfold{(min), x:T if nvec{T} and ~nveci{T,64}} = fold_min{x}
|
||||
def vfold{(max), x:T if nvec{T} and ~nveci{T,64}} = fold_max{x}
|
||||
def vfold{(+), x:T if nvec{T}} = fold_add{x}
|
||||
def vfold{(__min), x:T if nvec{T} and ~nveci{T,64}} = fold_min{x}
|
||||
def vfold{(__max), x:T if nvec{T} and ~nveci{T,64}} = fold_max{x}
|
||||
def vfold{(__add), x:T if nvec{T}} = fold_add{x}
|
||||
|
||||
def store{ptr:*E, x:V=[_]E, vl if nvec{V} and vl*width{E}<=64} = {
|
||||
def E=ty_u{vl*width{E}}
|
||||
|
||||
@ -510,7 +510,7 @@ def swap_elts{x:V=[k]_, el_bytes} = {
|
||||
if (el_bytes == 8) vshl{x, x, k/2}
|
||||
else V~~reverse_units{2, re_el{ty_u{8*el_bytes}, x}}
|
||||
} else if (any_sel or (hasarch{'SSE2'} and el_bytes >= 2)) {
|
||||
def wd = min{el_bytes, 8}
|
||||
def wd = __min{el_bytes, 8}
|
||||
def l = el_bytes/wd
|
||||
def i = iota{l}
|
||||
shuf{ty_u{wd*8}, x, merge{i+l, i}}
|
||||
@ -643,7 +643,7 @@ def rep_const_bool_odd_mask4{
|
||||
{mc4, s4} := double{double{tup{sm0, s1}}}
|
||||
# Submasks pick one mask out of a combination of 4
|
||||
def or_adv{m, s} = { m |= advance{m,s} }
|
||||
@for (min{k/4 - 1, n/4}) or_adv{sm0,s4}
|
||||
@for (__min{k/4 - 1, n/4}) or_adv{sm0,s4}
|
||||
submasks := scan{advance, tup{sm0, ...3**s1}}
|
||||
mask_tail := advance{sm0, s4} &~ sm0
|
||||
|
||||
@ -730,7 +730,7 @@ def rep_const_bool_small_odd{W=[wl](u64), max_wv, wv, get_perm_x, output} = {
|
||||
b := selx{xv, ind} & make{W, mask}
|
||||
r := (b<<k) - b
|
||||
def selx_nz{x, i} = { def nz = i != -1; selx{x, i * nz} & W~~make{[4]i32, -nz} }
|
||||
o := (if (ew==8) selx else selx_nz){xo, flat_table{min, oi, 255-256*(0<iota{se})}}
|
||||
o := (if (ew==8) selx else selx_nz){xo, flat_table{__min, oi, 255-256*(0<iota{se})}}
|
||||
output{r|o}
|
||||
}
|
||||
each{step,
|
||||
|
||||
@ -41,7 +41,7 @@ def get_scan_last{op, pre} = {
|
||||
def scan_idem = scan_scal
|
||||
fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
def {scan, last} = get_scan_last{op, make_scan_idem{T, op}}
|
||||
def cmp = match (op) { {(min)} => (>); {(max)} => (<) }
|
||||
def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) }
|
||||
def step = arch_defvw/width{T}
|
||||
def V = [step]T
|
||||
p:= V**init
|
||||
@ -65,17 +65,17 @@ fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||
if (q!=0) store_blended_hom{rv+e, mask_of_first{V, q}, last{load{xv,e}, p}}
|
||||
}
|
||||
|
||||
export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}}
|
||||
export{'si_scan_min_init_i16', scan_idem{i16, min}}; export{'si_scan_max_init_i16', scan_idem{i16, max}}
|
||||
export{'si_scan_min_init_i32', scan_idem{i32, min}}; export{'si_scan_max_init_i32', scan_idem{i32, max}}
|
||||
export{'si_scan_min_init_f64', scan_idem{f64, min}}; export{'si_scan_max_init_f64', scan_idem{f64, max}}
|
||||
export{'si_scan_min_init_i8', scan_idem{i8 , __min}}; export{'si_scan_max_init_i8', scan_idem{i8 , __max}}
|
||||
export{'si_scan_min_init_i16', scan_idem{i16, __min}}; export{'si_scan_max_init_i16', scan_idem{i16, __max}}
|
||||
export{'si_scan_min_init_i32', scan_idem{i32, __min}}; export{'si_scan_max_init_i32', scan_idem{i32, __max}}
|
||||
export{'si_scan_min_init_f64', scan_idem{f64, __min}}; export{'si_scan_max_init_f64', scan_idem{f64, __max}}
|
||||
|
||||
fn scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = {
|
||||
scan_idem{T, op}(x, r, len, get_id{op, T})
|
||||
}
|
||||
export{'si_scan_min_i8', scan_idem_id{i8 , min}}; export{'si_scan_max_i8', scan_idem_id{i8 , max}}
|
||||
export{'si_scan_min_i16', scan_idem_id{i16, min}}; export{'si_scan_max_i16', scan_idem_id{i16, max}}
|
||||
export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', scan_idem_id{i32, max}}
|
||||
export{'si_scan_min_i8', scan_idem_id{i8 , __min}}; export{'si_scan_max_i8', scan_idem_id{i8 , __max}}
|
||||
export{'si_scan_min_i16', scan_idem_id{i16, __min}}; export{'si_scan_max_i16', scan_idem_id{i16, __max}}
|
||||
export{'si_scan_min_i32', scan_idem_id{i32, __min}}; export{'si_scan_max_i32', scan_idem_id{i32, __max}}
|
||||
|
||||
# Assumes identity is 0
|
||||
def scan_plus = scan_assoc_id0{+}
|
||||
@ -103,7 +103,7 @@ def shift_first{c:V=[l]_, p:V} = {
|
||||
fn scan_stride_assoc{op, T, Ret, check_over}(xv:*void, rv:*void, ia:usz, l:usz) : Ret = {
|
||||
def minvalue{(f64)} = -1/0; def maxvalue{(f64)} = 1/0
|
||||
def id = match (op) {
|
||||
{(min)} => maxvalue; {(max)} => minvalue
|
||||
{(__min)} => maxvalue; {(__max)} => minvalue
|
||||
{(+)} => ({_}=>0)
|
||||
}
|
||||
x:= *T~~xv; r:= *T~~rv
|
||||
@ -202,7 +202,7 @@ def check_add_over{M, w:V=[_]E, x:V, r:V} = {
|
||||
}
|
||||
def check_add_over{M, x, r} = check_add_over{M, r-x, x, r}
|
||||
export_tab{'si_scan_stride_minmax',
|
||||
flat_table{scan_stride_assoc, tup{min,max}, tup{i8,i16,i32,f64}}
|
||||
flat_table{scan_stride_assoc, tup{__min,__max}, tup{i8,i16,i32,f64}}
|
||||
}
|
||||
export_tab{'si_scan_stride_add', tup{
|
||||
...each{scan_stride_assoc{+, ., u1, check_add_over}, tup{i8,i16,i32}},
|
||||
@ -394,7 +394,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
|
||||
}
|
||||
# Sum as many vector registers as possible; modifies c and i
|
||||
def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
|
||||
def b = max{width{R}/2, width{X}}
|
||||
def b = __max{width{R}/2, width{X}}
|
||||
def bulk = arch_defvw/b
|
||||
|
||||
def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size
|
||||
|
||||
@ -39,7 +39,7 @@ def prefix_byshift{op, sh} = {
|
||||
{v:T} => pre{v, if (isvec{T}) elwidth{T} else 1}
|
||||
}
|
||||
|
||||
def get_id{op,T} = (match (op) { {(min)}=>maxvalue; {(max)}=>minvalue }){T}
|
||||
def get_id{op,T} = (match (op) { {(__min)}=>maxvalue; {(__max)}=>minvalue }){T}
|
||||
|
||||
def make_scan_idem{T, op, up} = {
|
||||
# Within each lane, scan using shifts by powers of 2. First k elements
|
||||
|
||||
@ -220,7 +220,7 @@ def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = {
|
||||
if ((m&(m-1)) != 0) { # More bits than one
|
||||
# Filter out values equal to the previous, or first new
|
||||
def pind = (iota{k}&15) - 1
|
||||
prev:= make{VI, each{max{0, .}, pind}}
|
||||
prev:= make{VI, each{__max{0, .}, pind}}
|
||||
e:= ~hom_to_int{v == VI**TI~~xi}
|
||||
e&= base{2,pind<0} | ~hom_to_int{v == sel{[16]i8, v, prev}}
|
||||
if (rbit) rv&= e | -m # Don't remove first bit
|
||||
@ -301,7 +301,7 @@ export{'simd_index_tab_u8', simd_index_tab_u8{usz}}
|
||||
def bad_float_i64{x:T=[_](f64)} = {
|
||||
a:= abs{x}
|
||||
s:= T**(1<<53)
|
||||
min{s,a} != s-(s-a) # ~ ((a<=s) & (a == s-(s-a)))
|
||||
__min{s,a} != s-(s-a) # ~ ((a<=s) & (a == s-(s-a)))
|
||||
}
|
||||
def bad_float_i64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x!=floor{x}) | (abs{x}>T**(1<<53))
|
||||
def bad_float_i64{x:T=[_](f64) if hasarch{'AARCH64'}} = x != cvt{f64, cvt{i64, x}}
|
||||
@ -315,8 +315,8 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
||||
def bulk = arch_defvw/width{E}
|
||||
def VT = [bulk]E
|
||||
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
|
||||
def min_a = assoc_accumulator{min, unr, VT}
|
||||
def max_a = assoc_accumulator{max, unr, VT}
|
||||
def min_a = assoc_accumulator{__min, unr, VT}
|
||||
def max_a = assoc_accumulator{__max, unr, VT}
|
||||
@for_mu{bulk, unr, mu_extra{min_a,max_a}}(cx in tup{VT,x}, M in 'm' over is to n) {
|
||||
if (E==f64 and any_hom{M, ...each{bad_float_i64, cx}}) return{0}
|
||||
min_a{'acc', M, cx}
|
||||
@ -328,8 +328,8 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
||||
min1 = max1 = *x
|
||||
@for (x over i to n) {
|
||||
if (E==f64 and rare{x != emit{f64, '', emit{i64, '', x}}}) return{0}
|
||||
min1 = min{min1, x}
|
||||
max1 = max{max1, x}
|
||||
min1 = __min{min1, x}
|
||||
max1 = __max{max1, x}
|
||||
}
|
||||
}
|
||||
store{res, 0, cast_i{i64, min1}}
|
||||
@ -356,7 +356,7 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it
|
||||
|
||||
log := clzc{m}
|
||||
# Max size
|
||||
msl := max{clzc{(m+m/2)|4}+1, min{ux~~14,clzc{m+n/4}}}
|
||||
msl := __max{clzc{(m+m/2)|4}+1, __min{ux~~14,clzc{m+n/4}}}
|
||||
msz := usz~~1 << msl
|
||||
# Starting log-size (try_vec_memb requires size>4)
|
||||
sl := msl; if (msl>=14) sl = 12+(msl&1)
|
||||
@ -365,7 +365,7 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it
|
||||
# Filling e slots past the end requires e*(e+1)/2 collisions, so
|
||||
# m entries with <2 each can fill <sqrt(4*m)
|
||||
def cc_stop = 2*cast_i{u64,m}
|
||||
ext := promote{usz, tern{m<=b, max{m,U~~4}, b + (U~~1 << (log/2 + 1))}}
|
||||
ext := promote{usz, tern{m<=b, __max{m,U~~4}, b + (U~~1 << (log/2 + 1))}}
|
||||
maxh := T~~maxvalue{T}
|
||||
def aux = prim!='∊'
|
||||
def {tabs, sz, sh, div_thresh, hash_resize, hash_free} = hash_alloc{
|
||||
@ -576,7 +576,7 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it
|
||||
if (has_maxh) { ++uniq; unsequester_maxh{inds} }
|
||||
ind_rev_lookup{uniq, has_maxh}
|
||||
# Propagate to later occurrences
|
||||
@for (r in rp over i to n) r = load{rp, min{i, u32~~r+n}}
|
||||
@for (r in rp over i to n) r = load{rp, __min{i, u32~~r+n}}
|
||||
}
|
||||
|
||||
{'⊒', inds, link} => {
|
||||
@ -614,11 +614,11 @@ def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
|
||||
io := make{V, each{{k} => T~~k<<sh - 1, 1+iota{vl}}}
|
||||
id := V**(T~~vl<<sh)
|
||||
mv := V**0
|
||||
@for (h in *V~~hash over sz/vl) { mv=max{mv, io-min{h,io}}; io+=id }
|
||||
max_off := vfold{max, mv} >> sh
|
||||
@for (h in *V~~hash over sz/vl) { mv=__max{mv, io-__min{h,io}}; io+=id }
|
||||
max_off := vfold{__max, mv} >> sh
|
||||
# sz==1<<sh, so when i>=sz the above overflows
|
||||
# And we have to handle maxh specially anyway
|
||||
def mx{i,h} = { max_off = max{max_off, i-h>>sh} }
|
||||
def mx{i,h} = { max_off = __max{max_off, i-h>>sh} }
|
||||
i:=cast_i{T,sz}; h:T=i; while ((h=load{hash,i})<maxh) { mx{i,h}; ++i }
|
||||
if (has_maxh) mx{i,maxh}
|
||||
|
||||
|
||||
@ -99,8 +99,8 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
|
||||
try_nt{2}
|
||||
}
|
||||
def select_rows_parts = each{select_rows, tup{u8, u16, u32, u64}}
|
||||
def max_nt = fold{max, each{select{.,0}, join{select_rows_parts}}}
|
||||
def max_ni = fold{max, each{select{.,1}, join{select_rows_parts}}}
|
||||
def max_nt = fold{__max, each{select{.,0}, join{select_rows_parts}}}
|
||||
def max_ni = fold{__max, each{select{.,1}, join{select_rows_parts}}}
|
||||
def select_rows_tab_h = lb{max_nt}+1
|
||||
|
||||
# >{t ← (¯1+⌊16÷𝕩)⌾(¯1⊸⊑) ⌊𝕩÷˜↕16 ⋄ ⟨𝕩, ⌊16÷𝕩, 1+¯1⊑t, t⟩}¨ 2↓↕8
|
||||
@ -153,8 +153,8 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
|
||||
}, select_rows_parts}}}}
|
||||
|
||||
def exportP{T, n, vs} = { a:*T = vs; export{n, a} }
|
||||
exportP{u8, 'select_rows_max_indn', each{{row} => if (length{row}==0) 0 else one_val{ each{select{.,1}, row}}, select_rows_parts}}
|
||||
exportP{u8, 'select_rows_min_logcsz', each{{row} => if (length{row}==0) 0 else lb{fold{min, each{select{.,0}, row}}}, select_rows_parts}}
|
||||
exportP{u8, 'select_rows_max_indn', each{{row} => if (length{row}==0) 0 else one_val{ each{select{.,1}, row}}, select_rows_parts}}
|
||||
exportP{u8, 'select_rows_min_logcsz', each{{row} => if (length{row}==0) 0 else lb{fold{__min, each{select{.,0}, row}}}, select_rows_parts}}
|
||||
|
||||
def select_rows_better = scan{{p,{v,i}} => if (length{v}==0) p else i, 0, each{tup, select_rows_parts, range{4}}}
|
||||
exportP{u8, 'select_rows_better', select_rows_better+1}
|
||||
|
||||
@ -37,9 +37,9 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
# fold with either Max or Bitwise Or, truncating/zero-extending elements to TE
|
||||
def fold_total{TE, x:[_]T} = cast_i{TE, vfold{|, x}}
|
||||
def fold_total{TE, x:[_]T if hasarch{'AARCH64'}} = {
|
||||
if (width{T}!=64) cast_i{TE, vfold{max, x}}
|
||||
if (width{T}!=64) cast_i{TE, vfold{__max, x}}
|
||||
else if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
|
||||
else vfold{max, narrow{TE, x}}
|
||||
else vfold{__max, narrow{TE, x}}
|
||||
}
|
||||
|
||||
def mix_pos{x:T=[_]E} = x & (T ** ~E~~1)
|
||||
@ -50,19 +50,19 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
minv:= T**0
|
||||
maxv:= T**0
|
||||
def me{M, minc, maxc} = {
|
||||
minv = min{minv, M{minc}}
|
||||
maxv = max{maxv, M{maxc}}
|
||||
minv = __min{minv, M{minc}}
|
||||
maxv = __max{maxv, M{maxc}}
|
||||
}
|
||||
def me{M, vs} = {
|
||||
minc:= zero_promote{T, tree_fold{min, vs}} # could pack pairs in v to low & high halves, but an extra min costs the same or less than an insert
|
||||
maxc:= zero_promote{T, tree_fold{max, vs}}
|
||||
minc:= zero_promote{T, tree_fold{__min, vs}} # could pack pairs in v to low & high halves, but an extra min costs the same or less than an insert
|
||||
maxc:= zero_promote{T, tree_fold{__max, vs}}
|
||||
me{M, minc, maxc}
|
||||
}
|
||||
|
||||
def me{if hasarch{'AARCH64'}} = {
|
||||
mint:= ty_u{vfold{min, minv}}
|
||||
maxt:= ty_u{vfold{max, maxv}} &~ 1
|
||||
cast_i{u32, tern{mint==0, maxt, max{maxt, -mint-1} | 2}}
|
||||
mint:= ty_u{vfold{__min, minv}}
|
||||
maxt:= ty_u{vfold{__max, maxv}} &~ 1
|
||||
cast_i{u32, tern{mint==0, maxt, __max{maxt, -mint-1} | 2}}
|
||||
}
|
||||
def me{} = fold_total{u32, mix_neg{minv} | mix_pos{maxv}}
|
||||
def me{'minmax'} = 1
|
||||
@ -101,8 +101,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
def acc = int_acc{XV}
|
||||
if (acc{'minmax'}) {
|
||||
@for_mu{bulk, 4}(xs in xb, M in 'm' over len) {
|
||||
minc:= tree_fold{min, xs}
|
||||
maxc:= tree_fold{max, xs}
|
||||
minc:= tree_fold{__min, xs}
|
||||
maxc:= tree_fold{__max, xs}
|
||||
if (M{0}==0 and any_hom{M, (minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}) return{0xffff_ffff}
|
||||
acc{M, minc, maxc}
|
||||
}
|
||||
|
||||
@ -428,9 +428,9 @@ fn transpose{T, {k, kh}}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : voi
|
||||
}
|
||||
}
|
||||
# Small width: fixed, or over-reading partial kernel
|
||||
def use_part_w = wT<=16 and k>max{4,8/wT}
|
||||
def w_max = 1 + (if (use_part_w) max{4, k/2-1} else 7)
|
||||
if (has_simd and w < max{w_max, k}) {
|
||||
def use_part_w = wT<=16 and k>__max{4,8/wT}
|
||||
def w_max = 1 + (if (use_part_w) __max{4, k/2-1} else 7)
|
||||
if (has_simd and w < __max{w_max, k}) {
|
||||
try_fixed_dim{transpose_fixed_width {rp, xp, ., h, hs}, w, ws, h, w_max}
|
||||
if (use_part_w and h>=kh and w>=k/2 and w<k) {
|
||||
loop_fixed_width{xp, rp, h, kh, ws, kernel_part{w}{., ., k, kh, ws, hs}}
|
||||
@ -439,7 +439,7 @@ fn transpose{T, {k, kh}}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : voi
|
||||
}
|
||||
# Small height: fixed, or kernel with overlapping writes
|
||||
# Overlapping is slower than over-reading, so it's only used when needed
|
||||
if (has_simd and h < max{8, k}) {
|
||||
if (has_simd and h < __max{8, k}) {
|
||||
try_fixed_dim{transpose_fixed_height{rp, xp, w, ws, .}, h, hs, w, 8}
|
||||
if (k>8 and w>=k and h>=kh/2) {
|
||||
loop_fixed_height{xp, rp, w, k, hs, kernel_part_h{h}{., ., k, kh, ws, hs}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user