rename isvec→vect and isprim→ptrt
This commit is contained in:
parent
6eb454f859
commit
2202726005
@ -43,10 +43,10 @@ local def scalar_acc_impl{F, T, ident} = {
|
||||
local def scal_bool{T} = is{T,'!'} or isunsigned{T}
|
||||
|
||||
local def add_promote{a:A,b} = a + promote{A, b}
|
||||
def assoc_accumulator{F, '!', T if isprim{T}, ident} = scalar_acc_impl{F, T, ident}
|
||||
def assoc_accumulator{F, '!', T if primt{T}, ident} = scalar_acc_impl{F, T, ident}
|
||||
def count_accumulator{DE, '!', T if scal_bool{T} and isunsigned{DE}} = scalar_acc_impl{add_promote, DE, 0}
|
||||
def bool_accumulator {F, '!', T if scal_bool{T}, ident} = scalar_acc_impl{F, u1, ident}
|
||||
def sum_accumulator {DE, '!', T if isprim{T}} = scalar_acc_impl{add_promote, DE, 0}
|
||||
def sum_accumulator {DE, '!', T if primt{T}} = scalar_acc_impl{add_promote, DE, 0}
|
||||
|
||||
|
||||
|
||||
@ -69,7 +69,7 @@ local def acc_impl{A if kgen{A}, M if kgen{M}, unr if knum{unr}, VT=[k]E, vinit0
|
||||
{_:[_]_} => acc_vec = A{mask_none, acc_vec, v}
|
||||
{{v0:_}} => acc_vec = A{mask_none, acc_vec, v0}
|
||||
{{..._}} => each{{a,c} => a = A{mask_none, a,c}, acc_tup, v}
|
||||
{_:T if isprim{T}} => acc_scal = A{mask_none, acc_scal, v}
|
||||
{_:T if primt{T}} => acc_scal = A{mask_none, acc_scal, v}
|
||||
}
|
||||
}
|
||||
def me{'acc', M if kgen{M}, v} = {
|
||||
@ -114,7 +114,7 @@ def sum_accumulator{DE, unr, VT=[_]SE if int_els{DE,SE} and DE>SE and hasarch{'A
|
||||
def VM = el_m{VT}
|
||||
def [_]ME = VM
|
||||
def exact = DE == ME
|
||||
def addpwa{a:(A), x:E if isprim{E}} = a + promote{A,x}
|
||||
def addpwa{a:(A), x:E if primt{E}} = a + promote{A,x}
|
||||
def {acc, acc_scal, acc_tup, acc_vec} = acc_impl{mask_ident{addpwa,0}, +, unr, VM, A, 0}
|
||||
|
||||
def me{...} = acc
|
||||
@ -270,7 +270,7 @@ def count_accumulator{DE, unr, VT=[k]SE if isunsigned{DE} and int_els{DE,SE}} =
|
||||
|
||||
# implicit identity values
|
||||
local def of_e{[_]E, G} = G{E}
|
||||
local def of_e{E if isprim{E}, G} = G{E}
|
||||
local def of_e{E if primt{E}, G} = G{E}
|
||||
def assoc_accumulator{F==__min, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) E~~1/0 else maxvalue{E}}}
|
||||
def assoc_accumulator{F==__max, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, of_e{T, {E} => if (isfloat{E}) -E~~1/0 else minvalue{E}}}
|
||||
def assoc_accumulator{F==__add, unr, T if ktyp{T}} = assoc_accumulator{F, unr, T, 0}
|
||||
|
||||
@ -15,10 +15,9 @@ def extend_each{G, ...args}{...fs} = each{{f, ...args} => (if (length{args}>0) G
|
||||
|
||||
def isunsigned{T} = isint{T} and not issigned{T}
|
||||
|
||||
def isvec {T} = ktyp{T} and same{typekind{T}, 'vector'} # TODO rename these to better reflect that they apply to types
|
||||
def isprim{T} = ktyp{T} and same{typekind{T}, 'primitive'}
|
||||
def isptr {T} = ktyp{T} and same{typekind{T}, 'pointer'}
|
||||
def any_num = match { {x:T}=>isprim{T}; {x} => knum{x} }
|
||||
def vect {T} = ktyp{T} and same{typekind{T}, 'vector'}
|
||||
def primt{T} = ktyp{T} and same{typekind{T}, 'primitive'}
|
||||
def any_num = match { {x:T}=>primt{T}; {x} => knum{x} }
|
||||
def any_int = match { {x:T}=>isint{T}; {x} => knum{x} and (x>>0) == x }
|
||||
def int_idx{_, _} = 0
|
||||
def int_idx{k if knum{k}, l} = (k>>0)==k and k>=0 and k<l
|
||||
@ -28,8 +27,8 @@ def reinterpret{T, x:T} = x
|
||||
def export_tab{name, fs} = { v:*one_type{fs} = fs; export{name, v} }
|
||||
|
||||
oper &~ andnot infix none 35
|
||||
def andnot{a, b:T if any_int{a} and isprim{T}} = a & ~b
|
||||
def andnot{a:T, b if isprim{T} and knum{b}} = a & ~T~~b
|
||||
def andnot{a, b:T if any_int{a} and primt{T}} = a & ~b
|
||||
def andnot{a:T, b if primt{T} and knum{b}} = a & ~T~~b
|
||||
|
||||
oper &- ({v:T,m:(u1)} => v & -promote{T,m}) infix left 35
|
||||
|
||||
@ -52,8 +51,8 @@ def eachx{F, ...args} = {
|
||||
}
|
||||
|
||||
|
||||
def load {...args if match (...args) { {ptr:*E, idx if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad load', ...args}
|
||||
def store{...args if match (...args) { {ptr:*E, idx, val if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad store', ...args}
|
||||
def load {...args if match (...args) { {ptr:*E, idx if not vect{E}}=>0; {..._}=>1 } } = assert{0, 'bad load', ...args}
|
||||
def store{...args if match (...args) { {ptr:*E, idx, val if not vect{E}}=>0; {..._}=>1 } } = assert{0, 'bad store', ...args}
|
||||
def load{p:*_} = load{p, 0}
|
||||
# def store{p:*_, v} = store{p, 0, v}
|
||||
def loadu {p:*T if isunsigned{T}} = emit{T, merge{'loadu_u', fmtnat{width{T}}}, p}
|
||||
@ -93,7 +92,7 @@ def all_same{{}} = 1
|
||||
def try_same_type{_, default} = default
|
||||
def try_same_type{{h:T, ...t} if all{hastype{.,T}, t}, _} = T
|
||||
|
||||
def broadcast{T, v if isprim{T}} = v
|
||||
def broadcast{T, v if primt{T}} = v
|
||||
def broadcast{n, v if knum{n}} = each{{_}=>v, range{n}}
|
||||
|
||||
# type stats
|
||||
@ -103,11 +102,11 @@ def minvalue{T if issigned{T}} = - (1<<(width{T}-1))
|
||||
def maxvalue{T if issigned{T}} = (1<<(width{T}-1))-1
|
||||
|
||||
# vector width/type checks
|
||||
def w64 {T} = isvec{T} and width{T}==64; def w64 {T,w} = w64{T} and elwidth{T}==w
|
||||
def w128{T} = isvec{T} and width{T}==128; def w128{T,w} = w128{T} and elwidth{T}==w
|
||||
def w256{T} = isvec{T} and width{T}==256; def w256{T,w} = w256{T} and elwidth{T}==w
|
||||
def w512{T} = isvec{T} and width{T}==512; def w512{T,w} = w512{T} and elwidth{T}==w
|
||||
def veca{T} = isvec{T}; def veca{T,w} =isvec{T} and elwidth{T}==w
|
||||
def w64 {T} = vect{T} and width{T}==64; def w64 {T,w} = w64{T} and elwidth{T}==w
|
||||
def w128{T} = vect{T} and width{T}==128; def w128{T,w} = w128{T} and elwidth{T}==w
|
||||
def w256{T} = vect{T} and width{T}==256; def w256{T,w} = w256{T} and elwidth{T}==w
|
||||
def w512{T} = vect{T} and width{T}==512; def w512{T,w} = w512{T} and elwidth{T}==w
|
||||
def veca{T} = vect{T}; def veca{T,w} = vect{T} and elwidth{T}==w
|
||||
|
||||
# width+type checks
|
||||
def genchk{B, F} = match {
|
||||
@ -139,7 +138,7 @@ def re_el{E, x:V} = re_el{E,V} ~~ x
|
||||
|
||||
local def change_qual{q} = {
|
||||
def f{w if knum{w}} = primtype{q, w}
|
||||
def f{T if isprim{T}} = primtype{q, width{T}}
|
||||
def f{T if primt{T}} = primtype{q, width{T}}
|
||||
def f{V=[_]T} = re_el{f{T}, V}
|
||||
def f{x:T} = f{T}~~x
|
||||
}
|
||||
@ -147,9 +146,9 @@ def ty_u = change_qual{'u'}
|
||||
def ty_s = change_qual{'i'}
|
||||
def ty_f = change_qual{'f'}
|
||||
|
||||
def w_n{T, w if isprim{T}} = primtype{quality{T}, w}
|
||||
def w_d{T if isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width
|
||||
def w_h{T if isprim{T}} = to_w{T, width{T}/2}
|
||||
def w_n{T, w if primt{T}} = primtype{quality{T}, w}
|
||||
def w_d{T if primt{T}} = to_w{T, width{T}*2} # double/halve primitive type width
|
||||
def w_h{T if primt{T}} = to_w{T, width{T}/2}
|
||||
|
||||
def n_d{[k]T} = [k*2]T # double/halve vector count
|
||||
def n_h{[k]T} = [k/2]T
|
||||
|
||||
@ -7,7 +7,7 @@ def shuf_impl{rw, ...iw, data} = assert{0, 'shuffling failed', ...data{'info'}}
|
||||
|
||||
def type0{vs} = type{select{vs,0}}
|
||||
# new_shuf{v0:[k]E, v1:[k]E, ..., indices} : [length{indices}]E; -1 for zero, -2 for arbitrary
|
||||
def new_shuf{...vs0 if isvec{try_same_type{vs0,'!'}}, {...is}} = { # : [length{is}]E
|
||||
def new_shuf{...vs0 if vect{try_same_type{vs0,'!'}}, {...is}} = { # : [length{is}]E
|
||||
def vs = each{ty_u, vs0}
|
||||
def vn = length{vs}
|
||||
def S = type0{vs}
|
||||
@ -39,9 +39,9 @@ def new_shuf{...vs0 if isvec{try_same_type{vs0,'!'}}, {...is}} = { # : [length{i
|
||||
|
||||
re_el{E, shuf_impl{width{E}*ni, ...vn**width{S}, data}}
|
||||
}
|
||||
def new_shuf{w if ktyp{w} or knum{w}, ...vs if isvec{try_same_type{vs,'!'}} and (not isvec{w} or width{w} == width{type0{vs}}), {...is}} = {
|
||||
def new_shuf{w if ktyp{w} or knum{w}, ...vs if vect{try_same_type{vs,'!'}} and (not vect{w} or width{w} == width{type0{vs}}), {...is}} = {
|
||||
def S = type0{vs}
|
||||
def D = if (isvec{w}) w else re_el{if (isprim{w}) w else ty_u{w}, S}
|
||||
def D = if (vect{w}) w else re_el{if (primt{w}) w else ty_u{w}, S}
|
||||
S~~new_shuf{...each{~~{D,.}, vs}, is}
|
||||
}
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ include './f64'
|
||||
include './bitops'
|
||||
include './mask'
|
||||
|
||||
def rootty{T if isprim{T}} = T
|
||||
def rootty{T if primt{T}} = T
|
||||
def rootty{[_]T} = T
|
||||
|
||||
def is_s{X} = issigned{rootty{X}}
|
||||
|
||||
@ -202,8 +202,8 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
||||
# Shuffle to undo interleaving of add_res
|
||||
def __shr{x:(V), sh if hasarch{'X86_64'}} = V~~(H~~x >> sh)
|
||||
def __shl{x:(V), sh if hasarch{'X86_64'}} = V~~(H~~x << sh)
|
||||
def __shr{x:(V), sh:T if hasarch{'AARCH64'} and not isvec{T}} = x << V**cast_i{u8,-sh}
|
||||
def __shl{x:(V), sh:T if hasarch{'AARCH64'} and not isvec{T}} = x << V**cast_i{u8, sh}
|
||||
def __shr{x:(V), sh:T if hasarch{'AARCH64'} and not vect{T}} = x << V**cast_i{u8,-sh}
|
||||
def __shl{x:(V), sh:T if hasarch{'AARCH64'} and not vect{T}} = x << V**cast_i{u8, sh}
|
||||
def uz_lane = {
|
||||
l := V**(u8~~1<<el - 1)
|
||||
h := V**(16 - u8~~16>>p2)
|
||||
|
||||
@ -14,8 +14,8 @@ def mask_of_first{T,n if w256{T}} = mask_of_impl{T, n, 256}
|
||||
def mask_of_first{T,n if w128{T}} = mask_of_impl{T, n, 128}
|
||||
def mask_of_first{T,n if w64{T}} = mask_of_impl{T, n, 64}
|
||||
|
||||
def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y}
|
||||
def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M, x!=y}
|
||||
def anyne{x:T, y:T, M if M{0}==0 and vect{T}} = ~all_hom{x==y}
|
||||
def anyne{x:T, y:T, M if M{0}==1 and vect{T}} = any_hom{M, x!=y}
|
||||
def anyne{x:T, y:T, M if M{0}==0 and any_int{x}} = x!=y
|
||||
def anyne{x:T, y:T, M if M{0}==1 and any_int{x}} = M{x^y} != 0
|
||||
def anyne_bit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
@ -36,7 +36,7 @@ def mask_first{n} = {
|
||||
def mask{X, 'to homogeneous bits'} = mask_of_first{X,n}
|
||||
def mask{'count'} = n
|
||||
def mask{{x}} = tup{mask{x}}
|
||||
def mask{x:X if isvec{X}} = x & (X~~mask_of_first{X,n})
|
||||
def mask{x:X if vect{X}} = x & (X~~mask_of_first{X,n})
|
||||
def mask{x:X if any_int{x}} = x & ((1<<n) - 1)
|
||||
def mask{0} = 1
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
def nvec{T} = isvec{T} and (width{T}==64 or width{T}==128)
|
||||
def nvec{T} = vect{T} and (width{T}==64 or width{T}==128)
|
||||
def nvec{T,w} = nvec{T} and elwidth{T}==w
|
||||
def {nveci,nvecs,nvecu,nvecf} = genchks{nvec}
|
||||
|
||||
|
||||
@ -291,8 +291,8 @@ export_tab{'si_constrep', each{rep_const, dat_types}}
|
||||
# Constant replicate on boolean
|
||||
def any_sel = hasarch{'SSSE3'} or hasarch{'AARCH64'}
|
||||
if_inline (hasarch{'AARCH64'}) {
|
||||
def __shl{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,b}
|
||||
def __shr{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,-b}
|
||||
def __shl{a:V=[_]T, b:U if not vect{U}} = a << V**cast_i{T,b}
|
||||
def __shr{a:V=[_]T, b:U if not vect{U}} = a << V**cast_i{T,-b}
|
||||
}
|
||||
|
||||
fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : void = {
|
||||
|
||||
@ -7,8 +7,8 @@ include './f64'
|
||||
include './spaced'
|
||||
include './scan_common'
|
||||
if_inline (hasarch{'AARCH64'}) {
|
||||
def __shl{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,b}
|
||||
def __shr{a:V=[_]T, b:U if not isvec{U}} = a << V**cast_i{T,-b}
|
||||
def __shl{a:V=[_]T, b:U if not vect{U}} = a << V**cast_i{T,b}
|
||||
def __shr{a:V=[_]T, b:U if not vect{U}} = a << V**cast_i{T,-b}
|
||||
}
|
||||
|
||||
# Initialized scan, generic implementation
|
||||
|
||||
@ -36,7 +36,7 @@ def broadcast_last{n:VT} = broadcast_last{n, 1}
|
||||
# at increasing power-of-two shifts
|
||||
def prefix_byshift{op, sh} = {
|
||||
def pre{v:V, k} = if (k < width{V}) pre{op{v, sh{v,k}}, 2*k} else v
|
||||
{v:T} => pre{v, if (isvec{T}) elwidth{T} else 1}
|
||||
{v:T} => pre{v, if (vect{T}) elwidth{T} else 1}
|
||||
}
|
||||
|
||||
def get_id{op,T} = (match (op) { {(__min)}=>maxvalue; {(__max)}=>minvalue }){T}
|
||||
|
||||
@ -268,14 +268,14 @@ export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh{1
|
||||
export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh{1, i32}}
|
||||
export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1, i64}}
|
||||
|
||||
def scalwidth{T} = if (isvec{T}) elwidth{T} else width{T}
|
||||
def scalwidth{T} = if (vect{T}) elwidth{T} else width{T}
|
||||
|
||||
# pext, or boolean compress
|
||||
def pext_width{} = if (hasarch{'AVX2'}) 4 else 1
|
||||
def thresh_bool{} = if (hasarch{'AVX2'}) 128 else 16
|
||||
def pext_popc{x:T, m:T} = {
|
||||
def w = scalwidth{T}
|
||||
def scal{v} = if (isvec{T}) T**v else v
|
||||
def scal{v} = if (vect{T}) T**v else v
|
||||
def lowbits{w,k} = base{1<<k, cdiv{w,k}**1}
|
||||
# At each step, x and z are split into groups of length k
|
||||
# - z tells how many bits in the group are NOT used
|
||||
@ -290,7 +290,7 @@ def pext_popc{x:T, m:T} = {
|
||||
z0 := z & low
|
||||
zm := z>>1 & low
|
||||
tup{ x - (x>>1 & z0), zm + z0 }
|
||||
} else if (hasarch{'AVX2'} and isvec{T} and k >= 32) {
|
||||
} else if (hasarch{'AVX2'} and vect{T} and k >= 32) {
|
||||
# We have variable shifts at these sizes
|
||||
lh := scal{low_s*(1<<h - 1)}
|
||||
zl := z & lh
|
||||
@ -324,12 +324,12 @@ def pext_popc{x:T, m:T} = {
|
||||
pe := fold{|, x&s0, each{gr, g*slice{iota{k/g},1}}}
|
||||
tup{pe, o>>(k-g)}
|
||||
}
|
||||
def build{k==32 if hasarch{'AVX2'} and isvec{T}} = {
|
||||
def build{k==32 if hasarch{'AVX2'} and vect{T}} = {
|
||||
def S = re_el{ty_u{k}, T}
|
||||
def c{T,vs} = each{{v}=>T~~v, vs}
|
||||
c{T, multi_shift{...c{S, build{8}}, 8, k, {s}=>S**s}}
|
||||
}
|
||||
def build{k if not isvec{T} and k > 8} = {
|
||||
def build{k if not vect{T} and k > 8} = {
|
||||
multi_shift{...build{8}, 8, k, {s}=>s}
|
||||
}
|
||||
# Final result
|
||||
|
||||
Loading…
Reference in New Issue
Block a user