singeli code cleanup
This commit is contained in:
parent
7cd19c2840
commit
37da98d871
@ -18,14 +18,10 @@ def w256s = wgen256{{T} => isint{T} & issigned{T}}
|
||||
def w256u = wgen256{{T} => isint{T} & isunsigned{T}}
|
||||
def w256f = wgen256{{T} => isfloat{T}}
|
||||
|
||||
|
||||
def v2i{x:T & w256{T}} = [32]u8 ~~ x # for compact casting for the annoying intrinsic type system
|
||||
def v2f{x:T & w256{T}} = [8]f32 ~~ x
|
||||
def v2d{x:T & w256{T}} = [4]f64 ~~ x
|
||||
|
||||
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
|
||||
def to_el{E, x:T} = to_el{E,T} ~~ x
|
||||
def forv{T & w256{T}} = forc{{v} => *T ~~ v}
|
||||
|
||||
# load & store
|
||||
def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n}
|
||||
@ -42,17 +38,17 @@ def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f3
|
||||
def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',ext{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256f{T, 64}} = emit{T, '_mm256_set1_pd', v}
|
||||
def broadcast{T, v & w256f{T, 32}} = emit{T, '_mm256_set1_ps', v}
|
||||
|
||||
def make{T==[4]f64,a,b,c,d} = emit{T,'_mm256_setr_pd',a,b,c,d}
|
||||
def make{T==[8]f32,a,b,c,d,e,f,g,h} = emit{T,'_mm256_setr_ps',a,b,c,d,e,f,g,h}
|
||||
|
||||
def makeGen{T,s,x} = apply{emit, merge{tup{T,s}, each{{c}=>ext{eltype{T},c}, x}}}
|
||||
def makeGen{T,s,x} = apply{emit, merge{tup{T,s}, each{{c}=>promote{eltype{T},c}, x}}}
|
||||
def make{T,a,b,c,d & w256i{T,64}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}}
|
||||
def make{T,a,b,c,d,e,f,g,h & w256i{T,32}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}}
|
||||
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T,16}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}}
|
||||
@ -70,14 +66,15 @@ def __or {a:T, b:T & w256i{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{
|
||||
|
||||
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
|
||||
|
||||
# f64 comparison
|
||||
# float comparison
|
||||
def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
||||
def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
|
||||
def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0}
|
||||
def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4}
|
||||
def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30}
|
||||
def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29}
|
||||
def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17}
|
||||
def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18}
|
||||
def __eq{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 0}; def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0}
|
||||
def __ne{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 4}; def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4}
|
||||
def __gt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,30}; def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30}
|
||||
def __ge{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,29}; def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29}
|
||||
def __lt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,17}; def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17}
|
||||
def __le{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,18}; def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18}
|
||||
|
||||
# f32 arith
|
||||
def __add{a:T,b:T & T==[8]f32} = emit{T, '_mm256_add_ps', a, b}
|
||||
|
||||
@ -1,11 +1,3 @@
|
||||
# maskstore
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, v}
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, v}
|
||||
|
||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
||||
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
|
||||
|
||||
# min & max
|
||||
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}
|
||||
@ -65,7 +57,7 @@ def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
||||
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
|
||||
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
|
||||
# super questionable pack - takes assumes high halves are zero
|
||||
# super questionable pack - assumes high halves are zero
|
||||
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
|
||||
def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}}
|
||||
|
||||
@ -108,6 +100,13 @@ def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
||||
|
||||
|
||||
# structural operations
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', *i32 ~~ (a+n), m, v}
|
||||
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', *f64 ~~ (a+n), m, v}
|
||||
|
||||
# maskstore with all cases defined, at the cost of not being a single instruction
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
||||
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
|
||||
@ -132,6 +131,7 @@ def sel{I, x:T, i:[32]i8 & w128{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8'
|
||||
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
|
||||
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
|
||||
|
||||
|
||||
# mask stuff
|
||||
def getmask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
||||
def getmask{x:T & w256{T, 16}} = {
|
||||
@ -142,6 +142,7 @@ def any{x:T & w256i{T, 16}} = getmask{[32]u8~~x}!=0
|
||||
def all{x:T & w256i{T, 16}} = getmask{[32]u8~~x} == (1<<32)-1
|
||||
def anyneg{x:T & w256s{T, 16}} = getmask{[32]u8 ~~ ([16]i16~~x < broadcast{[16]i16, 0})}!=0
|
||||
|
||||
|
||||
# conversion
|
||||
# convert packed elements of type F to a result T
|
||||
def cvt{F==u8, T==[16]u16, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi16', a}
|
||||
|
||||
@ -1,17 +1,11 @@
|
||||
include 'skin/c'
|
||||
include 'arch/c'
|
||||
def Size = u64
|
||||
|
||||
oper infix right ~~ reinterpret 55
|
||||
|
||||
def Size = u64
|
||||
def load{x} = *x
|
||||
# TODO move these to a more base file
|
||||
def knum{x} = match{'number',kind{x}}
|
||||
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
||||
def ext {T, x:U & isint{T} & isint{U} & T>=U} = emit{T, '', x}
|
||||
def trunc{T, x & knum{x}} = cast{T, x}
|
||||
def ext {T, x & knum{x}} = cast{T, x}
|
||||
|
||||
# basic ceiling divide
|
||||
# ceiling divide
|
||||
def cdiv{a,b} = (a+b-1)/b
|
||||
|
||||
# base-2 log of a constant power of two
|
||||
@ -22,21 +16,31 @@ def tail{n,x} = x - (x>>n << n) # get the n least significant bits
|
||||
def bit {k,x} = tail{1,x>>k} << k # get the k-th bit
|
||||
|
||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||
def isunsigned{T} = isint{T} & ~issigned{T}
|
||||
def assert{x:u1} = emit{void, 'si_assert', x}
|
||||
|
||||
def cast_p{T, x} = emit{*T, '(void*)', x}
|
||||
|
||||
def anyNum{x} = knum{x}
|
||||
def anyNum{x & match{'constant',kind{x}}} = 1
|
||||
def anyNum{x:T} = match{'primitive',typekind{T}}
|
||||
# various checks
|
||||
def knum{x} = match{kind{x},'number'}
|
||||
def isreg{x} = match{kind{x},'register'}
|
||||
def isconst{x} = match{kind{x},'constant'}
|
||||
def istype{x} = match{kind{x},'type'}
|
||||
def istup{x} = match{kind{x},'tuple'}
|
||||
|
||||
def anyInt{x} = knum{x}
|
||||
def anyInt{x & match{'register',kind{x}} | match{'constant',kind{x}}} = isint{x}
|
||||
def isunsigned{T} = isint{T} & ~issigned{T}
|
||||
|
||||
def isvec {T} = 0; def isvec {T & istype{T}} = match{typekind{T},'vector'}
|
||||
def isprim{T} = 0; def isprim{T & istype{T}} = match{typekind{T},'primitive'}
|
||||
|
||||
|
||||
def __add{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a, b}
|
||||
def __sub{a:T,b & match{'pointer',typekind{T}} & anyInt{b}} = emit{T, 'op +', a, b}
|
||||
def anyNum{x} = isconst{x} | knum{x}
|
||||
def anyNum{x:T} = isprim{T}
|
||||
|
||||
def anyInt{x} = 0
|
||||
def anyInt{x & knum{x}} = (x>>0) == x
|
||||
def anyInt{x & isreg{x}|isconst{x}} = isint{x}
|
||||
|
||||
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
||||
def trunc{T, x & knum{x}} = cast{T, x}
|
||||
|
||||
def tern{c, T, F & anyInt{c}} = {
|
||||
if(c) T
|
||||
@ -44,22 +48,21 @@ def tern{c, T, F & anyInt{c}} = {
|
||||
}
|
||||
|
||||
|
||||
def ty_u{T==i8 } = u8; def ty_u{w== 8} = u8
|
||||
def ty_u{T==i16} = u16; def ty_u{w==16} = u16
|
||||
def ty_u{T==i32} = u32; def ty_u{w==32} = u32
|
||||
def ty_u{T==i64} = u64; def ty_u{w==64} = u64
|
||||
def ty_s{T==u8 } = i8; def ty_s{w== 8} = i8
|
||||
def ty_s{T==u16} = i16; def ty_s{w==16} = i16
|
||||
def ty_s{T==u32} = i32; def ty_s{w==32} = i32
|
||||
def ty_s{T==u64} = i64; def ty_s{w==64} = i64
|
||||
def ty_s{w== 8} = i8; def ty_u{w== 8} = u8
|
||||
def ty_s{w==16} = i16; def ty_u{w==16} = u16
|
||||
def ty_s{w==32} = i32; def ty_u{w==32} = u32
|
||||
def ty_s{w==64} = i64; def ty_u{w==64} = u64
|
||||
|
||||
def ty_u{T==f64} = u64; def ty_u{T==f32} = u32;
|
||||
def ty_s{T==f64} = i64; def ty_s{T==f32} = i32;
|
||||
def ty_u{T & isprim{T}} = ty_u{width{T}}
|
||||
def ty_s{T & isprim{T}} = ty_s{width{T}}
|
||||
|
||||
def ty_vu{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
|
||||
def ty_vs{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def ty_vs{x:T} = ty_vs{T}~~x
|
||||
def ty_vu{x:T} = ty_vu{T}~~x
|
||||
def ty_u{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
|
||||
def ty_s{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def ty_u{x:T & isvec{T}} = ty_u{T}~~x
|
||||
def ty_s{x:T & isvec{T}} = ty_s{T}~~x
|
||||
|
||||
def to_el{E, T & isvec{T}} = [width{T}/width{E}]E
|
||||
def to_el{E, x:T} = to_el{E,T} ~~ x
|
||||
|
||||
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
|
||||
def f{i,l & i==l} = 0
|
||||
@ -84,30 +87,12 @@ def for{vars,begin,end,block} = {
|
||||
i = i+1
|
||||
}
|
||||
}
|
||||
def forc{F} {vars,begin,end,block} = {
|
||||
i:u64 = begin
|
||||
while (i < end) {
|
||||
exec{i, each{F, vars}, block}
|
||||
i = i+1
|
||||
}
|
||||
}
|
||||
|
||||
def maxvalue{T & T==u8 } = 0xff
|
||||
def maxvalue{T & T==u16} = 0xffff
|
||||
def maxvalue{T & T==u32} = 0xffffffff
|
||||
|
||||
# various utilities
|
||||
def isintv{T} = isint{eltype{T}}
|
||||
def isfloatv{T} = isfloat{eltype{T}}
|
||||
def issignedv{T} = issigned{eltype{T}}
|
||||
def isunsignedv{T} = isunsigned{eltype{T}}
|
||||
def istype{T} = match{kind{T},'type'}
|
||||
def istup{T} = match{kind{T},'tuple'}
|
||||
def isvec{T} = 0
|
||||
def isvec{T & istype{T}} = match{typekind{T},'vector'}
|
||||
|
||||
|
||||
# non-vector variants of vector defs
|
||||
# tuple operations
|
||||
def broadcast{T, v & match{typekind{T},'primitive'}} = v
|
||||
def any{v:T & match{typekind{T},'primitive'}} = v
|
||||
def anyneg{v:T & match{typekind{T},'primitive'}} = v<0
|
||||
|
||||
@ -14,7 +14,8 @@ def b_set{x:*u64, n:(Size), v:u1} = {
|
||||
else store{x,n>>6,p & ~m}
|
||||
}
|
||||
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v} = {
|
||||
vc:u64 = promote{u64,v}
|
||||
am:u64 = 64/sz;
|
||||
w:u64 = load{x,n/am}
|
||||
sh:u64 = (n&(am-1)) * sz
|
||||
|
||||
@ -6,15 +6,6 @@ include './avx'
|
||||
include './avx2'
|
||||
include './bitops'
|
||||
|
||||
def fmt{T} = 'unknown'
|
||||
def fmt{T&match{T,__eq}}='='; def fmt{T&match{T,__ne}}='≠'
|
||||
def fmt{T&match{T,__gt}}='>'; def fmt{T&match{T,__ge}}='≥'
|
||||
def fmt{T&match{T,__lt}}='<'; def fmt{T&match{T,__le}}='≤'
|
||||
|
||||
|
||||
def cif{v, G} = { show{'cif argument not known at compile time!'}; G{} }
|
||||
def cif{v, G & match{v,0}} = 0
|
||||
def cif{v, G & match{v,1}} = G{0}
|
||||
|
||||
def fillbits{dst:*u64, len:(Size), v } = { emit{void, 'fillBits', dst, len, v }; return{}; }
|
||||
def fillbits{dst:*u64, len:(Size), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
|
||||
@ -39,10 +30,10 @@ def pathAS{dst, len, T, op, x & issigned{T}} = {
|
||||
xf:f64 = interp_f64{x}
|
||||
xi64:i64 = R{xf}
|
||||
xT:T = trunc{T, xi64}
|
||||
if (rare{ext{i64, xT}!=xi64}) {
|
||||
cif{~eqne{op}, {_}=>{ # NaN was already checked for ≠/=
|
||||
if (rare{promote{i64, xT}!=xi64}) {
|
||||
if(~eqne{op}) { # NaN was already checked for ≠/=
|
||||
if (isNaN{xf}) { call{cmpIX, dst, len, x, op{0,1}}; return{}; }
|
||||
}}
|
||||
}
|
||||
fillbits{dst, len, op{0,xf}}
|
||||
}
|
||||
xT
|
||||
@ -50,7 +41,7 @@ def pathAS{dst, len, T, op, x & issigned{T}} = {
|
||||
|
||||
def pathAS{dst, len, T, op, x & T==f64} = {
|
||||
if (rare{~q_f64{x}}) {
|
||||
cif{~eqne{op}, {_}=>{ if(~q_chr{x}) cmp_err{x}; }}
|
||||
if (~eqne{op}) if (~q_chr{x}) cmp_err{x};
|
||||
fillbits{dst, len, op{0,1}, x}
|
||||
}
|
||||
from_B{T,x}
|
||||
@ -58,7 +49,7 @@ def pathAS{dst, len, T, op, x & T==f64} = {
|
||||
|
||||
def pathAS{dst, len, T, op, x & isunsigned{T}} = {
|
||||
if (rare{~q_chr{x}}) {
|
||||
cif{~eqne{op}, {_}=>{ if(~q_f64{x}) cmp_err{x}; }}
|
||||
if (~eqne{op}) if (~q_f64{x}) cmp_err{x};
|
||||
fillbits{dst, len, op{1,0}, x}
|
||||
}
|
||||
xc32:u32 = from_B{u32,x}
|
||||
@ -71,37 +62,35 @@ def pathAS{dst, len, T, op, x & isunsigned{T}} = {
|
||||
def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
|
||||
xi:Size = 0
|
||||
ri:Size = 0
|
||||
def bam = vcount{VT}*unr
|
||||
am:Size = cdiv{len,bam}
|
||||
def bulk = vcount{VT}*unr
|
||||
am:Size = cdiv{len,bulk}
|
||||
assert{am>0}
|
||||
while (ri < am) {
|
||||
r:u64 = 0
|
||||
@unroll (j from 0 to unr) r = r | (ext{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}))
|
||||
b_setBatch{bam, dst, ri, r}
|
||||
@unroll (j to unr) r = r | (promote{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}))
|
||||
b_setBatch{bulk, dst, ri, r}
|
||||
xi = xi+unr
|
||||
ri = ri+1
|
||||
}
|
||||
}
|
||||
aa2bit{VT, unr, op}(dst:*u64, wr:*u8, xr:*u8, len:Size) : void = {
|
||||
wv:= *VT ~~ wr; ws:= *eltype{VT} ~~ wr
|
||||
xv:= *VT ~~ xr; xs:= *eltype{VT} ~~ xr
|
||||
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
|
||||
xv:= *VT~~xr; xs:= *eltype{VT}~~xr
|
||||
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>load{xs,i}, {i}=>load{xv,i}, dst, len}
|
||||
}
|
||||
|
||||
as2bit{VT, unr, op}(dst:*u64, wr:*u8, x:u64, len:Size) : void = { # show{VT,unr,fmt{op}}
|
||||
as2bit{VT, unr, op}(dst:*u64, wr:*u8, x:u64, len:Size) : void = {
|
||||
wv:= *VT~~wr; ws:= *eltype{VT}~~wr
|
||||
xv:= broadcast{VT, pathAS{dst, len, eltype{VT}, op, x}}
|
||||
any2bit{VT, unr, op, {i}=>load{ws,i}, {i}=>load{wv,i}, {i}=>x, {i}=>xv, dst, len}
|
||||
}
|
||||
|
||||
bitAA{bitop}(dst:*u64, wr:*u8, xr:*u8, len:Size) : void = {
|
||||
ws:= *u64~~wr
|
||||
xs:= *u64~~xr
|
||||
@for (dst,ws,xs over _ from 0 to cdiv{len,64}) dst = bitop{ws,xs}
|
||||
@for (dst, w in *u64~~wr, x in *u64~~xr over _ to cdiv{len,64}) dst = bitop{w,x}
|
||||
}
|
||||
|
||||
not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; assert{am>0}; @for (dst,x over _ from 0 to am) dst = ~x }
|
||||
cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; assert{am>0}; @for (dst,x over _ from 0 to am) dst = x }
|
||||
not(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; assert{am>0}; @for (dst,x over _ to am) dst = ~x }
|
||||
cpy(dst:*u64, x:*u64, len:Size) : void = { am:=cdiv{len,64}; assert{am>0}; @for (dst,x over _ to am) dst = x }
|
||||
|
||||
bitAS{op}(dst:*u64, wr:*u8, x:u64, len:Size) : void = { # show{'bitAS'}
|
||||
xf:f64 = interp_f64{x}
|
||||
|
||||
@ -11,12 +11,15 @@ include './mask'
|
||||
def rootty{T & match{typekind{T},'primitive'}} = T
|
||||
def rootty{T & match{typekind{T},'vector'}} = eltype{T}
|
||||
|
||||
# TODO more to some more headerlike file
|
||||
def ty_dbl{T & i8==T} = i16
|
||||
def ty_dbl{T & i16==T} = i32
|
||||
def ty_dbl{T & i32==T} = i64
|
||||
def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}})
|
||||
def dcast_i{x} = ext{ty_dbl{type{x}}, x}
|
||||
|
||||
# old scalar code
|
||||
# def ty_dbl{T & isprim{T}} = apply{tern{issigned{T}, ty_s, ty_u}, tup{width{T}*2}}
|
||||
# def ty_dbl{T & isvec{T}} = [vcount{T}/2](ty_dbl{eltype{T}})
|
||||
# def dcast_i{x:T} = promote{ty_dbl{T}, x}
|
||||
# def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & match{typekind{T},'primitive'}} = {
|
||||
# r:= F{dcast_i{w}, dcast_i{x}}
|
||||
# tup{r, r!=promote{type{r}, trunc{T, r}}}
|
||||
# }
|
||||
|
||||
# + & -
|
||||
def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}}
|
||||
@ -30,10 +33,6 @@ def arithChk2{F, M, w:T, x:T, i & issigned{rootty{T}}} = {
|
||||
}
|
||||
|
||||
# ×/∧
|
||||
def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & match{typekind{T},'primitive'}} = {
|
||||
r:= F{dcast_i{w}, dcast_i{x}}
|
||||
tup{r, r!=ext{type{r}, trunc{T, r}}}
|
||||
}
|
||||
|
||||
def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i16==eltype{T}} = {
|
||||
rl:= __mul {w,x}
|
||||
@ -55,17 +54,17 @@ def arithChk2{F, M, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} =
|
||||
max:= [8]f32 ~~ broadcast{[8]u32, 0x4efffffe}
|
||||
def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x}
|
||||
f32mul:= cf32{w} * cf32{x}
|
||||
tup{w*x, any{M{[8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', abs{f32mul}, max, 29}}}}
|
||||
tup{w*x, any{M{abs{f32mul} >= max}}}
|
||||
# TODO fallback to the below if the above fails
|
||||
# TODO don't do this, but instead shuffle one half, do math, unshuffle that half
|
||||
# def wp = unpackQ{w, broadcast{T, 0}}
|
||||
# def xp = unpackQ{x, broadcast{T, 0}}
|
||||
# def rp = each{__mul32, wp, xp}
|
||||
# def T2 = ty_dbl{T}
|
||||
# def bad = each{{v}=>{
|
||||
# def T2 = to_el{i64, T}
|
||||
# def bad = each{{v} => {
|
||||
# (((T2~~v) + broadcast{T2,0x80000000}) ^ broadcast{T2, cast{i64,1}<<63}) > broadcast{T2, cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF}}
|
||||
# }, rp}
|
||||
# tup{packQQ{each{{v} => v&broadcast{T2, 0xFFFFFFFF}, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} TODO use M
|
||||
# tup{packQQ{each{{v} => v&broadcast{T2, 0xFFFFFFFF}, rp}}, any{tupsel{0,bad}|tupsel{1,bad}}} this doesn't use M
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ def swap{w,x} = {
|
||||
equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = {
|
||||
def bulk = 256 / width{X}
|
||||
if (W!=X) if (d!=0) swap{w,x}
|
||||
|
||||
if (W==u1) {
|
||||
if (X==u1) { # bitarr ≡ bitarr
|
||||
maskedLoop{256, l, {i, M} => {
|
||||
|
||||
@ -6,7 +6,6 @@ include './mask'
|
||||
include './bitops'
|
||||
|
||||
|
||||
|
||||
def bitspread{T==[32]u8, a:u32} = {
|
||||
def idxs = iota{32}
|
||||
b:= broadcast{[8]u32, a}
|
||||
@ -21,10 +20,9 @@ def bitspread{T, a & vcount{T} <= width{eltype{T}} & w256u{T}} = {
|
||||
}
|
||||
|
||||
|
||||
|
||||
bitexpand{T}(xp: *u64, rp:*eltype{T}, l:u64) : void = {
|
||||
def bulk = vcount{T}
|
||||
def TU = ty_vu{T}
|
||||
def TU = ty_u{T}
|
||||
|
||||
maskedLoop{vcount{T}, l, {i, M} => {
|
||||
x:= b_getBatch{bulk, xp, i} # TODO unroll f64 by two to make b_getBatch not sad
|
||||
|
||||
@ -72,7 +72,7 @@ def loadBatch{ptr:P, n, T} = {
|
||||
|
||||
def maskedLoop{bulk, l, step} = {
|
||||
m:u64 = l/bulk
|
||||
@for (i from 0 to m) step{i, maskNone}
|
||||
@for (i to m) step{i, maskNone}
|
||||
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) step{m, maskAfter{left}}
|
||||
|
||||
@ -13,6 +13,7 @@ avx2_bcs32(x:*u64, r:*i32, l:u64) : void = {
|
||||
def sums{n} = (if (n==0) tup{0}; else { def s=sums{n-1}; merge{s,s+1} })
|
||||
def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}}
|
||||
def widen{v:T} = unpackQ{shuf{[4]u64, v, 4b3120}, broadcast{T, 0}}
|
||||
|
||||
def step{x:u32, i, store1} = {
|
||||
b:= broadcast{[8]u32, x} >> make{[8]u32, 4*tail{1, iota{8}}}
|
||||
s:= sel8{[32]u8~~b, ii32>>3 + bit{2}}
|
||||
@ -25,7 +26,7 @@ avx2_bcs32(x:*u64, r:*i32, l:u64) : void = {
|
||||
def out{v, k} = each{out, widen{v}, 2*k+iota{2}}
|
||||
def out{v0:[8]i32, k} = {
|
||||
v := [8]u32~~v0 + c
|
||||
if (tail{1,k}) c = sel{[8]u32, v, make{[8]i32, each{{i}=>7,iota{8}}}}
|
||||
if (tail{1,k}) c = sel{[8]u32, v, make{[8]i32, broadcast{8, 7}}}
|
||||
store1{rv, j+k, v}
|
||||
}
|
||||
out{[32]i8~~d, 0}
|
||||
|
||||
@ -39,7 +39,7 @@ select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
|
||||
cw0:= loadBatch{w, i, TIF}
|
||||
cw1:= cw0+xlf
|
||||
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
|
||||
if (any{M{ty_vu{cw} >= ty_vu{xlf}}}) return{0}
|
||||
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
got:= gather{broadcast{TDF,0}, x, cw, M}
|
||||
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
|
||||
storeBatch{r, i, got, M}
|
||||
|
||||
@ -20,6 +20,7 @@ def v2i{x:T & w128{T}} = [16]u8 ~~ x # for compact casting for the annoying intr
|
||||
def v2f{x:T & w128{T}} = [4]f32 ~~ x
|
||||
def v2d{x:T & w128{T}} = [2]f64 ~~ x
|
||||
|
||||
|
||||
# load & store
|
||||
def load {a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si128', a+n}
|
||||
def loada{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_load_si128', a+n}
|
||||
@ -35,16 +36,13 @@ def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', *f32 ~
|
||||
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', ext{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',ext{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
|
||||
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
|
||||
|
||||
|
||||
|
||||
|
||||
# signed comparison
|
||||
def __eq{a:T,b:T & T==[16]i8 } = emit{[16]u8, '_mm_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & T==[ 8]i16} = emit{[ 8]u16, '_mm_cmpeq_epi16', a, b}
|
||||
@ -66,10 +64,10 @@ def __gt{a:T,b:T & w128u{T}} = ~(a<=b)
|
||||
# rest of comparison
|
||||
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
|
||||
|
||||
|
||||
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
||||
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
||||
|
||||
|
||||
# mask stuff
|
||||
def getmask{x:T & w128{T, 8}} = emit{u32, '_mm_movemask_epi8', x}
|
||||
def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, broadcast{[8]u16, 0}}}
|
||||
@ -89,6 +87,8 @@ def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16
|
||||
def printGen{x, s, n} = apply{emit, merge{tup{void, 'printf', merge{'"', apply{merge,each{{c}=>{if(c>0) merge{',',s}; else s}, iota{n}}}, '"'}}, each{{c}=>extract{x,c}, iota{n}}}}
|
||||
def print{x:T & isvec{T}} = printGen{x, merge{'%', tern{width{eltype{T}}==64, 'l', ''}, tern{issigned{x}, 'd', 'u'}}, vcount{T}}
|
||||
|
||||
|
||||
|
||||
### SSE4.1; TODO either move to own file or rename this file
|
||||
|
||||
# rest of extract
|
||||
|
||||
Loading…
Reference in New Issue
Block a user