Merge pull request #92 from mlochbaum/iintrinsic
Built-in basic vector includes
This commit is contained in:
commit
6a0805eb9d
@ -1 +1 @@
|
||||
Subproject commit ac9e7b7517a8b84aeebecfa360f0694629f83ec0
|
||||
Subproject commit 49a6a90d83992171a2db749e9f7fd400ec65ef2c
|
||||
@ -6,19 +6,6 @@ def v2d{x:T & w256{T}} = [4]f64 ~~ x
|
||||
def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
|
||||
|
||||
# load & store
|
||||
def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n}
|
||||
def loada{a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_load_si256', a+n}
|
||||
def load {a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_loadu_pd', *f64 ~~ (a+n)}
|
||||
def loada{a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_load_pd', *f64 ~~ (a+n)}
|
||||
def load {a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_loadu_ps', *f32 ~~ (a+n)}
|
||||
def loada{a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_load_ps', *f32 ~~ (a+n)}
|
||||
def store {a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_storeu_si256', a+n, v}
|
||||
def storea{a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_store_si256', a+n, v}
|
||||
def store {a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_storeu_pd', *f64 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f64 ~~ (a+n), v}
|
||||
def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}}
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr}
|
||||
|
||||
@ -26,69 +13,15 @@ def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x}
|
||||
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256f{T, 64}} = emit{T, '_mm256_set1_pd', v}
|
||||
def broadcast{T, v & w256f{T, 32}} = emit{T, '_mm256_set1_ps', v}
|
||||
|
||||
local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T}, .}, x}}
|
||||
def make{T, ...xs & w256f{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_pd', xs}
|
||||
def make{T, ...xs & w256f{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_ps', xs}
|
||||
def make{T, ...xs & w256i{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_epi64x', xs}
|
||||
def make{T, ...xs & w256i{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_epi32', xs}
|
||||
def make{T, ...xs & w256i{T,16} & tuplen{xs}==16} = makeGen{T, '_mm256_setr_epi16', xs}
|
||||
def make{T, ...xs & w256i{T, 8} & tuplen{xs}==32} = makeGen{T, '_mm256_setr_epi8', xs}
|
||||
def make{T, x & w256{T} & istup{x}} = make{T, ...x}
|
||||
|
||||
def iota{T & w256{T}} = make{T, ...iota{vcount{T}}}
|
||||
|
||||
# bit arith
|
||||
def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b}}
|
||||
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
|
||||
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
|
||||
def __not{a:T & w256u{T}} = a ^ (T ** ~cast{eltype{T},0})
|
||||
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
|
||||
|
||||
# float comparison
|
||||
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
||||
local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
|
||||
def __eq{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 0}; def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0}
|
||||
def __ne{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 4}; def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4}
|
||||
def __gt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,30}; def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30}
|
||||
def __ge{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,29}; def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29}
|
||||
def __lt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,17}; def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17}
|
||||
def __le{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,18}; def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18}
|
||||
def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3}
|
||||
def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3}
|
||||
|
||||
# f32 arith
|
||||
def __add{a:T,b:T & T==[8]f32} = emit{T, '_mm256_add_ps', a, b}
|
||||
def __sub{a:T,b:T & T==[8]f32} = emit{T, '_mm256_sub_ps', a, b}
|
||||
def __mul{a:T,b:T & T==[8]f32} = emit{T, '_mm256_mul_ps', a, b}
|
||||
def __div{a:T,b:T & T==[8]f32} = emit{T, '_mm256_div_ps', a, b}
|
||||
def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b}
|
||||
def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b}
|
||||
def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a}
|
||||
def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
|
||||
def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
|
||||
def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}}
|
||||
def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a}
|
||||
def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a}
|
||||
|
||||
# f64 arith
|
||||
def __add{a:T,b:T & T==[4]f64} = emit{T, '_mm256_add_pd', a, b}
|
||||
def __sub{a:T,b:T & T==[4]f64} = emit{T, '_mm256_sub_pd', a, b}
|
||||
def __mul{a:T,b:T & T==[4]f64} = emit{T, '_mm256_mul_pd', a, b}
|
||||
def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b}
|
||||
def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b}
|
||||
def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b}
|
||||
def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a}
|
||||
def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}}
|
||||
def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
|
||||
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
|
||||
|
||||
# conversion
|
||||
def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
|
||||
@ -102,19 +35,6 @@ def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
|
||||
|
||||
|
||||
# structural operations
|
||||
def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i}
|
||||
def extract{x:T, i & w256i{T,64} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi64', x, i}
|
||||
def extract{x:T, i & w256f{T,64} & knum{i}} = emit{f64, '_mm256_cvtsd_f64', emit{T, '_mm256_permute4x64_pd', x, i}}
|
||||
def extract{x:T, i & w256f{T,32} & knum{i}} = {
|
||||
t:= emit{[4]f32, '_mm256_extractf128_ps', x, i>>2}
|
||||
emit{f32, '_mm_cvtss_f32', emit{[4]f32, '_mm_shuffle_ps', t, t, i}}
|
||||
}
|
||||
|
||||
def insert{x:T, i, v & w256i{T, 8} & knum{i}} = emit{T, '_mm256_insert_epi8', x, v, i}
|
||||
def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x, v, i}
|
||||
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
|
||||
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
|
||||
|
||||
def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}
|
||||
|
||||
@ -1,58 +1,3 @@
|
||||
# min & max
|
||||
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}
|
||||
def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b}
|
||||
def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b}
|
||||
def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b}
|
||||
def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b}
|
||||
|
||||
# signed comparison
|
||||
def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
|
||||
def __eq{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
|
||||
def __gt{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpgt_epi8', a, b}
|
||||
def __gt{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpgt_epi16', a, b}
|
||||
def __gt{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpgt_epi32', a, b}
|
||||
def __gt{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpgt_epi64', a, b}
|
||||
def __lt{a:T,b:T & w256s{T}} = b>a
|
||||
def __ge{a:T,b:T & w256s{T}} = ~(b>a)
|
||||
def __le{a:T,b:T & w256s{T}} = ~(a>b)
|
||||
# unsigned comparison
|
||||
def __eq{a:T,b:T & T==[32]u8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & T==[16]u16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & T==[ 8]u32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
|
||||
def __eq{a:T,b:T & T==[ 4]u64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
|
||||
def __le{a:T,b:T & w256u{T}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w256u{T}} = a==max{a,b}
|
||||
def __lt{a:T,b:T & w256u{T}} = ~(a>=b)
|
||||
def __gt{a:T,b:T & w256u{T}} = ~(a<=b)
|
||||
# rest of comparison
|
||||
def __ne{a:T,b:T & w256i{T}} = ~(b==a)
|
||||
|
||||
# cast-less bitwise fns
|
||||
def __xor{a:T, b:T & w256i{T}} = emit{T, '_mm256_xor_si256', a, b}
|
||||
def __and{a:T, b:T & w256i{T}} = emit{T, '_mm256_and_si256', a, b}
|
||||
def __or {a:T, b:T & w256i{T}} = emit{T, '_mm256_or_si256', a, b}
|
||||
def andnot{a:T, b:T & w256i{T}} = emit{T, '_mm256_andnot_si256', b, a}
|
||||
|
||||
# shift
|
||||
def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b}
|
||||
def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b}
|
||||
def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b}
|
||||
def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b}
|
||||
def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b}
|
||||
def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b}
|
||||
def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b}
|
||||
def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b}
|
||||
# no 64-bit arithmetic shift :/
|
||||
def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b}
|
||||
def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b}
|
||||
def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b}
|
||||
def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b}
|
||||
def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b}
|
||||
# still no 64-bit arithmetic shift :/
|
||||
|
||||
# questionable pack
|
||||
def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
|
||||
def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
|
||||
@ -69,30 +14,11 @@ def packQQ{{a, b}} = packQQ{a, b}
|
||||
|
||||
|
||||
# arith
|
||||
def __mul{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_mullo_epi16', a, b}
|
||||
def mulHi{a:T,b:T & [16]i16==T } = emit{T, '_mm256_mulhi_epi16', a, b}
|
||||
def mulHi{a:T,b:T & [16]u16==T } = emit{T, '_mm256_mulhi_epu16', a, b}
|
||||
def __mul{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_mullo_epi32', a, b}
|
||||
def mul32{a:T,b:T & [ 4]i64==T } = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
|
||||
def mul32{a:T,b:T & [ 4]u64==T } = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
|
||||
|
||||
def abs{a:T & w256i{T,8 }} = emit{T, '_mm256_abs_epi8', a}
|
||||
def abs{a:T & w256i{T,16}} = emit{T, '_mm256_abs_epi16', a}
|
||||
def abs{a:T & w256i{T,32}} = emit{T, '_mm256_abs_epi32', a}
|
||||
def absu{a:T & w256i{T}} = ty_u{abs{a}}
|
||||
|
||||
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
|
||||
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
|
||||
|
||||
# saturating add/sub
|
||||
def adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
|
||||
def subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
||||
def adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
|
||||
def subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
|
||||
|
||||
|
||||
# structural operations
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
@ -113,9 +39,6 @@ def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_perm
|
||||
def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
|
||||
def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
|
||||
|
||||
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
|
||||
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
|
||||
|
||||
# masked store; F variants may not be a single instruction
|
||||
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
|
||||
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
include 'skin/c'
|
||||
include 'arch/c'
|
||||
include 'util/for'
|
||||
include 'util/kind'
|
||||
|
||||
def ux = u64
|
||||
@ -21,6 +22,9 @@ def isprim{T} = 0; def isprim{T & istype{T}} = same{typekind{T},'primitive'}
|
||||
def isptr {T} = 0; def isptr {T & istype{T}} = same{typekind{T},'pointer'}
|
||||
def elwidth{T} = width{eltype{T}}
|
||||
|
||||
oper &~ andnot infix none 35
|
||||
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
|
||||
|
||||
|
||||
|
||||
def load{p:P, n & isvec{eltype{P}}} = assert{0}
|
||||
@ -40,37 +44,6 @@ def exportN{f, ...ns} = each{export{.,f}, ns}
|
||||
def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} }
|
||||
|
||||
|
||||
# more arith
|
||||
def cdiv{a,b} = (a+b-1)/b # ceiling divide
|
||||
def cdiv{a,b & knum{a} & knum{b}} = ((a+b-1)/b)>>0
|
||||
def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x}
|
||||
def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x}
|
||||
def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x}
|
||||
def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x}
|
||||
def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x}
|
||||
def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x}
|
||||
# count-leading-zeros complement, less type-dependent
|
||||
def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x}
|
||||
def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x}
|
||||
|
||||
def ceil_log2{n} = clzc{n-1}
|
||||
|
||||
def truncBits{n, v & n<=8} = cast_i{u8, v}
|
||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||
|
||||
# base-2 log of a constant power of two
|
||||
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
||||
def lb{n==1} = 0
|
||||
|
||||
def zlow{n,x} = (x >> n) << n # zero out n least significant bits
|
||||
def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
|
||||
def bit {k,x} = x & (1<<k) # get the k-th bit
|
||||
|
||||
# Convert tuple to number in little-endian base b
|
||||
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
|
||||
|
||||
# hints
|
||||
def rare{x & knum{x}} = x
|
||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||
@ -118,8 +91,6 @@ def w256f = genchk{w256, isfloat}; def w128f = genchk{w128, isfloat}; de
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
|
||||
def trunc{T, x & knum{x}} = cast{T, x}
|
||||
|
||||
@ -161,22 +132,39 @@ def el_h{T & isvec{T}} = [vcount{T}](w_h{eltype{T}})
|
||||
def el_m{T & isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width
|
||||
def el_s{T & isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# test if vector has a specific width & element type
|
||||
def lvec{T, n, w} = 0
|
||||
def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1
|
||||
# type stats
|
||||
def minvalue{T & isunsigned{T}} = 0
|
||||
def maxvalue{T & isunsigned{T}} = (1<<width{T})-1
|
||||
def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
|
||||
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
|
||||
|
||||
|
||||
|
||||
|
||||
# tuple operations
|
||||
def iota{n & knum{n}} = range{n}
|
||||
def broadcast{T, v & isprim{T}} = v
|
||||
def broadcast{n, v & knum{n}} = each{{_}=>v, range{n}}
|
||||
def collect{vars,begin,end,iter & knum{begin} & knum{end}} = {
|
||||
each{iter{., vars}, range{end-begin}+begin}
|
||||
}
|
||||
|
||||
# convert tuple to number in little-endian base b
|
||||
def base{b,l} = if (0==tuplen{l}) 0 else tupsel{0,l}+b*base{b,slice{l,1}}
|
||||
|
||||
|
||||
|
||||
# vector definitions
|
||||
def arch_defvw = if (hasarch{'AVX2'}) 256 else 128
|
||||
def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'}
|
||||
|
||||
# test if vector has a specific width & element type
|
||||
def lvec{T, n, w} = 0
|
||||
def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1
|
||||
|
||||
# base cases
|
||||
def {
|
||||
abs,andAllZero,andnz,b_getBatch,clmul,cvt,extract,floor,fold_addw,half,
|
||||
absu,andAllZero,andnz,b_getBatch,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep,
|
||||
pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow,
|
||||
@ -184,20 +172,21 @@ def {
|
||||
zip,zipHi,zipLo
|
||||
}
|
||||
|
||||
def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}}
|
||||
def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b}
|
||||
def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b}
|
||||
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
|
||||
oper &~ andnot infix none 35
|
||||
def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def packQ{{a, b}} = packQ{a, b}
|
||||
def pair{{a, b}} = pair{a, b}
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||
|
||||
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
|
||||
if (hasarch{'X86_64'}) {
|
||||
include 'arch/iintrinsic/basic'
|
||||
include './sse2'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include 'arch/neon_intrin/basic'
|
||||
include './neon'
|
||||
} else {
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
|
||||
}
|
||||
|
||||
def homMask{...vs & tuplen{vs}>1} = {
|
||||
def n = tuplen{vs}
|
||||
@ -205,9 +194,64 @@ def homMask{...vs & tuplen{vs}>1} = {
|
||||
def RT = ty_u{max{8,vcount{T}*n}}
|
||||
tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}}
|
||||
}
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
|
||||
def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}}
|
||||
def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b}
|
||||
def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b}
|
||||
def packQ{{a, b}} = packQ{a, b}
|
||||
def pair{{a, b}} = pair{a, b}
|
||||
|
||||
def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
def undefPromote{T, x:X & T==X} = T~~x
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
|
||||
def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}}
|
||||
def make{T, ...xs & isvec{T}} = vec_make{T, ...xs}
|
||||
def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}}
|
||||
def absu{a:T & isvec{T}} = ty_u{abs{a}}
|
||||
|
||||
def floor = __floor
|
||||
def ceil = __ceil
|
||||
def abs = __abs
|
||||
def min = __min
|
||||
def max = __max
|
||||
def adds = __adds
|
||||
def subs = __subs
|
||||
def sqrt = __sqrt
|
||||
|
||||
|
||||
|
||||
# more arith
|
||||
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
|
||||
def cdiv{a,b} = (a+b-1)/b # ceiling divide
|
||||
def cdiv{a,b & knum{a} & knum{b}} = ceil{a/b}
|
||||
def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x}
|
||||
def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x}
|
||||
def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x}
|
||||
def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x}
|
||||
def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x}
|
||||
def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x}
|
||||
# count-leading-zeros complement, less type-dependent
|
||||
def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x}
|
||||
def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x}
|
||||
|
||||
def ceil_log2{n} = clzc{n-1}
|
||||
|
||||
def truncBits{n, v & n<=8} = cast_i{u8, v}
|
||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||
|
||||
# base-2 log of a constant power of two
|
||||
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
||||
def lb{n==1} = 0
|
||||
|
||||
def zlow{n,x} = (x >> n) << n # zero out n least significant bits
|
||||
def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
|
||||
def bit {k,x} = x & (1<<k) # get the k-th bit
|
||||
|
||||
# range logic
|
||||
def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count)
|
||||
def TU = ty_u{TS}
|
||||
(TU~~(x-TS**start)) < TU**count
|
||||
@ -221,17 +265,6 @@ def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} # ∊ [star
|
||||
def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} # ∊ [start;end)
|
||||
|
||||
|
||||
# tuple operations
|
||||
def broadcast{T, v & isprim{T}} = v
|
||||
def iota{n & knum{n}} = range{n}
|
||||
def collect{vars,begin,end,iter & knum{begin} & knum{end}} = {
|
||||
each{iter{., vars}, range{end-begin}+begin}
|
||||
}
|
||||
def broadcast{n, v & knum{n}} = each{{_}=>v, range{n}}
|
||||
|
||||
def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}
|
||||
def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'}
|
||||
|
||||
|
||||
def load{p,i & kgen{p}} = p{i}
|
||||
def store{p,i,x & kgen{p}} = p{i,x}
|
||||
@ -240,21 +273,7 @@ def tptr{l,s} = { # create "pointer" generator with given load & store definitio
|
||||
def r{i,x} = s{i,x}
|
||||
}
|
||||
|
||||
def unroll{vars,begin,end,iter & knum{begin} & knum{end}} = {
|
||||
def f{i,l & i==l} = 0
|
||||
def f{i,l & i!=l} = {
|
||||
iter{i, vars}
|
||||
f{i+1, l}
|
||||
}
|
||||
f{begin,end}
|
||||
}
|
||||
def for{vars,begin,end,iter} = {
|
||||
i:ux = begin
|
||||
while (i < end) {
|
||||
iter{i, vars}
|
||||
++i
|
||||
}
|
||||
}
|
||||
def unroll = for_const
|
||||
def forNZ{vars,begin,end,iter} = {
|
||||
i:ux = begin
|
||||
assert{i!=end}
|
||||
@ -263,13 +282,6 @@ def forNZ{vars,begin,end,iter} = {
|
||||
++i
|
||||
}
|
||||
}
|
||||
def for_backwards{vars,begin,end,iter} = {
|
||||
i:ux = end
|
||||
while (i > begin) {
|
||||
--i
|
||||
iter{i, vars}
|
||||
}
|
||||
}
|
||||
def forUnroll{exp,unr}{vars,begin,end,iter} = {
|
||||
i:ux = begin
|
||||
while ((i+unr) <= end) {
|
||||
@ -309,8 +321,8 @@ def tree_fold{F, x} = {
|
||||
def tree_fold{F, {x}} = x
|
||||
|
||||
def eachx{F, ...args} = {
|
||||
def l = tree_fold{max, each{{x} => {if(ktup{x}) tuplen{x}; else 0}, args}}
|
||||
each{F, ...each{{x} => {if (istup{x}) x; else l**x}, args}}
|
||||
def l = tree_fold{max, each{{x} => if(ktup{x}) tuplen{x} else 0, args}}
|
||||
each{F, ...each{{x} => if (istup{x}) x else l**x, args}}
|
||||
}
|
||||
|
||||
def undef{T, n & istype{T}} = @collect(n) undef{T}
|
||||
|
||||
@ -1,12 +1,5 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse'
|
||||
}
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './f64'
|
||||
include './bitops'
|
||||
include './mask'
|
||||
|
||||
@ -28,4 +19,4 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
||||
|
||||
def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}}
|
||||
|
||||
exportT{'simd_bitsel', table{arch_defvw}}
|
||||
exportT{'simd_bitsel', table{arch_defvw}}
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './bitops'
|
||||
|
||||
|
||||
|
||||
@ -1,11 +1,4 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
def incl{a,b} = slice{iota{b+1},a}
|
||||
@ -225,4 +218,4 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}}
|
||||
exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}}
|
||||
|
||||
@ -1,13 +1,4 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include './cbqnDefs'
|
||||
include './bitops'
|
||||
@ -104,4 +95,4 @@ gen{'simd_copy_'}
|
||||
export{'simd_copy_1u_i8', copy_ubit{i8}}
|
||||
export{'simd_copy_1u_i16', copy_ubit{i16}}
|
||||
export{'simd_copy_1u_i32', copy_ubit{i32}}
|
||||
export{'simd_copy_1u_f64', copy_ubit{f64}}
|
||||
export{'simd_copy_1u_f64', copy_ubit{f64}}
|
||||
|
||||
@ -1,7 +1,4 @@
|
||||
include './base'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include 'util/tup'
|
||||
include './vecfold'
|
||||
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './f64'
|
||||
include './bitops'
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include './bitops'
|
||||
|
||||
|
||||
@ -1,13 +1,4 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
|
||||
def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
|
||||
|
||||
@ -1,15 +1,6 @@
|
||||
include './base'
|
||||
include './f64'
|
||||
include './cbqnDefs'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './f64'
|
||||
include './bitops'
|
||||
include './mask'
|
||||
|
||||
@ -24,4 +15,4 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
len
|
||||
}
|
||||
|
||||
exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}}
|
||||
exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}}
|
||||
|
||||
@ -10,46 +10,21 @@ def nvecf = genchk{nvec, isfloat}
|
||||
|
||||
def reinterpret{T, v & same{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v }
|
||||
|
||||
def nty{T== u8} = 'u8'; def nty{T== i8} = 's8'
|
||||
def nty{T==u16} = 'u16'; def nty{T==i16} = 's16'
|
||||
def nty{T==u32} = 'u32'; def nty{T==i32} = 's32'
|
||||
def nty{T==u64} = 'u64'; def nty{T==i64} = 's64'
|
||||
def nty{T==f32} = 'f32'; def nty{T==f64} = 'f64'
|
||||
def nty{T} = {
|
||||
def q = quality{T}
|
||||
merge{if (q=='i') 's' else q, fmtnat{width{T}}}
|
||||
}
|
||||
def nty{T & isvec{T}} = nty{eltype{T}}
|
||||
def ntyp{S, ...S2, T & w128{T}} = merge{S, 'q', ...S2, '_', nty{T}}
|
||||
def ntyp{S, ...S2, T & w64{T}} = merge{S, ...S2, '_', nty{T}}
|
||||
def ntyp0{S, T} = merge{S, '_', nty{T}}
|
||||
|
||||
def load{a:T, n & nvec{eltype{T}}} = emit{eltype{T}, ntyp{'vld1', eltype{T}}, *eltype{eltype{T}} ~~ (a+n)}
|
||||
def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a+n), v}
|
||||
|
||||
def __or{a:T,b:T & nvecf{T}} = T~~ __or{ty_u{a}, ty_u{b}}
|
||||
def __and{a:T,b:T & nvecf{T}} = T~~__and{ty_u{a}, ty_u{b}}
|
||||
def __xor{a:T,b:T & nvecf{T}} = T~~__xor{ty_u{a}, ty_u{b}}
|
||||
def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b}
|
||||
def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b}
|
||||
def __mul{a:T,b:T & nvec {T}} = emit{T, ntyp{'vmul', T}, a, b}
|
||||
def __div{a:T,b:T & nvecf{T}} = emit{T, ntyp{'vdiv', T}, a, b}
|
||||
def __and{a:T,b:T & nveci{T}} = emit{T, ntyp{'vand', T}, a, b}
|
||||
def __or{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorr', T}, a, b}
|
||||
def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b}
|
||||
def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b}
|
||||
def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b}
|
||||
def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
|
||||
def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b}
|
||||
def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b}
|
||||
def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
|
||||
def adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b}
|
||||
def subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b}
|
||||
def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b}
|
||||
def addwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b}
|
||||
def subwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b}
|
||||
def mulwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b}
|
||||
def mulwHi{a:T,b:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b}
|
||||
def mulw {a:T,b:T & w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}}
|
||||
|
||||
def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshl', '_n', T}, a, s}; def __shl{a:T,s==0 & nveci{T}} = a
|
||||
def __shr{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshr', '_n', T}, a, s}; def __shr{a:T,s==0 & nveci{T}} = a
|
||||
def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
|
||||
def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
|
||||
def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
|
||||
@ -57,13 +32,6 @@ def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a
|
||||
def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M,elwidth{T}} & width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
|
||||
def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m}
|
||||
|
||||
def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a}
|
||||
def __not{a:T & nvecu{T}} = T~~emit{re_el{u8,T}, ntyp{'vmvn', re_el{u8,T}}, a}
|
||||
def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a}
|
||||
def floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a}
|
||||
def ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a}
|
||||
def abs{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vabs', T}, a}
|
||||
def absu{a:T & nveci{T}} = ty_u{abs{a}}
|
||||
def addpw { x:T & nveci{T} & elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
|
||||
def addpwa{a:D, x:T & nveci{T} & elwidth{T}<=32 & D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
|
||||
def mla{a:T, x:T, y:T & nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
|
||||
@ -77,13 +45,6 @@ def popc{x:T & nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
|
||||
def clz{x:T & nvecu{T} & elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
|
||||
def cls{x:T & nveci{T} & elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
|
||||
|
||||
def __eq{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vceq', T}, a, b}
|
||||
def __ge{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcge', T}, a, b}
|
||||
def __gt{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcgt', T}, a, b}
|
||||
def __lt{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vclt', T}, a, b}
|
||||
def __le{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcle', T}, a, b}
|
||||
def __ne{a:T,b:T & nvec{T}} = ~(a==b)
|
||||
|
||||
def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
|
||||
def fold_addw{a:T & nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a}
|
||||
def fold_min {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
|
||||
@ -109,7 +70,6 @@ def undefPromote{T, x:X & w64{X} & w128{T} & eltype{T}==eltype{X}} = emit{T, nty
|
||||
def half{x:T, n==0 & w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x}
|
||||
def half{x:T, n==1 & w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x}
|
||||
def pair{a:T, b:T & w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
|
||||
def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n}
|
||||
def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
|
||||
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
|
||||
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
|
||||
@ -161,16 +121,6 @@ def homAny{x:T & nvec{T}} = bitAny{x}
|
||||
def homAll{x:T & nvec{T}} = bitAll{x}
|
||||
|
||||
|
||||
def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x}
|
||||
|
||||
def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = {
|
||||
def TE = eltype{T}
|
||||
load{*T ~~ *TE ~~ each{promote{eltype{T},.}, xs}, 0}
|
||||
}
|
||||
def make{T, x & nvec{T} & istup{x}} = make{T, ...x}
|
||||
def iota{T & nvec{T}} = make{T, ...iota{vcount{T}}}
|
||||
|
||||
|
||||
def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
|
||||
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
|
||||
}
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
include './base'
|
||||
include './sse'
|
||||
include './clmul'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
include './f64'
|
||||
include './scan_common'
|
||||
|
||||
@ -1,15 +1,4 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'SSSE3'}) {
|
||||
include './sse'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include './vecfold'
|
||||
|
||||
|
||||
@ -1,8 +1,5 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
include './bitops'
|
||||
include 'util/tup'
|
||||
|
||||
@ -1,21 +1,7 @@
|
||||
include './base'
|
||||
if (hasarch{'X86_64'}) include './sse'
|
||||
if (hasarch{'PCLMUL'}) include './clmul'
|
||||
if (hasarch{'AVX2'}) { include './avx'; include './avx2' }
|
||||
if (hasarch{'BMI2'}) include './bmi2'
|
||||
if (hasarch{'AARCH64'}) include './neon'
|
||||
if (hasarch{'AVX512F'}) {
|
||||
local def mti{s,T} = merge{'_mm512_',s,'_epi',fmtnat{elwidth{T}}}
|
||||
def load{a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n}
|
||||
def make{T, xs & 512==width{T} & tuplen{xs}==vcount{T}} = {
|
||||
def p = each{promote{eltype{T},.}, reverse{xs}}
|
||||
emit{T, mti{'set',T}, ...p}
|
||||
}
|
||||
def iota{T & isvec{T} & 512==width{T}} = make{T, iota{vcount{T}}}
|
||||
def broadcast{T, v & isvec{T} & 512==width{T}} = {
|
||||
emit{T, mti{'set1',T}, promote{eltype{T},v}}
|
||||
}
|
||||
def __add{a:T,b:T & 512==width{T}} = emit{T, mti{'add',T}, a, b}
|
||||
if (hasarch{'X86_64'}) {
|
||||
if (hasarch{'PCLMUL'}) include './clmul'
|
||||
if (hasarch{'BMI2'}) include './bmi2'
|
||||
}
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
@ -1,13 +1,4 @@
|
||||
include './base'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include './cbqnDefs'
|
||||
include 'util/tup'
|
||||
|
||||
@ -1,47 +1,29 @@
|
||||
include './sse2'
|
||||
|
||||
### SSSE3 ###
|
||||
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
def vshl{a:T, b:T, n} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
|
||||
def sel{L, x:T, i:I & hasarch{'SSSE3'} & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
def vshl{a:T, b:T, n & hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
|
||||
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
|
||||
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
|
||||
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
|
||||
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
# arith
|
||||
def min{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi8', a, b}
|
||||
def min{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi32', a, b}
|
||||
def min{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu16', a, b}
|
||||
def min{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu32', a, b}
|
||||
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
|
||||
|
||||
def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b}
|
||||
|
||||
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
|
||||
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
|
||||
def packs{a:T,b:T & hasarch{'SSE4.1'} & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
|
||||
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
|
||||
# conversion
|
||||
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
|
||||
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
|
||||
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
|
||||
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
|
||||
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
|
||||
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
|
||||
def widen{T==[8]u16, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
|
||||
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
|
||||
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
|
||||
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
|
||||
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
|
||||
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
|
||||
def widen{T==[2]f64, x:X & hasarch{'SSE4.1'} & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
|
||||
|
||||
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
# assumes all bits are the same in each mask item
|
||||
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
|
||||
@ -49,4 +31,3 @@ def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlen
|
||||
|
||||
|
||||
### SSE4.2 ###
|
||||
def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b}
|
||||
|
||||
@ -5,19 +5,6 @@ def v2d{x:T & w128{T}} = [2]f64 ~~ x
|
||||
|
||||
|
||||
# load & store
|
||||
def load {a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si128', a+n}
|
||||
def loada{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_load_si128', a+n}
|
||||
def load {a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_loadu_pd', *f64 ~~ (a+n)}
|
||||
def loada{a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_load_pd', *f64 ~~ (a+n)}
|
||||
def load {a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_loadu_ps', *f32 ~~ (a+n)}
|
||||
def loada{a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_load_ps', *f32 ~~ (a+n)}
|
||||
def store {a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si128', a+n, v}
|
||||
def storea{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_store_si128', a+n, v}
|
||||
def store {a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_storeu_pd', *f64 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_store_pd', *f64 ~~ (a+n), v}
|
||||
def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', *f32 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
|
||||
@ -30,96 +17,17 @@ def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
|
||||
|
||||
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
|
||||
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
|
||||
|
||||
# make from elements
|
||||
local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T},.}, x}}
|
||||
def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs}
|
||||
def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs}
|
||||
def make{T, ...xs & w128i{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_set_epi64x', tup{tupsel{1,xs}, tupsel{0,xs}}}
|
||||
def make{T, ...xs & w128i{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_epi32', xs}
|
||||
def make{T, ...xs & w128i{T,16} & tuplen{xs}== 8} = makeGen{T, '_mm_setr_epi16', xs}
|
||||
def make{T, ...xs & w128i{T, 8} & tuplen{xs}==16} = makeGen{T, '_mm_setr_epi8', xs}
|
||||
def make{T, x & w128{T} & istup{x}} = make{T, ...x}
|
||||
|
||||
def iota{T & w128{T}} = make{T, ...iota{vcount{T}}}
|
||||
|
||||
|
||||
|
||||
# integer comparison
|
||||
def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b}
|
||||
def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) }
|
||||
# signed comparison
|
||||
def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b}
|
||||
def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b}
|
||||
def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b}
|
||||
def __lt{a:T,b:T & w128s{T}} = b>a
|
||||
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
|
||||
def __le{a:T,b:T & w128s{T}} = ~(a>b)
|
||||
# unsigned comparison
|
||||
def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{b^s} }
|
||||
def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{b^s} }
|
||||
def __le{a:T,b:T & w128u{T}} = ~(a>b)
|
||||
def __ge{a:T,b:T & w128u{T}} = ~(a<b)
|
||||
def __le{a:T,b:T & w128u{T, 8}} = a==min{a,b}
|
||||
def __ge{a:T,b:T & w128u{T, 8}} = a==max{a,b}
|
||||
def __lt{a:T,b:T & w128u{T, 8}} = ~(a>=b)
|
||||
def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b)
|
||||
# rest of integer comparison
|
||||
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
|
||||
# float comparison
|
||||
def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b}
|
||||
def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b}
|
||||
def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b}
|
||||
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
|
||||
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
|
||||
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
|
||||
def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
|
||||
def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
|
||||
|
||||
|
||||
|
||||
# bit arith
|
||||
def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
|
||||
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
|
||||
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
|
||||
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
|
||||
def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}}
|
||||
|
||||
# shift
|
||||
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
|
||||
def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b}
|
||||
def __shl{a:T,b & w128i{T, 64} & knum{b}} = emit{T, '_mm_slli_epi64', a, b}
|
||||
def __shr{a:T,b & w128u{T, 16} & knum{b}} = emit{T, '_mm_srli_epi16', a, b}
|
||||
def __shr{a:T,b & w128u{T, 32} & knum{b}} = emit{T, '_mm_srli_epi32', a, b}
|
||||
def __shr{a:T,b & w128u{T, 64} & knum{b}} = emit{T, '_mm_srli_epi64', a, b}
|
||||
def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b}
|
||||
def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
|
||||
# no 64-bit arithmetic shift :/
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
|
||||
|
||||
# integer arith
|
||||
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
|
||||
def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b}
|
||||
def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b}
|
||||
def __add{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_add_epi64', a, b}
|
||||
|
||||
def __sub{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_sub_epi8', a, b}
|
||||
def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b}
|
||||
def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b}
|
||||
def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
|
||||
|
||||
def __mul{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_mullo_epi16', a, b}
|
||||
def mulHi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
|
||||
def mulHi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
|
||||
def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
|
||||
@ -131,36 +39,9 @@ def __mul{a:T,b:T & [4]i32==T} = {
|
||||
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
|
||||
}
|
||||
|
||||
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
|
||||
def max{a:T,b:T & w128i{T}} = homBlend{a, b, a<b}
|
||||
def max{a:T,b:T & w128u{T,16}} = subs{a,b}+b
|
||||
def min{a:T,b:T & w128u{T,16}} = a-subs{a,b}
|
||||
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
|
||||
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
|
||||
|
||||
def abs{a:T & w128i{T,8 }} = { def U=[16]u8; T~~min{U~~a, U**0 - U~~a} }
|
||||
def abs{a:T & w128i{T,16}} = max{a, T**0 - a}
|
||||
def abs{a:T & w128i{T,32}} = { t:= a>>31; (t^a) - t }
|
||||
def absu{a:T & w128i{T}} = ty_u{abs{a}}
|
||||
|
||||
# saturating
|
||||
def adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b}
|
||||
def subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b}
|
||||
def adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b}
|
||||
def subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b}
|
||||
|
||||
# float arith
|
||||
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
|
||||
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
|
||||
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
|
||||
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
|
||||
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
|
||||
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
|
||||
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
|
||||
def rsqrtE{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
|
||||
def rcpE{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
|
||||
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
|
||||
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0}
|
||||
@ -196,11 +77,6 @@ def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
|
||||
|
||||
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
|
||||
|
||||
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
|
||||
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
|
||||
def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
|
||||
def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x}
|
||||
|
||||
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
|
||||
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
|
||||
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
|
||||
|
||||
@ -1,13 +1,6 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
include './mask'
|
||||
include './bitops'
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user