Combine x86 vector extensions and use iintrinsic/basic when possible

This commit is contained in:
Marshall Lochbaum 2023-08-26 09:58:08 -04:00
parent ffa4b38512
commit 082f800b3e
23 changed files with 83 additions and 427 deletions

@ -1 +1 @@
Subproject commit ac9e7b7517a8b84aeebecfa360f0694629f83ec0
Subproject commit eccae1e072b0b16d663d7ac243916604de366160

View File

@ -6,19 +6,6 @@ def v2d{x:T & w256{T}} = [4]f64 ~~ x
def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
# load & store
def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n}
def loada{a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_load_si256', a+n}
def load {a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_loadu_pd', *f64 ~~ (a+n)}
def loada{a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_load_pd', *f64 ~~ (a+n)}
def load {a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_loadu_ps', *f32 ~~ (a+n)}
def loada{a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_load_ps', *f32 ~~ (a+n)}
def store {a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_storeu_si256', a+n, v}
def storea{a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_store_si256', a+n, v}
def store {a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_storeu_pd', *f64 ~~ (a+n), v}
def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f64 ~~ (a+n), v}
def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v}
def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v}
def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}}
def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr}
@ -26,69 +13,15 @@ def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x}
# broadcast
def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}}
def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}}
def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', promote{eltype{T},v}}
def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',promote{eltype{T},v}}
def broadcast{T, v & w256f{T, 64}} = emit{T, '_mm256_set1_pd', v}
def broadcast{T, v & w256f{T, 32}} = emit{T, '_mm256_set1_ps', v}
local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T}, .}, x}}
def make{T, ...xs & w256f{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_pd', xs}
def make{T, ...xs & w256f{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_ps', xs}
def make{T, ...xs & w256i{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_epi64x', xs}
def make{T, ...xs & w256i{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_epi32', xs}
def make{T, ...xs & w256i{T,16} & tuplen{xs}==16} = makeGen{T, '_mm256_setr_epi16', xs}
def make{T, ...xs & w256i{T, 8} & tuplen{xs}==32} = makeGen{T, '_mm256_setr_epi8', xs}
def make{T, x & w256{T} & istup{x}} = make{T, ...x}
def iota{T & w256{T}} = make{T, ...iota{vcount{T}}}
# bit arith
def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b}}
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w256u{T}} = a ^ (T ** ~cast{eltype{T},0})
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
def __eq{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 0}; def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0}
def __ne{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 4}; def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4}
def __gt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,30}; def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30}
def __ge{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,29}; def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29}
def __lt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,17}; def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17}
def __le{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,18}; def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18}
def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3}
def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3}
# f32 arith
def __add{a:T,b:T & T==[8]f32} = emit{T, '_mm256_add_ps', a, b}
def __sub{a:T,b:T & T==[8]f32} = emit{T, '_mm256_sub_ps', a, b}
def __mul{a:T,b:T & T==[8]f32} = emit{T, '_mm256_mul_ps', a, b}
def __div{a:T,b:T & T==[8]f32} = emit{T, '_mm256_div_ps', a, b}
def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b}
def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b}
def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a}
def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}}
def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a}
def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a}
# f64 arith
def __add{a:T,b:T & T==[4]f64} = emit{T, '_mm256_add_pd', a, b}
def __sub{a:T,b:T & T==[4]f64} = emit{T, '_mm256_sub_pd', a, b}
def __mul{a:T,b:T & T==[4]f64} = emit{T, '_mm256_mul_pd', a, b}
def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b}
def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b}
def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b}
def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a}
def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}}
def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a}
def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
# conversion
def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
@ -102,19 +35,6 @@ def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
# structural operations
def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i}
def extract{x:T, i & w256i{T,64} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi64', x, i}
def extract{x:T, i & w256f{T,64} & knum{i}} = emit{f64, '_mm256_cvtsd_f64', emit{T, '_mm256_permute4x64_pd', x, i}}
def extract{x:T, i & w256f{T,32} & knum{i}} = {
t:= emit{[4]f32, '_mm256_extractf128_ps', x, i>>2}
emit{f32, '_mm_cvtss_f32', emit{[4]f32, '_mm_shuffle_ps', t, t, i}}
}
def insert{x:T, i, v & w256i{T, 8} & knum{i}} = emit{T, '_mm256_insert_epi8', x, v, i}
def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x, v, i}
def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i}
def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i}
def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}

View File

@ -1,58 +1,3 @@
# min & max
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}
def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b}
def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b}
def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b}
def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b}
# signed comparison
def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
def __eq{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
def __gt{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpgt_epi8', a, b}
def __gt{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpgt_epi16', a, b}
def __gt{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpgt_epi32', a, b}
def __gt{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpgt_epi64', a, b}
def __lt{a:T,b:T & w256s{T}} = b>a
def __ge{a:T,b:T & w256s{T}} = ~(b>a)
def __le{a:T,b:T & w256s{T}} = ~(a>b)
# unsigned comparison
def __eq{a:T,b:T & T==[32]u8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
def __eq{a:T,b:T & T==[16]u16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
def __eq{a:T,b:T & T==[ 8]u32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
def __eq{a:T,b:T & T==[ 4]u64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
def __le{a:T,b:T & w256u{T}} = a==min{a,b}
def __ge{a:T,b:T & w256u{T}} = a==max{a,b}
def __lt{a:T,b:T & w256u{T}} = ~(a>=b)
def __gt{a:T,b:T & w256u{T}} = ~(a<=b)
# rest of comparison
def __ne{a:T,b:T & w256i{T}} = ~(b==a)
# cast-less bitwise fns
def __xor{a:T, b:T & w256i{T}} = emit{T, '_mm256_xor_si256', a, b}
def __and{a:T, b:T & w256i{T}} = emit{T, '_mm256_and_si256', a, b}
def __or {a:T, b:T & w256i{T}} = emit{T, '_mm256_or_si256', a, b}
def andnot{a:T, b:T & w256i{T}} = emit{T, '_mm256_andnot_si256', b, a}
# shift
def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b}
def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b}
def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b}
def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b}
def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b}
def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b}
def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b}
def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b}
# no 64-bit arithmetic shift :/
def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b}
def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b}
def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b}
def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b}
def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b}
# still no 64-bit arithmetic shift :/
# questionable pack
def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
@ -69,30 +14,11 @@ def packQQ{{a, b}} = packQQ{a, b}
# arith
def __mul{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_mullo_epi16', a, b}
def mulHi{a:T,b:T & [16]i16==T } = emit{T, '_mm256_mulhi_epi16', a, b}
def mulHi{a:T,b:T & [16]u16==T } = emit{T, '_mm256_mulhi_epu16', a, b}
def __mul{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_mullo_epi32', a, b}
def mul32{a:T,b:T & [ 4]i64==T } = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
def mul32{a:T,b:T & [ 4]u64==T } = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
def abs{a:T & w256i{T,8 }} = emit{T, '_mm256_abs_epi8', a}
def abs{a:T & w256i{T,16}} = emit{T, '_mm256_abs_epi16', a}
def abs{a:T & w256i{T,32}} = emit{T, '_mm256_abs_epi32', a}
def absu{a:T & w256i{T}} = ty_u{abs{a}}
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
# saturating add/sub
def adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
def subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
def adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
def subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
# structural operations
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
@ -113,9 +39,6 @@ def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_perm
def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
# masked store; F variants may not be a single instruction
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}

View File

@ -176,7 +176,7 @@ def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
# base cases
def {
abs,andAllZero,andnz,b_getBatch,clmul,cvt,extract,floor,fold_addw,half,
andAllZero,andnz,b_getBatch,clmul,cvt,fold_addw,half,
homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep,
pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow,
@ -196,8 +196,10 @@ def packQ{{a, b}} = packQ{a, b}
def pair{{a, b}} = pair{a, b}
def cvt{T, x:X & T==eltype{X}} = x
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
def min{...x} = __min{...x}
def max{...x} = __max{...x}
def __min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
def __max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
def homMask{...vs & tuplen{vs}>1} = {
def n = tuplen{vs}

View File

@ -1,11 +1,7 @@
include './base'
include './cbqnDefs'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse'
if (hasarch{'X86_64'}) {
include './x86_vec'
}
include './mask'
include 'util/tup'

View File

@ -1,15 +1,11 @@
include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './bitops'
include './mask'
@ -28,4 +24,4 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}}
exportT{'simd_bitsel', table{arch_defvw}}
exportT{'simd_bitsel', table{arch_defvw}}

View File

@ -1,15 +1,11 @@
include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './bitops'

View File

@ -1,8 +1,6 @@
include './base'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
@ -225,4 +223,4 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
}
}
exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}}
exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}}

View File

@ -1,10 +1,6 @@
include './base'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
@ -104,4 +100,4 @@ gen{'simd_copy_'}
export{'simd_copy_1u_i8', copy_ubit{i8}}
export{'simd_copy_1u_i16', copy_ubit{i16}}
export{'simd_copy_1u_i32', copy_ubit{i32}}
export{'simd_copy_1u_f64', copy_ubit{f64}}
export{'simd_copy_1u_f64', copy_ubit{f64}}

View File

@ -1,7 +1,5 @@
include './base'
include './sse'
include './avx'
include './avx2'
include './x86_vec'
include 'util/tup'
include './vecfold'

View File

@ -1,15 +1,11 @@
include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './bitops'
include './mask'
include 'util/tup'

View File

@ -1,15 +1,11 @@
include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './mask'
include './bitops'

View File

@ -1,10 +1,6 @@
include './base'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}

View File

@ -1,15 +1,11 @@
include './base'
include './f64'
include './cbqnDefs'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './bitops'
include './mask'
@ -24,4 +20,4 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
len
}
exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}}
exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}}

View File

@ -1,8 +1,8 @@
include './base'
include './sse'
if (hasarch{'X86_64'}) {
include './x86_vec'
}
include './clmul'
include './avx'
include './avx2'
include './mask'
include './f64'
include './scan_common'

View File

@ -1,12 +1,6 @@
include './base'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'SSSE3'}) {
include './sse'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}

View File

@ -1,8 +1,6 @@
include './base'
include './cbqnDefs'
include './sse'
include './avx'
include './avx2'
include './x86_vec'
include './mask'
include './bitops'
include 'util/tup'

View File

@ -1,21 +1,10 @@
include './base'
if (hasarch{'X86_64'}) include './sse'
if (hasarch{'PCLMUL'}) include './clmul'
if (hasarch{'AVX2'}) { include './avx'; include './avx2' }
if (hasarch{'BMI2'}) include './bmi2'
if (hasarch{'AARCH64'}) include './neon'
if (hasarch{'AVX512F'}) {
local def mti{s,T} = merge{'_mm512_',s,'_epi',fmtnat{elwidth{T}}}
def load{a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n}
def make{T, xs & 512==width{T} & tuplen{xs}==vcount{T}} = {
def p = each{promote{eltype{T},.}, reverse{xs}}
emit{T, mti{'set',T}, ...p}
}
def iota{T & isvec{T} & 512==width{T}} = make{T, iota{vcount{T}}}
def broadcast{T, v & isvec{T} & 512==width{T}} = {
emit{T, mti{'set1',T}, promote{eltype{T},v}}
}
def __add{a:T,b:T & 512==width{T}} = emit{T, mti{'add',T}, a, b}
if (hasarch{'X86_64'}) {
include './x86_vec'
if (hasarch{'PCLMUL'}) include './clmul'
if (hasarch{'BMI2'}) include './bmi2'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './mask'
include 'util/tup'

View File

@ -1,10 +1,6 @@
include './base'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'X86_64'}) {
include './sse2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}

View File

@ -1,47 +1,29 @@
include './sse2'
### SSSE3 ###
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:T, b:T, n} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
def sel{L, x:T, i:I & hasarch{'SSSE3'} & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:T, b:T, n & hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
### SSE4.1 ###
def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i}
def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i}
def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i}
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# arith
def min{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi8', a, b}
def min{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi32', a, b}
def min{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu16', a, b}
def min{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu32', a, b}
def __le{a:T,b:T & w128u{T}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T}} = a==max{a,b}
def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b}
def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a}
def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a}
def packs{a:T,b:T & hasarch{'SSE4.1'} & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# conversion
def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[8]u16, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & hasarch{'SSE4.1'} & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
# mask stuff
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
@ -49,4 +31,3 @@ def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlen
### SSE4.2 ###
def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b}

View File

@ -5,19 +5,6 @@ def v2d{x:T & w128{T}} = [2]f64 ~~ x
# load & store
def load {a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si128', a+n}
def loada{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_load_si128', a+n}
def load {a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_loadu_pd', *f64 ~~ (a+n)}
def loada{a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_load_pd', *f64 ~~ (a+n)}
def load {a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_loadu_ps', *f32 ~~ (a+n)}
def loada{a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_load_ps', *f32 ~~ (a+n)}
def store {a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si128', a+n, v}
def storea{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_store_si128', a+n, v}
def store {a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_storeu_pd', *f64 ~~ (a+n), v}
def storea{a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_store_pd', *f64 ~~ (a+n), v}
def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', *f32 ~~ (a+n), v}
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', *f32 ~~ (a+n), v}
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
@ -30,96 +17,17 @@ def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
# broadcast
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', promote{eltype{T},v}}
def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T},v}}
def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v}
def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v}
# make from elements
local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T},.}, x}}
def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs}
def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs}
def make{T, ...xs & w128i{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_set_epi64x', tup{tupsel{1,xs}, tupsel{0,xs}}}
def make{T, ...xs & w128i{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_epi32', xs}
def make{T, ...xs & w128i{T,16} & tuplen{xs}== 8} = makeGen{T, '_mm_setr_epi16', xs}
def make{T, ...xs & w128i{T, 8} & tuplen{xs}==16} = makeGen{T, '_mm_setr_epi8', xs}
def make{T, x & w128{T} & istup{x}} = make{T, ...x}
def iota{T & w128{T}} = make{T, ...iota{vcount{T}}}
# integer comparison
def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b}
def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b}
def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b}
def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) }
# signed comparison
def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b}
def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b}
def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b}
def __lt{a:T,b:T & w128s{T}} = b>a
def __ge{a:T,b:T & w128s{T}} = ~(b>a)
def __le{a:T,b:T & w128s{T}} = ~(a>b)
# unsigned comparison
def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{b^s} }
def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{b^s} }
def __le{a:T,b:T & w128u{T}} = ~(a>b)
def __ge{a:T,b:T & w128u{T}} = ~(a<b)
def __le{a:T,b:T & w128u{T, 8}} = a==min{a,b}
def __ge{a:T,b:T & w128u{T, 8}} = a==max{a,b}
def __lt{a:T,b:T & w128u{T, 8}} = ~(a>=b)
def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b)
# rest of integer comparison
def __ne{a:T,b:T & w128i{T}} = ~(b==a)
# float comparison
def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b}
def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b}
def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b}
def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b}
def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b}
def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b}
def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
# bit arith
def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}}
def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0})
def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}}
# shift
def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b}
def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b}
def __shl{a:T,b & w128i{T, 64} & knum{b}} = emit{T, '_mm_slli_epi64', a, b}
def __shr{a:T,b & w128u{T, 16} & knum{b}} = emit{T, '_mm_srli_epi16', a, b}
def __shr{a:T,b & w128u{T, 32} & knum{b}} = emit{T, '_mm_srli_epi32', a, b}
def __shr{a:T,b & w128u{T, 64} & knum{b}} = emit{T, '_mm_srli_epi64', a, b}
def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b}
def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b}
# no 64-bit arithmetic shift :/
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
# integer arith
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b}
def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b}
def __add{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_add_epi64', a, b}
def __sub{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_sub_epi8', a, b}
def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b}
def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b}
def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
def __mul{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_mullo_epi16', a, b}
def mulHi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def mulHi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
@ -131,36 +39,9 @@ def __mul{a:T,b:T & [4]i32==T} = {
[4]i32~~mzipLo{...sw{4b3120, lo, hi}}
}
def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b}
def max{a:T,b:T & w128i{T}} = homBlend{a, b, a<b}
def max{a:T,b:T & w128u{T,16}} = subs{a,b}+b
def min{a:T,b:T & w128u{T,16}} = a-subs{a,b}
def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b}
def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b}
def abs{a:T & w128i{T,8 }} = { def U=[16]u8; T~~min{U~~a, U**0 - U~~a} }
def abs{a:T & w128i{T,16}} = max{a, T**0 - a}
def abs{a:T & w128i{T,32}} = { t:= a>>31; (t^a) - t }
def absu{a:T & w128i{T}} = ty_u{abs{a}}
# saturating
def adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b}
def subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b}
def adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b}
def subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b}
# float arith
def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}
def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b}
def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b}
def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b}
def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b}
def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b}
def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a}
def rsqrtE{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
def rcpE{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
# mask stuff
def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0}
@ -196,11 +77,6 @@ def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i}
def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x}
def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x}
def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x}
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}

View File

@ -1,13 +1,11 @@
include './base'
include './cbqnDefs'
include './f64'
if (hasarch{'AVX2'}) {
include './sse'
include './avx'
include './avx2'
if (hasarch{'X86_64'}) {
include './x86_vec'
} else if (hasarch{'AARCH64'}) {
include './neon'
}
include './f64'
include './mask'
include './bitops'

View File

@ -0,0 +1,15 @@
include 'arch/iintrinsic/basic'
def adds = __adds
def subs = __subs
def sqrt = __sqrt
def floor = __floor
def ceil = __ceil
def abs = __abs
def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}}
def make{T, ...xs & isvec{T}} = vec_make{T, ...xs}
def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}}
def absu{a:T & isvec{T}} = ty_u{abs{a}}
include './sse2'
include './sse'
include './avx'
include './avx2'