diff --git a/build/singeliSubmodule b/build/singeliSubmodule index ac9e7b75..eccae1e0 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit ac9e7b7517a8b84aeebecfa360f0694629f83ec0 +Subproject commit eccae1e072b0b16d663d7ac243916604de366160 diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 3e7f07e2..c3c645b0 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -6,19 +6,6 @@ def v2d{x:T & w256{T}} = [4]f64 ~~ x def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store -def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n} -def loada{a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_load_si256', a+n} -def load {a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_loadu_pd', *f64 ~~ (a+n)} -def loada{a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_load_pd', *f64 ~~ (a+n)} -def load {a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_loadu_ps', *f32 ~~ (a+n)} -def loada{a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_load_ps', *f32 ~~ (a+n)} -def store {a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_storeu_si256', a+n, v} -def storea{a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_store_si256', a+n, v} -def store {a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_storeu_pd', *f64 ~~ (a+n), v} -def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f64 ~~ (a+n), v} -def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v} -def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v} - def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr} @@ -26,69 +13,15 @@ def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}} def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x} -# broadcast -def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',promote{eltype{T},v}} -def broadcast{T, v & w256f{T, 64}} = emit{T, '_mm256_set1_pd', v} -def broadcast{T, v & w256f{T, 32}} = emit{T, '_mm256_set1_ps', v} - -local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T}, .}, x}} -def make{T, ...xs & w256f{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_pd', xs} -def make{T, ...xs & w256f{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_ps', xs} -def make{T, ...xs & w256i{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_epi64x', xs} -def make{T, ...xs & w256i{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_epi32', xs} -def make{T, ...xs & w256i{T,16} & tuplen{xs}==16} = makeGen{T, '_mm256_setr_epi16', xs} -def make{T, ...xs & w256i{T, 8} & tuplen{xs}==32} = makeGen{T, '_mm256_setr_epi8', xs} -def make{T, x & w256{T} & istup{x}} = make{T, ...x} - -def iota{T & w256{T}} = make{T, ...iota{vcount{T}}} - -# bit arith -def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b}} -def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}} -def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}} -def __not{a:T & w256u{T}} = a ^ (T ** ~cast{eltype{T},0}) -def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}} - # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n} -def __eq{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 0}; def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0} -def __ne{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 4}; def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4} -def __gt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,30}; def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30} -def __ge{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,29}; def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29} -def __lt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,17}; def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17} -def __le{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,18}; def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18} def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3} def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3} # f32 arith -def __add{a:T,b:T & T==[8]f32} = emit{T, '_mm256_add_ps', a, b} -def __sub{a:T,b:T & T==[8]f32} = emit{T, '_mm256_sub_ps', a, b} -def __mul{a:T,b:T & T==[8]f32} = emit{T, '_mm256_mul_ps', a, b} -def __div{a:T,b:T & T==[8]f32} = emit{T, '_mm256_div_ps', a, b} -def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b} -def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b} -def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a} def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a} def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a} -def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}} -def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a} -def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a} - -# f64 arith -def __add{a:T,b:T & T==[4]f64} = emit{T, '_mm256_add_pd', a, b} -def __sub{a:T,b:T & T==[4]f64} = emit{T, '_mm256_sub_pd', a, b} -def __mul{a:T,b:T & T==[4]f64} = emit{T, '_mm256_mul_pd', a, b} -def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b} -def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b} -def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b} -def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a} -def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}} -def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a} -def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} @@ -102,19 +35,6 @@ def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} # structural operations -def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i} -def extract{x:T, i & w256i{T,64} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi64', x, i} -def extract{x:T, i & w256f{T,64} & knum{i}} = emit{f64, '_mm256_cvtsd_f64', emit{T, '_mm256_permute4x64_pd', x, i}} -def extract{x:T, i & w256f{T,32} & knum{i}} = { - t:= emit{[4]f32, '_mm256_extractf128_ps', x, i>>2} - emit{f32, '_mm_cvtss_f32', emit{[4]f32, '_mm_shuffle_ps', t, t, i}} -} - -def insert{x:T, i, v & w256i{T, 8} & knum{i}} = emit{T, '_mm256_insert_epi8', x, v, i} -def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x, v, i} -def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i} -def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i} - def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}} def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index d1f1f30a..6b157ad9 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,58 +1,3 @@ -# min & max -def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b} -def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b} -def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b} -def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b} -def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b} -def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b} - -# signed comparison -def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b} -def __eq{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b} -def __gt{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpgt_epi8', a, b} -def __gt{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpgt_epi16', a, b} -def __gt{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpgt_epi32', a, b} -def __gt{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpgt_epi64', a, b} -def __lt{a:T,b:T & w256s{T}} = b>a -def __ge{a:T,b:T & w256s{T}} = ~(b>a) -def __le{a:T,b:T & w256s{T}} = ~(a>b) -# unsigned comparison -def __eq{a:T,b:T & T==[32]u8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[16]u16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 8]u32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b} -def __eq{a:T,b:T & T==[ 4]u64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b} -def __le{a:T,b:T & w256u{T}} = a==min{a,b} -def __ge{a:T,b:T & w256u{T}} = a==max{a,b} -def __lt{a:T,b:T & w256u{T}} = ~(a>=b) -def __gt{a:T,b:T & w256u{T}} = ~(a<=b) -# rest of comparison -def __ne{a:T,b:T & w256i{T}} = ~(b==a) - -# cast-less bitwise fns -def __xor{a:T, b:T & w256i{T}} = emit{T, '_mm256_xor_si256', a, b} -def __and{a:T, b:T & w256i{T}} = emit{T, '_mm256_and_si256', a, b} -def __or {a:T, b:T & w256i{T}} = emit{T, '_mm256_or_si256', a, b} -def andnot{a:T, b:T & w256i{T}} = emit{T, '_mm256_andnot_si256', b, a} - -# shift -def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b} -def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b} -def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b} -def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b} -def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b} -def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b} -def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b} -def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b} -# no 64-bit arithmetic shift :/ -def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b} -def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b} -def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b} -def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b} -def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b} -# still no 64-bit arithmetic shift :/ - # questionable pack def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}} def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}} @@ -69,30 +14,11 @@ def packQQ{{a, b}} = packQQ{a, b} # arith -def __mul{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_mullo_epi16', a, b} def mulHi{a:T,b:T & [16]i16==T } = emit{T, '_mm256_mulhi_epi16', a, b} def mulHi{a:T,b:T & [16]u16==T } = emit{T, '_mm256_mulhi_epu16', a, b} -def __mul{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_mullo_epi32', a, b} def mul32{a:T,b:T & [ 4]i64==T } = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments def mul32{a:T,b:T & [ 4]u64==T } = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments -def abs{a:T & w256i{T,8 }} = emit{T, '_mm256_abs_epi8', a} -def abs{a:T & w256i{T,16}} = emit{T, '_mm256_abs_epi16', a} -def abs{a:T & w256i{T,32}} = emit{T, '_mm256_abs_epi32', a} -def absu{a:T & w256i{T}} = ty_u{abs{a}} - -def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b} -def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b} -def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b} -def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b} - -# saturating add/sub -def adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b} -def subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} -def adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} -def subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b} - - # structural operations def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} @@ -113,9 +39,6 @@ def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_perm def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i} def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i} -def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} -def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} - # masked store; F variants may not be a single instruction def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v} def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 6ba188ec..daa1d1a5 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -176,7 +176,7 @@ def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1 # base cases def { - abs,andAllZero,andnz,b_getBatch,clmul,cvt,extract,floor,fold_addw,half, + andAllZero,andnz,b_getBatch,clmul,cvt,fold_addw,half, homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow, @@ -196,8 +196,10 @@ def packQ{{a, b}} = packQ{a, b} def pair{{a, b}} = pair{a, b} def cvt{T, x:X & T==eltype{X}} = x -def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +def min{...x} = __min{...x} +def max{...x} = __max{...x} +def __min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} def homMask{...vs & tuplen{vs}>1} = { def n = tuplen{vs} diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index b4af2a70..a7d140b3 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,11 +1,7 @@ include './base' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse' +if (hasarch{'X86_64'}) { + include './x86_vec' } include './mask' include 'util/tup' diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 9950b4ff..8c3d5dd1 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -1,15 +1,11 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './bitops' include './mask' @@ -28,4 +24,4 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}} -exportT{'simd_bitsel', table{arch_defvw}} \ No newline at end of file +exportT{'simd_bitsel', table{arch_defvw}} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index c014bbff..8c6b6f86 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -1,15 +1,11 @@ include './base' include './cbqnDefs' -include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './bitops' diff --git a/src/singeli/src/constrep.singeli b/src/singeli/src/constrep.singeli index 45dc95a6..2b634fcd 100644 --- a/src/singeli/src/constrep.singeli +++ b/src/singeli/src/constrep.singeli @@ -1,8 +1,6 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } @@ -225,4 +223,4 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { } } -exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}} \ No newline at end of file +exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 1538154e..77f0dfe8 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -1,10 +1,6 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } @@ -104,4 +100,4 @@ gen{'simd_copy_'} export{'simd_copy_1u_i8', copy_ubit{i8}} export{'simd_copy_1u_i16', copy_ubit{i16}} export{'simd_copy_1u_i32', copy_ubit{i32}} -export{'simd_copy_1u_f64', copy_ubit{f64}} \ No newline at end of file +export{'simd_copy_1u_f64', copy_ubit{f64}} diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index d7ae54c3..2a4acb32 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -1,7 +1,5 @@ include './base' -include './sse' -include './avx' -include './avx2' +include './x86_vec' include 'util/tup' include './vecfold' diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 80679a2d..e4adfee7 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -1,15 +1,11 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './bitops' include './mask' include 'util/tup' diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 0a8afb45..4a847a4c 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -1,15 +1,11 @@ include './base' include './cbqnDefs' -include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './mask' include './bitops' diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 79f70053..e8f79cd6 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,10 +1,6 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 2ef1b720..9afcfd75 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -1,15 +1,11 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './bitops' include './mask' @@ -24,4 +20,4 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { len } -exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}} \ No newline at end of file +exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 773e8675..12ea2969 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -1,8 +1,8 @@ include './base' -include './sse' +if (hasarch{'X86_64'}) { + include './x86_vec' +} include './clmul' -include './avx' -include './avx2' include './mask' include './f64' include './scan_common' diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index bc7a0cd8..0900000d 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -1,12 +1,6 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'SSSE3'}) { - include './sse' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index e45180b4..50cfc105 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -1,8 +1,6 @@ include './base' include './cbqnDefs' -include './sse' -include './avx' -include './avx2' +include './x86_vec' include './mask' include './bitops' include 'util/tup' diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 3fce06a5..5e85383b 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -1,21 +1,10 @@ include './base' -if (hasarch{'X86_64'}) include './sse' -if (hasarch{'PCLMUL'}) include './clmul' -if (hasarch{'AVX2'}) { include './avx'; include './avx2' } -if (hasarch{'BMI2'}) include './bmi2' -if (hasarch{'AARCH64'}) include './neon' -if (hasarch{'AVX512F'}) { - local def mti{s,T} = merge{'_mm512_',s,'_epi',fmtnat{elwidth{T}}} - def load{a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n} - def make{T, xs & 512==width{T} & tuplen{xs}==vcount{T}} = { - def p = each{promote{eltype{T},.}, reverse{xs}} - emit{T, mti{'set',T}, ...p} - } - def iota{T & isvec{T} & 512==width{T}} = make{T, iota{vcount{T}}} - def broadcast{T, v & isvec{T} & 512==width{T}} = { - emit{T, mti{'set1',T}, promote{eltype{T},v}} - } - def __add{a:T,b:T & 512==width{T}} = emit{T, mti{'add',T}, a, b} +if (hasarch{'X86_64'}) { + include './x86_vec' + if (hasarch{'PCLMUL'}) include './clmul' + if (hasarch{'BMI2'}) include './bmi2' +} else if (hasarch{'AARCH64'}) { + include './neon' } include './mask' include 'util/tup' diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 1c017e2e..1ed62054 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -1,10 +1,6 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index 145ae2e5..a190880c 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -1,47 +1,29 @@ -include './sse2' - ### SSSE3 ### -def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} -def vshl{a:T, b:T, n} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)} +def sel{L, x:T, i:I & hasarch{'SSSE3'} & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} +def vshl{a:T, b:T, n & hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)} ### SSE4.1 ### -def packs{a:T,b:T & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} -def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8', x, i} -def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i} -def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i} -def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} - -# arith -def min{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi8', a, b} -def min{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi32', a, b} -def min{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu16', a, b} -def min{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu32', a, b} -def __le{a:T,b:T & w128u{T}} = a==min{a,b} -def __ge{a:T,b:T & w128u{T}} = a==max{a,b} - -def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b} - -def floor{a:[4]f32} = emit{[4]f32, '_mm_floor_ps', a}; def ceil{a:[4]f32} = emit{[4]f32, '_mm_ceil_ps', a} -def floor{a:[2]f64} = emit{[2]f64, '_mm_floor_pd', a}; def ceil{a:[2]f64} = emit{[2]f64, '_mm_ceil_pd', a} +def packs{a:T,b:T & hasarch{'SSE4.1'} & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b} +def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} # conversion -def widen{T==[8]u16, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} -def widen{T==[4]u32, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} -def widen{T==[4]u32, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} -def widen{T==[2]u64, x:X & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} -def widen{T==[2]u64, x:X & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} -def widen{T==[2]u64, x:X & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} -def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T==[8]u16, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} +def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} +def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} +def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} +def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} +def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} +def widen{T==[2]f64, x:X & hasarch{'SSE4.1'} & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} -def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} +def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} +def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} # mask stuff -def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} -def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} +def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} +def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} +def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} +def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} # assumes all bits are the same in each mask item def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m} def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m} @@ -49,4 +31,3 @@ def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlen ### SSE4.2 ### -def __gt{a:T,b:T & T==[2]i64} = emit{[2]u64, '_mm_cmpgt_epi64', a, b} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index c9a86a03..7924b840 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -5,19 +5,6 @@ def v2d{x:T & w128{T}} = [2]f64 ~~ x # load & store -def load {a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si128', a+n} -def loada{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_load_si128', a+n} -def load {a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_loadu_pd', *f64 ~~ (a+n)} -def loada{a:T, n & w128f{eltype{T},64}} = emit{eltype{T}, '_mm_load_pd', *f64 ~~ (a+n)} -def load {a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_loadu_ps', *f32 ~~ (a+n)} -def loada{a:T, n & w128f{eltype{T},32}} = emit{eltype{T}, '_mm_load_ps', *f32 ~~ (a+n)} -def store {a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si128', a+n, v} -def storea{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_store_si128', a+n, v} -def store {a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_storeu_pd', *f64 ~~ (a+n), v} -def storea{a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_store_pd', *f64 ~~ (a+n), v} -def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', *f32 ~~ (a+n), v} -def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', *f32 ~~ (a+n), v} - def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr} def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr} def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr} @@ -30,96 +17,17 @@ def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x} -# broadcast -def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}} -def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}} -def broadcast{T, v & w128i{T, 32}} = emit{T, '_mm_set1_epi32', promote{eltype{T},v}} -def broadcast{T, v & w128i{T, 64}} = emit{T, '_mm_set1_epi64x',promote{eltype{T},v}} -def broadcast{T, v & w128f{T, 64}} = emit{T, '_mm_set1_pd', v} -def broadcast{T, v & w128f{T, 32}} = emit{T, '_mm_set1_ps', v} - -# make from elements -local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T},.}, x}} -def make{T, ...xs & w128f{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_setr_pd', xs} -def make{T, ...xs & w128f{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_ps', xs} -def make{T, ...xs & w128i{T,64} & tuplen{xs}== 2} = makeGen{T, '_mm_set_epi64x', tup{tupsel{1,xs}, tupsel{0,xs}}} -def make{T, ...xs & w128i{T,32} & tuplen{xs}== 4} = makeGen{T, '_mm_setr_epi32', xs} -def make{T, ...xs & w128i{T,16} & tuplen{xs}== 8} = makeGen{T, '_mm_setr_epi16', xs} -def make{T, ...xs & w128i{T, 8} & tuplen{xs}==16} = makeGen{T, '_mm_setr_epi8', xs} -def make{T, x & w128{T} & istup{x}} = make{T, ...x} - -def iota{T & w128{T}} = make{T, ...iota{vcount{T}}} - - - -# integer comparison -def __eq{a:T,b:T & w128i{T,8 }} = emit{ty_u{T}, '_mm_cmpeq_epi8', a, b} -def __eq{a:T,b:T & w128i{T,16}} = emit{ty_u{T}, '_mm_cmpeq_epi16', a, b} -def __eq{a:T,b:T & w128i{T,32}} = emit{ty_u{T}, '_mm_cmpeq_epi32', a, b} -def __eq{a:T,b:T & w128i{T,64}} = { t:= [4]i32~~a == [4]i32~~b; [2]u64~~(t & shuf{[4]i32, t, 4b2301}) } -# signed comparison -def __gt{a:T,b:T & w128s{T,8 }} = emit{ty_u{T}, '_mm_cmpgt_epi8', a, b} -def __gt{a:T,b:T & w128s{T,16}} = emit{ty_u{T}, '_mm_cmpgt_epi16', a, b} -def __gt{a:T,b:T & w128s{T,32}} = emit{ty_u{T}, '_mm_cmpgt_epi32', a, b} -def __lt{a:T,b:T & w128s{T}} = b>a -def __ge{a:T,b:T & w128s{T}} = ~(b>a) -def __le{a:T,b:T & w128s{T}} = ~(a>b) -# unsigned comparison -def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{b^s} } -def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{b^s} } -def __le{a:T,b:T & w128u{T}} = ~(a>b) -def __ge{a:T,b:T & w128u{T}} = ~(a=b) -def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b) -# rest of integer comparison -def __ne{a:T,b:T & w128i{T}} = ~(b==a) # float comparison -def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b} -def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b} -def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b} -def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b} -def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b} -def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b} def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} -# bit arith -def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}} -def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} -def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} -def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) -def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}} - # shift -def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} -def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b} -def __shl{a:T,b & w128i{T, 64} & knum{b}} = emit{T, '_mm_slli_epi64', a, b} -def __shr{a:T,b & w128u{T, 16} & knum{b}} = emit{T, '_mm_srli_epi16', a, b} -def __shr{a:T,b & w128u{T, 32} & knum{b}} = emit{T, '_mm_srli_epi32', a, b} -def __shr{a:T,b & w128u{T, 64} & knum{b}} = emit{T, '_mm_srli_epi64', a, b} -def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b} -def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} -# no 64-bit arithmetic shift :/ - def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} # integer arith -def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b} -def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b} -def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b} -def __add{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_add_epi64', a, b} - -def __sub{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_sub_epi8', a, b} -def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b} -def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b} -def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} - -def __mul{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_mullo_epi16', a, b} def mulHi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} def mulHi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments @@ -131,36 +39,9 @@ def __mul{a:T,b:T & [4]i32==T} = { [4]i32~~mzipLo{...sw{4b3120, lo, hi}} } -def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b} -def max{a:T,b:T & w128i{T}} = homBlend{a, b, a>31; (t^a) - t } -def absu{a:T & w128i{T}} = ty_u{abs{a}} - -# saturating -def adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b} -def subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b} -def adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b} -def subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b} - # float arith -def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b} -def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b} -def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b} -def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b} -def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b} -def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b} -def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a} def rsqrtE{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a} def rcpE{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a} -def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}} -def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}} # mask stuff def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0} @@ -196,11 +77,6 @@ def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}} def unpackQ{a:T, b:T & w128{T}} = mzip{a, b} -def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} -def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x} -def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x} -def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x} - def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n} def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index d1ea188b..a03519a3 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -1,13 +1,11 @@ include './base' include './cbqnDefs' -include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' +if (hasarch{'X86_64'}) { + include './x86_vec' } else if (hasarch{'AARCH64'}) { include './neon' } +include './f64' include './mask' include './bitops' diff --git a/src/singeli/src/x86_vec.singeli b/src/singeli/src/x86_vec.singeli new file mode 100644 index 00000000..ecdfe658 --- /dev/null +++ b/src/singeli/src/x86_vec.singeli @@ -0,0 +1,15 @@ +include 'arch/iintrinsic/basic' +def adds = __adds +def subs = __subs +def sqrt = __sqrt +def floor = __floor +def ceil = __ceil +def abs = __abs +def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} +def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} +def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} +def absu{a:T & isvec{T}} = ty_u{abs{a}} +include './sse2' +include './sse' +include './avx' +include './avx2'