diff --git a/build/singeliSubmodule b/build/singeliSubmodule index ac9e7b75..49a6a90d 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit ac9e7b7517a8b84aeebecfa360f0694629f83ec0 +Subproject commit 49a6a90d83992171a2db749e9f7fd400ec65ef2c diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 3e7f07e2..c3c645b0 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -6,19 +6,6 @@ def v2d{x:T & w256{T}} = [4]f64 ~~ x def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store -def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n} -def loada{a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_load_si256', a+n} -def load {a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_loadu_pd', *f64 ~~ (a+n)} -def loada{a:T, n & w256f{eltype{T},64}} = emit{eltype{T}, '_mm256_load_pd', *f64 ~~ (a+n)} -def load {a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_loadu_ps', *f32 ~~ (a+n)} -def loada{a:T, n & w256f{eltype{T},32}} = emit{eltype{T}, '_mm256_load_ps', *f32 ~~ (a+n)} -def store {a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_storeu_si256', a+n, v} -def storea{a:T, n, v & w256i{eltype{T} }} = emit{void, '_mm256_store_si256', a+n, v} -def store {a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_storeu_pd', *f64 ~~ (a+n), v} -def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f64 ~~ (a+n), v} -def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v} -def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v} - def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr} @@ -26,69 +13,15 @@ def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}} def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x} -# broadcast -def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 32}} = emit{T, '_mm256_set1_epi32', promote{eltype{T},v}} -def broadcast{T, v & w256i{T, 64}} = emit{T, '_mm256_set1_epi64x',promote{eltype{T},v}} -def broadcast{T, v & w256f{T, 64}} = emit{T, '_mm256_set1_pd', v} -def broadcast{T, v & w256f{T, 32}} = emit{T, '_mm256_set1_ps', v} - -local def makeGen{T,s,x} = emit{T, s, ...each{promote{eltype{T}, .}, x}} -def make{T, ...xs & w256f{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_pd', xs} -def make{T, ...xs & w256f{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_ps', xs} -def make{T, ...xs & w256i{T,64} & tuplen{xs}== 4} = makeGen{T, '_mm256_setr_epi64x', xs} -def make{T, ...xs & w256i{T,32} & tuplen{xs}== 8} = makeGen{T, '_mm256_setr_epi32', xs} -def make{T, ...xs & w256i{T,16} & tuplen{xs}==16} = makeGen{T, '_mm256_setr_epi16', xs} -def make{T, ...xs & w256i{T, 8} & tuplen{xs}==32} = makeGen{T, '_mm256_setr_epi8', xs} -def make{T, x & w256{T} & istup{x}} = make{T, ...x} - -def iota{T & w256{T}} = make{T, ...iota{vcount{T}}} - -# bit arith -def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b}} -def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}} -def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}} -def __not{a:T & w256u{T}} = a ^ (T ** ~cast{eltype{T},0}) -def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}} - # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n} -def __eq{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 0}; def __eq{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 0} -def __ne{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b, 4}; def __ne{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b, 4} -def __gt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,30}; def __gt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,30} -def __ge{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,29}; def __ge{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,29} -def __lt{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,17}; def __lt{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,17} -def __le{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,18}; def __le{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,18} def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3} def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3} # f32 arith -def __add{a:T,b:T & T==[8]f32} = emit{T, '_mm256_add_ps', a, b} -def __sub{a:T,b:T & T==[8]f32} = emit{T, '_mm256_sub_ps', a, b} -def __mul{a:T,b:T & T==[8]f32} = emit{T, '_mm256_mul_ps', a, b} -def __div{a:T,b:T & T==[8]f32} = emit{T, '_mm256_div_ps', a, b} -def max{a:T,b:T & T==[8]f32} = emit{T, '_mm256_max_ps', a, b} -def min{a:T,b:T & T==[8]f32} = emit{T, '_mm256_min_ps', a, b} -def sqrt{a:[8]f32} = emit{[8]f32, '_mm256_sqrt_ps', a} def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a} def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a} -def abs{a:[8]f32} = emit{[8]f32, '_mm256_and_ps', a, v2f{[8]u32 ** 0x7FFFFFFF}} -def floor{a:[8]f32} = emit{[8]f32, '_mm256_floor_ps', a} -def ceil{a:[8]f32} = emit{[8]f32, '_mm256_ceil_ps', a} - -# f64 arith -def __add{a:T,b:T & T==[4]f64} = emit{T, '_mm256_add_pd', a, b} -def __sub{a:T,b:T & T==[4]f64} = emit{T, '_mm256_sub_pd', a, b} -def __mul{a:T,b:T & T==[4]f64} = emit{T, '_mm256_mul_pd', a, b} -def __div{a:T,b:T & T==[4]f64} = emit{T, '_mm256_div_pd', a, b} -def max{a:T,b:T & T==[4]f64} = emit{T, '_mm256_max_pd', a, b} -def min{a:T,b:T & T==[4]f64} = emit{T, '_mm256_min_pd', a, b} -def sqrt{a:[4]f64} = emit{[4]f64, '_mm256_sqrt_pd', a} -def abs{a:[4]f64} = emit{[4]f64, '_mm256_and_pd', a, v2d{[4]u64 ** ((cast{u64,1}<<63)-1)}} -def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a} -def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} @@ -102,19 +35,6 @@ def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} # structural operations -def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i} -def extract{x:T, i & w256i{T,64} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi64', x, i} -def extract{x:T, i & w256f{T,64} & knum{i}} = emit{f64, '_mm256_cvtsd_f64', emit{T, '_mm256_permute4x64_pd', x, i}} -def extract{x:T, i & w256f{T,32} & knum{i}} = { - t:= emit{[4]f32, '_mm256_extractf128_ps', x, i>>2} - emit{f32, '_mm_cvtss_f32', emit{[4]f32, '_mm_shuffle_ps', t, t, i}} -} - -def insert{x:T, i, v & w256i{T, 8} & knum{i}} = emit{T, '_mm256_insert_epi8', x, v, i} -def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x, v, i} -def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i} -def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i} - def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}} def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index d1f1f30a..6b157ad9 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,58 +1,3 @@ -# min & max -def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b} -def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b} -def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b} -def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b} -def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b} -def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b} - -# signed comparison -def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b} -def __eq{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b} -def __gt{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpgt_epi8', a, b} -def __gt{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpgt_epi16', a, b} -def __gt{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpgt_epi32', a, b} -def __gt{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpgt_epi64', a, b} -def __lt{a:T,b:T & w256s{T}} = b>a -def __ge{a:T,b:T & w256s{T}} = ~(b>a) -def __le{a:T,b:T & w256s{T}} = ~(a>b) -# unsigned comparison -def __eq{a:T,b:T & T==[32]u8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b} -def __eq{a:T,b:T & T==[16]u16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b} -def __eq{a:T,b:T & T==[ 8]u32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b} -def __eq{a:T,b:T & T==[ 4]u64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b} -def __le{a:T,b:T & w256u{T}} = a==min{a,b} -def __ge{a:T,b:T & w256u{T}} = a==max{a,b} -def __lt{a:T,b:T & w256u{T}} = ~(a>=b) -def __gt{a:T,b:T & w256u{T}} = ~(a<=b) -# rest of comparison -def __ne{a:T,b:T & w256i{T}} = ~(b==a) - -# cast-less bitwise fns -def __xor{a:T, b:T & w256i{T}} = emit{T, '_mm256_xor_si256', a, b} -def __and{a:T, b:T & w256i{T}} = emit{T, '_mm256_and_si256', a, b} -def __or {a:T, b:T & w256i{T}} = emit{T, '_mm256_or_si256', a, b} -def andnot{a:T, b:T & w256i{T}} = emit{T, '_mm256_andnot_si256', b, a} - -# shift -def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b} -def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b} -def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b} -def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b} -def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b} -def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b} -def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b} -def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b} -# no 64-bit arithmetic shift :/ -def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b} -def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b} -def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b} -def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b} -def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b} -# still no 64-bit arithmetic shift :/ - # questionable pack def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}} def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}} @@ -69,30 +14,11 @@ def packQQ{{a, b}} = packQQ{a, b} # arith -def __mul{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_mullo_epi16', a, b} def mulHi{a:T,b:T & [16]i16==T } = emit{T, '_mm256_mulhi_epi16', a, b} def mulHi{a:T,b:T & [16]u16==T } = emit{T, '_mm256_mulhi_epu16', a, b} -def __mul{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_mullo_epi32', a, b} def mul32{a:T,b:T & [ 4]i64==T } = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments def mul32{a:T,b:T & [ 4]u64==T } = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments -def abs{a:T & w256i{T,8 }} = emit{T, '_mm256_abs_epi8', a} -def abs{a:T & w256i{T,16}} = emit{T, '_mm256_abs_epi16', a} -def abs{a:T & w256i{T,32}} = emit{T, '_mm256_abs_epi32', a} -def absu{a:T & w256i{T}} = ty_u{abs{a}} - -def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}; def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b} -def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}; def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b} -def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b} -def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b} - -# saturating add/sub -def adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b} -def subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} -def adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} -def subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b} - - # structural operations def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} @@ -113,9 +39,6 @@ def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_perm def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i} def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i} -def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} -def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} - # masked store; F variants may not be a single instruction def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v} def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 6ba188ec..17eaf98e 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -1,5 +1,6 @@ include 'skin/c' include 'arch/c' +include 'util/for' include 'util/kind' def ux = u64 @@ -21,6 +22,9 @@ def isprim{T} = 0; def isprim{T & istype{T}} = same{typekind{T},'primitive'} def isptr {T} = 0; def isptr {T & istype{T}} = same{typekind{T},'pointer'} def elwidth{T} = width{eltype{T}} +oper &~ andnot infix none 35 +def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b + def load{p:P, n & isvec{eltype{P}}} = assert{0} @@ -40,37 +44,6 @@ def exportN{f, ...ns} = each{export{.,f}, ns} def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} } -# more arith -def cdiv{a,b} = (a+b-1)/b # ceiling divide -def cdiv{a,b & knum{a} & knum{b}} = ((a+b-1)/b)>>0 -def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x} -def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x} -def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x} -def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x} -def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x} -def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x} -# count-leading-zeros complement, less type-dependent -def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} -def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} - -def ceil_log2{n} = clzc{n-1} - -def truncBits{n, v & n<=8} = cast_i{u8, v} -def truncBits{n, v & n==16} = cast_i{u16, v} -def truncBits{n, v & n==32} = cast_i{u32, v} -def truncBits{n, v & n==64} = cast_i{u64, v} - -# base-2 log of a constant power of two -def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 -def lb{n==1} = 0 - -def zlow{n,x} = (x >> n) << n # zero out n least significant bits -def tail{n,x} = x & ((1<v, range{n}} +def collect{vars,begin,end,iter & knum{begin} & knum{end}} = { + each{iter{., vars}, range{end-begin}+begin} +} + +# convert tuple to number in little-endian base b +def base{b,l} = if (0==tuplen{l}) 0 else tupsel{0,l}+b*base{b,slice{l,1}} + + + +# vector definitions +def arch_defvw = if (hasarch{'AVX2'}) 256 else 128 +def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} + +# test if vector has a specific width & element type +def lvec{T, n, w} = 0 +def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 + # base cases def { - abs,andAllZero,andnz,b_getBatch,clmul,cvt,extract,floor,fold_addw,half, + absu,andAllZero,andnz,b_getBatch,clmul,cvt,extract,fold_addw,half, homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow, @@ -184,20 +172,21 @@ def { zip,zipHi,zipLo } -def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}} -def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b} -def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} -def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b -oper &~ andnot infix none 35 -def widen{T, x:X & T==X} = x -def narrow{T, x:X & T==eltype{X}} = x -def undefPromote{T, x:X & T==X} = T~~x -def packQ{{a, b}} = packQ{a, b} -def pair{{a, b}} = pair{a, b} -def cvt{T, x:X & T==eltype{X}} = x +def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times +def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX -def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +if (hasarch{'X86_64'}) { + include 'arch/iintrinsic/basic' + include './sse2' + include './sse' + include './avx' + include './avx2' +} else if (hasarch{'AARCH64'}) { + include 'arch/neon_intrin/basic' + include './neon' +} else { + def {__adds,__subs,__sqrt,vec_broadcast,vec_make} +} def homMask{...vs & tuplen{vs}>1} = { def n = tuplen{vs} @@ -205,9 +194,64 @@ def homMask{...vs & tuplen{vs}>1} = { def RT = ty_u{max{8,vcount{T}*n}} tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}} } -def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times -def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX +def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}} +def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b} +def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} +def packQ{{a, b}} = packQ{a, b} +def pair{{a, b}} = pair{a, b} +def widen{T, x:X & T==X} = x +def narrow{T, x:X & T==eltype{X}} = x +def undefPromote{T, x:X & T==X} = T~~x +def cvt{T, x:X & T==eltype{X}} = x + +def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} +def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} +def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} +def absu{a:T & isvec{T}} = ty_u{abs{a}} + +def floor = __floor +def ceil = __ceil +def abs = __abs +def min = __min +def max = __max +def adds = __adds +def subs = __subs +def sqrt = __sqrt + + + +# more arith +def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +def cdiv{a,b} = (a+b-1)/b # ceiling divide +def cdiv{a,b & knum{a} & knum{b}} = ceil{a/b} +def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x} +def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x} +def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x} +def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x} +def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x} +def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x} +# count-leading-zeros complement, less type-dependent +def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} +def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} + +def ceil_log2{n} = clzc{n-1} + +def truncBits{n, v & n<=8} = cast_i{u8, v} +def truncBits{n, v & n==16} = cast_i{u16, v} +def truncBits{n, v & n==32} = cast_i{u32, v} +def truncBits{n, v & n==64} = cast_i{u64, v} + +# base-2 log of a constant power of two +def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 +def lb{n==1} = 0 + +def zlow{n,x} = (x >> n) << n # zero out n least significant bits +def tail{n,x} = x & ((1<v, range{n}} - -def arch_defvw = tern{hasarch{'AVX2'}, 256, 128} -def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} - def load{p,i & kgen{p}} = p{i} def store{p,i,x & kgen{p}} = p{i,x} @@ -240,21 +273,7 @@ def tptr{l,s} = { # create "pointer" generator with given load & store definitio def r{i,x} = s{i,x} } -def unroll{vars,begin,end,iter & knum{begin} & knum{end}} = { - def f{i,l & i==l} = 0 - def f{i,l & i!=l} = { - iter{i, vars} - f{i+1, l} - } - f{begin,end} -} -def for{vars,begin,end,iter} = { - i:ux = begin - while (i < end) { - iter{i, vars} - ++i - } -} +def unroll = for_const def forNZ{vars,begin,end,iter} = { i:ux = begin assert{i!=end} @@ -263,13 +282,6 @@ def forNZ{vars,begin,end,iter} = { ++i } } -def for_backwards{vars,begin,end,iter} = { - i:ux = end - while (i > begin) { - --i - iter{i, vars} - } -} def forUnroll{exp,unr}{vars,begin,end,iter} = { i:ux = begin while ((i+unr) <= end) { @@ -309,8 +321,8 @@ def tree_fold{F, x} = { def tree_fold{F, {x}} = x def eachx{F, ...args} = { - def l = tree_fold{max, each{{x} => {if(ktup{x}) tuplen{x}; else 0}, args}} - each{F, ...each{{x} => {if (istup{x}) x; else l**x}, args}} + def l = tree_fold{max, each{{x} => if(ktup{x}) tuplen{x} else 0, args}} + each{F, ...each{{x} => if (istup{x}) x else l**x, args}} } def undef{T, n & istype{T}} = @collect(n) undef{T} diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index b4af2a70..c0e5ed4c 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,12 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse' -} include './mask' include 'util/tup' diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 9950b4ff..e3198fc6 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -1,15 +1,6 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} +include './f64' include './bitops' include './mask' @@ -28,4 +19,4 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}} -exportT{'simd_bitsel', table{arch_defvw}} \ No newline at end of file +exportT{'simd_bitsel', table{arch_defvw}} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index c014bbff..21cf4150 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -1,15 +1,6 @@ include './base' include './cbqnDefs' include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './bitops' diff --git a/src/singeli/src/constrep.singeli b/src/singeli/src/constrep.singeli index 45dc95a6..0b612860 100644 --- a/src/singeli/src/constrep.singeli +++ b/src/singeli/src/constrep.singeli @@ -1,11 +1,4 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include 'util/tup' def incl{a,b} = slice{iota{b+1},a} @@ -225,4 +218,4 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { } } -exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}} \ No newline at end of file +exportT{'simd_constrep', each{rep_const, tup{i8, i16, i32, u64}}} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 1538154e..a029b44f 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -1,13 +1,4 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './cbqnDefs' include './bitops' @@ -104,4 +95,4 @@ gen{'simd_copy_'} export{'simd_copy_1u_i8', copy_ubit{i8}} export{'simd_copy_1u_i16', copy_ubit{i16}} export{'simd_copy_1u_i32', copy_ubit{i32}} -export{'simd_copy_1u_f64', copy_ubit{f64}} \ No newline at end of file +export{'simd_copy_1u_f64', copy_ubit{f64}} diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index d7ae54c3..90c068a0 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -1,7 +1,4 @@ include './base' -include './sse' -include './avx' -include './avx2' include 'util/tup' include './vecfold' diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 80679a2d..eea7d28e 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -1,15 +1,6 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} +include './f64' include './bitops' include './mask' include 'util/tup' diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 0a8afb45..ade621f3 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -1,15 +1,6 @@ include './base' include './cbqnDefs' include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './bitops' diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 79f70053..f513360d 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,13 +1,4 @@ include './base' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}} diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 2ef1b720..ecdee8a3 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -1,15 +1,6 @@ include './base' -include './f64' include './cbqnDefs' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'X86_64'}) { - include './sse2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} +include './f64' include './bitops' include './mask' @@ -24,4 +15,4 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { len } -exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}} \ No newline at end of file +exportT{'simd_abs', each{absFn, tup{i8, i16, i32, f64}}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index f8f7ff3a..a6a812c2 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -10,46 +10,21 @@ def nvecf = genchk{nvec, isfloat} def reinterpret{T, v & same{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v } -def nty{T== u8} = 'u8'; def nty{T== i8} = 's8' -def nty{T==u16} = 'u16'; def nty{T==i16} = 's16' -def nty{T==u32} = 'u32'; def nty{T==i32} = 's32' -def nty{T==u64} = 'u64'; def nty{T==i64} = 's64' -def nty{T==f32} = 'f32'; def nty{T==f64} = 'f64' +def nty{T} = { + def q = quality{T} + merge{if (q=='i') 's' else q, fmtnat{width{T}}} +} def nty{T & isvec{T}} = nty{eltype{T}} def ntyp{S, ...S2, T & w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} def ntyp{S, ...S2, T & w64{T}} = merge{S, ...S2, '_', nty{T}} def ntyp0{S, T} = merge{S, '_', nty{T}} -def load{a:T, n & nvec{eltype{T}}} = emit{eltype{T}, ntyp{'vld1', eltype{T}}, *eltype{eltype{T}} ~~ (a+n)} -def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a+n), v} - -def __or{a:T,b:T & nvecf{T}} = T~~ __or{ty_u{a}, ty_u{b}} -def __and{a:T,b:T & nvecf{T}} = T~~__and{ty_u{a}, ty_u{b}} -def __xor{a:T,b:T & nvecf{T}} = T~~__xor{ty_u{a}, ty_u{b}} -def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b} -def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b} -def __mul{a:T,b:T & nvec {T}} = emit{T, ntyp{'vmul', T}, a, b} -def __div{a:T,b:T & nvecf{T}} = emit{T, ntyp{'vdiv', T}, a, b} -def __and{a:T,b:T & nveci{T}} = emit{T, ntyp{'vand', T}, a, b} -def __or{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorr', T}, a, b} -def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b} -def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b} -def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b} -def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b} -def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} -def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} -def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} -def adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} -def subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} -def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b} def addwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b} def subwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b} def mulwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b} def mulwHi{a:T,b:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b} def mulw {a:T,b:T & w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}} -def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s0 & s8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<=vcount{T}} = { truncBits{vcount{T}, fold_add{x & make{T, 1<a -def __ge{a:T,b:T & w128s{T}} = ~(b>a) -def __le{a:T,b:T & w128s{T}} = ~(a>b) -# unsigned comparison -def __lt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} < ty_s{b^s} } -def __gt{a:T,b:T & w128u{T}} = { s:=T**(1<<(elwidth{T}-1)); ty_s{a^s} > ty_s{b^s} } -def __le{a:T,b:T & w128u{T}} = ~(a>b) -def __ge{a:T,b:T & w128u{T}} = ~(a=b) -def __gt{a:T,b:T & w128u{T, 8}} = ~(a<=b) -# rest of integer comparison -def __ne{a:T,b:T & w128i{T}} = ~(b==a) # float comparison -def __eq{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpeq_ps', a, b}; def __eq{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpeq_pd', a, b} -def __ne{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpneq_ps', a, b}; def __ne{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpneq_pd', a, b} -def __gt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpgt_ps', a, b}; def __gt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpgt_pd', a, b} -def __ge{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpge_ps', a, b}; def __ge{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpge_pd', a, b} -def __lt{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmplt_ps', a, b}; def __lt{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmplt_pd', a, b} -def __le{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmple_ps', a, b}; def __le{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmple_pd', a, b} def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} -# bit arith -def __xor{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_xor_ps', v2f{a}, v2f{b}} -def __and{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_and_ps', v2f{a}, v2f{b}} -def __or {a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_or_ps', v2f{a}, v2f{b}} -def __not{a:T & w128u{T}} = a ^ (T ** ~cast{eltype{T},0}) -def andnot{a:T, b:T & w128{T}} = T ~~ emit{[4]f32, '_mm_andnot_ps', v2f{b}, v2f{a}} - # shift -def __shl{a:T,b & w128i{T, 16} & knum{b}} = emit{T, '_mm_slli_epi16', a, b} -def __shl{a:T,b & w128i{T, 32} & knum{b}} = emit{T, '_mm_slli_epi32', a, b} -def __shl{a:T,b & w128i{T, 64} & knum{b}} = emit{T, '_mm_slli_epi64', a, b} -def __shr{a:T,b & w128u{T, 16} & knum{b}} = emit{T, '_mm_srli_epi16', a, b} -def __shr{a:T,b & w128u{T, 32} & knum{b}} = emit{T, '_mm_srli_epi32', a, b} -def __shr{a:T,b & w128u{T, 64} & knum{b}} = emit{T, '_mm_srli_epi64', a, b} -def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b} -def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} -# no 64-bit arithmetic shift :/ - def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} # integer arith -def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b} -def __add{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_add_epi16', a, b} -def __add{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_add_epi32', a, b} -def __add{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_add_epi64', a, b} - -def __sub{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_sub_epi8', a, b} -def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b} -def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b} -def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} - -def __mul{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_mullo_epi16', a, b} def mulHi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} def mulHi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments @@ -131,36 +39,9 @@ def __mul{a:T,b:T & [4]i32==T} = { [4]i32~~mzipLo{...sw{4b3120, lo, hi}} } -def min{a:T,b:T & w128i{T}} = homBlend{a, b, a>b} -def max{a:T,b:T & w128i{T}} = homBlend{a, b, a>31; (t^a) - t } -def absu{a:T & w128i{T}} = ty_u{abs{a}} - -# saturating -def adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b} -def subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b} -def adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b} -def subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b} - # float arith -def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b} -def __sub{a:T,b:T & T==[4]f32} = emit{T, '_mm_sub_ps', a, b}; def __sub{a:T,b:T & T==[2]f64} = emit{T, '_mm_sub_pd', a, b} -def __mul{a:T,b:T & T==[4]f32} = emit{T, '_mm_mul_ps', a, b}; def __mul{a:T,b:T & T==[2]f64} = emit{T, '_mm_mul_pd', a, b} -def __div{a:T,b:T & T==[4]f32} = emit{T, '_mm_div_ps', a, b}; def __div{a:T,b:T & T==[2]f64} = emit{T, '_mm_div_pd', a, b} -def max{a:T,b:T & T==[4]f32} = emit{T, '_mm_max_ps', a, b}; def max{a:T,b:T & T==[2]f64} = emit{T, '_mm_max_pd', a, b} -def min{a:T,b:T & T==[4]f32} = emit{T, '_mm_min_ps', a, b}; def min{a:T,b:T & T==[2]f64} = emit{T, '_mm_min_pd', a, b} -def sqrt{a:[4]f32} = emit{[4]f32, '_mm_sqrt_ps', a}; def sqrt{a:[2]f64} = emit{[2]f64, '_mm_sqrt_pd', a} def rsqrtE{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a} def rcpE{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a} -def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}} -def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}} # mask stuff def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0} @@ -196,11 +77,6 @@ def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}} def unpackQ{a:T, b:T & w128{T}} = mzip{a, b} -def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} -def extract{x:T, i==0 & w128i{T,32}} = emit{eltype{T}, '_mm_cvtsi128_si32', x} -def extract{x:T, i==0 & w128i{T,64}} = emit{eltype{T}, '_mm_cvtsi128_si64', x} -def extract{x:T, i==0 & w128f{T,64}} = emit{eltype{T}, '_mm_cvtsd_f64', x} - def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n} def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index d1ea188b..08dbc977 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -1,13 +1,6 @@ include './base' include './cbqnDefs' include './f64' -if (hasarch{'AVX2'}) { - include './sse' - include './avx' - include './avx2' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './bitops'