137 lines
8.3 KiB
Plaintext
137 lines
8.3 KiB
Plaintext
# maskstore
|
|
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 32} & w256i{M, 32}} = emit{void, '_mm256_maskstore_epi32', cast_p{i32, a+n}, m, v}
|
|
def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, '_mm256_maskstore_pd', cast_p{f64, a+n}, m, v}
|
|
|
|
# min & max
|
|
def min{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_min_epi8', a, b}; def min{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_min_epu8', a, b}
|
|
def min{a:T,b:T & T==[16]i16} = emit{T, '_mm256_min_epi16', a, b}; def min{a:T,b:T & T==[16]u16} = emit{T, '_mm256_min_epu16', a, b}
|
|
def min{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_min_epi32', a, b}; def min{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_min_epu32', a, b}
|
|
def max{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_max_epi8', a, b}; def max{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_max_epu8', a, b}
|
|
def max{a:T,b:T & T==[16]i16} = emit{T, '_mm256_max_epi16', a, b}; def max{a:T,b:T & T==[16]u16} = emit{T, '_mm256_max_epu16', a, b}
|
|
def max{a:T,b:T & T==[ 8]i32} = emit{T, '_mm256_max_epi32', a, b}; def max{a:T,b:T & T==[ 8]u32} = emit{T, '_mm256_max_epu32', a, b}
|
|
|
|
# signed comparison
|
|
def __eq{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
|
|
def __eq{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
|
|
def __eq{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
|
|
def __eq{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
|
|
def __gt{a:T,b:T & T==[32]i8 } = emit{[32]u8, '_mm256_cmpgt_epi8', a, b}
|
|
def __gt{a:T,b:T & T==[16]i16} = emit{[16]u16, '_mm256_cmpgt_epi16', a, b}
|
|
def __gt{a:T,b:T & T==[ 8]i32} = emit{[ 8]u32, '_mm256_cmpgt_epi32', a, b}
|
|
def __gt{a:T,b:T & T==[ 4]i64} = emit{[ 4]u64, '_mm256_cmpgt_epi64', a, b}
|
|
def __lt{a:T,b:T & w256s{T}} = b>a
|
|
def __ge{a:T,b:T & w256s{T}} = ~(b>a)
|
|
def __le{a:T,b:T & w256s{T}} = ~(a>b)
|
|
# unsigned comparison
|
|
def __eq{a:T,b:T & T==[32]u8 } = emit{[32]u8, '_mm256_cmpeq_epi8', a, b}
|
|
def __eq{a:T,b:T & T==[16]u16} = emit{[16]u16, '_mm256_cmpeq_epi16', a, b}
|
|
def __eq{a:T,b:T & T==[ 8]u32} = emit{[ 8]u32, '_mm256_cmpeq_epi32', a, b}
|
|
def __eq{a:T,b:T & T==[ 4]u64} = emit{[ 4]u64, '_mm256_cmpeq_epi64', a, b}
|
|
def __le{a:T,b:T & w256u{T}} = a==min{a,b}
|
|
def __ge{a:T,b:T & w256u{T}} = a==max{a,b}
|
|
def __lt{a:T,b:T & w256u{T}} = ~(a>=b)
|
|
def __gt{a:T,b:T & w256u{T}} = ~(a<=b)
|
|
# rest of comparison
|
|
def __ne{a:T,b:T & w256i{T}} = ~(b==a)
|
|
|
|
# shift
|
|
def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b}
|
|
def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b}
|
|
def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b}
|
|
def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b}
|
|
def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b}
|
|
def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b}
|
|
def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b}
|
|
def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b}
|
|
# no 64-bit arithmetic shift :/
|
|
def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b}
|
|
def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b}
|
|
def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b}
|
|
def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b}
|
|
def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b}
|
|
# still no 64-bit arithmetic shift :/
|
|
|
|
# questionable pack
|
|
def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
|
|
def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
|
|
def unpackQ{a:[ 8]i32, b:[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
|
|
# inverse of questionable pack; these saturate the argument
|
|
def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
|
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
|
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
|
|
# super questionable pack - takes assumes high halves are zero
|
|
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
|
|
def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}}
|
|
|
|
|
|
# arith
|
|
def __mul {a:T,b:T & [16]i16==T} = emit{T, '_mm256_mullo_epi16', a, b}
|
|
def __mulhi{a:T,b:T & [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b}
|
|
def __mulhi{a:T,b:T & [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b}
|
|
def __mul {a:T,b:T & [ 8]i32==T} = emit{T, '_mm256_mullo_epi32', a, b}
|
|
def __mul32{a:T,b:T & [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
|
|
def __mul32{a:T,b:T & [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
|
|
|
|
def abs {a:[32]i8 } = emit{[32]i8 , '_mm256_abs_epi8', a}
|
|
def abs {a:[16]i16} = emit{[16]i16, '_mm256_abs_epi16', a}
|
|
def abs {a:[ 8]i32} = emit{[ 8]i32, '_mm256_abs_epi32', a}
|
|
def absu{a:[32]i8 } = emit{[32]u8 , '_mm256_abs_epi8', a}
|
|
def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a}
|
|
def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a}
|
|
|
|
def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b}
|
|
def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b}
|
|
def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}
|
|
def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}
|
|
|
|
def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b}
|
|
def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b}
|
|
def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b}
|
|
def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b}
|
|
|
|
# saturating add/sub
|
|
def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}
|
|
def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b}
|
|
def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}
|
|
def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b}
|
|
|
|
def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}
|
|
def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b}
|
|
def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}
|
|
def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b}
|
|
|
|
|
|
# structural operations
|
|
def shl{S==[16]u8, x:T, n & w256{T}} = cast_v{T, emit{T, '_mm256_bslli_epi128', x, n}}
|
|
def shr{S==[16]u8, x:T, n & w256{T}} = cast_v{T, emit{T, '_mm256_bsrli_epi128', x, n}}
|
|
|
|
def blend{I==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}}
|
|
def blend{I==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}}
|
|
def blend{I==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}}
|
|
|
|
# blend by sign bit; still no 16-bit case
|
|
def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = cast_v{T, emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}}
|
|
|
|
# assumes all bits are the same in each mask item
|
|
def blendf = blend
|
|
def blendf{f:T, t:T, m:M & w256{T} & w256i{M,16}} = blend{f, t, cast_v{[32]i8,m}}
|
|
|
|
def shuf{I==[4]u32, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}}
|
|
def shuf{I==[4]u64, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}}
|
|
|
|
def sel{I, x:T, i:[8]i32 & w256{I,32}} = cast_v{T, emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}}
|
|
def sel{I, x:T, i:[32]i8 & w128{I, 8}} = cast_v{T, emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}}
|
|
|
|
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
|
|
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}
|
|
|
|
# mask stuff
|
|
def getmask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
|
|
def getmask{x:T & w256{T, 16}} = {
|
|
msk:u32 = getmask{emit{[32]u8, '_mm256_packs_epi16', x, broadcast{[16]u16, 0}}}
|
|
(msk&255) | (msk>>8) # TODO try out permute4x64 for fixing up
|
|
}
|
|
def any{x:T & w256i{T, 16}} = getmask{cast_v{[32]u8,x}}!=0
|
|
def all{x:T & w256i{T, 16}} = getmask{cast_v{[32]u8,x}} == (1<<32)-1
|
|
def anyneg{x:T & w256s{T, 16}} = getmask{cast_v{[32]u8, cast_v{[16]i16,x} < broadcast{[16]i16, 0}}}!=0
|