diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 14494938..164c02c9 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -21,6 +21,10 @@ def cast_v{R, x:S & w256i{R } & w256{S}} = emit{R, '(__m256i)', x} def cast_v{R, x:S & w256f{R,32} & w256{S}} = emit{R, '(__m256)', x} def cast_v{R, x:S & w256f{R,64} & w256{S}} = emit{R, '(__m256d)', x} +def v2i{x:T & w256{T}} = cast_v{[32]u8, x} # for compact casting for the annoying intrinsic type system +def v2f{x:T & w256{T}} = cast_v{[8]f32, x} +def v2d{x:T & w256{T}} = cast_v{[4]f64, x} + def ty_vu{T & w256s{T}} = [vcount{T}](ty_iu{eltype{T}}) def ty_vs{T & w256u{T}} = [vcount{T}](ty_is{eltype{T}}) def forv{T & w256{T}} = forc{{v}=>cast_vp{T,v}} @@ -90,6 +94,20 @@ def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a} def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} +# structural operations +def extract{x:T, i & w256i{T,32} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi32', x, i} +def extract{x:T, i & w256i{T,64} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi64', x, i} + +def insert{x:T, i, v & w256i{T, 8} & knum{i}} = emit{T, '_mm256_insert_epi8', x, v, i} +def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x, v, i} +def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i} +def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i} + +# mixed-width operations +def half{x:[8]i32, i & knum{i}} = emit{[4]i32, '_mm256_extracti128_si256', x, i} +def pair{a:[4]i32,b:[4]i32} = emit{[8]i32, '_mm256_setr_m128i', a, b} + +# mask stuff def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', cast_v{[8]f32, x}} def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', cast_v{[4]f64, x}} def any{x:T & w256i{T}} = getmask{x}!=0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits) diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 1aa0a739..8f4730b1 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -31,15 +31,21 @@ def __gt{a:T,b:T & w256u{T}} = ~(a<=b) def __ne{a:T,b:T & w256i{T}} = ~(b==a) # shift -def __shl{a:T,b & w256i{T, 16}} = emit{T, '_mm256_slli_epi16', a, b} -def __shl{a:T,b & w256i{T, 32}} = emit{T, '_mm256_slli_epi32', a, b} -def __shl{a:T,b & w256i{T, 64}} = emit{T, '_mm256_slli_epi64', a, b} -def __shr{a:T,b & w256u{T, 16}} = emit{T, '_mm256_srli_epi16', a, b} -def __shr{a:T,b & w256u{T, 32}} = emit{T, '_mm256_srli_epi32', a, b} -def __shr{a:T,b & w256u{T, 64}} = emit{T, '_mm256_srli_epi64', a, b} -def __shr{a:T,b & w256s{T, 16}} = emit{T, '_mm256_srai_epi16', a, b} -def __shr{a:T,b & w256s{T, 32}} = emit{T, '_mm256_srai_epi32', a, b} +def __shl{a:T,b & w256i{T, 16} & knum{b}} = emit{T, '_mm256_slli_epi16', a, b} +def __shl{a:T,b & w256i{T, 32} & knum{b}} = emit{T, '_mm256_slli_epi32', a, b} +def __shl{a:T,b & w256i{T, 64} & knum{b}} = emit{T, '_mm256_slli_epi64', a, b} +def __shr{a:T,b & w256u{T, 16} & knum{b}} = emit{T, '_mm256_srli_epi16', a, b} +def __shr{a:T,b & w256u{T, 32} & knum{b}} = emit{T, '_mm256_srli_epi32', a, b} +def __shr{a:T,b & w256u{T, 64} & knum{b}} = emit{T, '_mm256_srli_epi64', a, b} +def __shr{a:T,b & w256s{T, 16} & knum{b}} = emit{T, '_mm256_srai_epi16', a, b} +def __shr{a:T,b & w256s{T, 32} & knum{b}} = emit{T, '_mm256_srai_epi32', a, b} # no 64-bit arithmetic shift :/ +def __shl{a:T,b:S & w256i{T, 32} & w256i{S, 32}} = emit{T, '_mm256_sllv_epi32', a, b} +def __shl{a:T,b:S & w256i{T, 64} & w256i{S, 64}} = emit{T, '_mm256_sllv_epi64', a, b} +def __shr{a:T,b:S & w256u{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srlv_epi32', a, b} +def __shr{a:T,b:S & w256u{T, 64} & w256i{S, 64}} = emit{T, '_mm256_srlv_epi64', a, b} +def __shr{a:T,b:S & w256s{T, 32} & w256i{S, 32}} = emit{T, '_mm256_srav_epi32', a, b} +# still no 64-bit arithmetic shift :/ # questionable pack def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}} @@ -55,21 +61,30 @@ def packQQ{a} = packQQ{tupsel{0,a}, tupsel{1,a}} # arith -def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b} +def __mul {a:T,b:T & [16]i16==T} = emit{T, '_mm256_mullo_epi16', a, b} +def __mulhi{a:T,b:T & [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b} +def __mulhi{a:T,b:T & [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b} +def __mul {a:T,b:T & [ 8]i32==T} = emit{T, '_mm256_mullo_epi32', a, b} +def __mul32{a:T,b:T & [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments +def __mul32{a:T,b:T & [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments + +def abs {a:[32]i8 } = emit{[32]i8 , '_mm256_abs_epi8', a} +def abs {a:[16]i16} = emit{[16]i16, '_mm256_abs_epi16', a} +def abs {a:[ 8]i32} = emit{[ 8]i32, '_mm256_abs_epi32', a} +def absu{a:[32]i8 } = emit{[32]u8 , '_mm256_abs_epi8', a} +def absu{a:[16]i16} = emit{[16]u16, '_mm256_abs_epi16', a} +def absu{a:[ 8]i32} = emit{[ 8]u32, '_mm256_abs_epi32', a} + +def __add{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_add_epi8', a, b} def __add{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_add_epi16', a, b} def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b} def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b} -def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b} +def __sub{a:T,b:T & w256i{T, 8}} = emit{T, '_mm256_sub_epi8', a, b} def __sub{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_sub_epi16', a, b} def __sub{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_sub_epi32', a, b} def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b} -def __mul {a:T,b:T & [16]i16==T} = emit{T, '_mm256_mullo_epi16', a, b} -def __mulhi{a:T,b:T & [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b} -def __mul {a:T,b:T & [ 8]i32==T} = emit{T, '_mm256_mullo_epi32', a, b} -def __mul32{a:T,b:T & [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of the arguments - # saturating add/sub def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b} def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} @@ -82,6 +97,22 @@ def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b} def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} +# structural operations +def shl{S==[16]u8, x:T, n & w256{T}} = cast_v{T, emit{T, '_mm256_bslli_epi128', x, n}} +def shr{S==[16]u8, x:T, n & w256{T}} = cast_v{T, emit{T, '_mm256_bsrli_epi128', x, n}} + +def blend{I==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}} +def blend{I==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}} +def blend{I==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = cast_v{T, emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}} + +def shuf{I==[4]u32, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}} +def shuf{I==[4]u64, x:T, n & w256{T} & knum{n}} = cast_v{T, emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}} + +def sel{x:T, i:[8]i32 & w256{T,32}} = cast_v{T, emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}} +def sel{x:T, i:[32]i8 & w256{T, 8}} = cast_v{T, emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}} + +def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} +def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} # mask stuff def getmask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 62afe723..17126263 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -46,10 +46,10 @@ def arithChk2{F, w:T, x:T, i & match{F,__mul} & isvec{T} & i8==eltype{T}} = { tup{packQ{rp}, any{tupsel{0,bad}|tupsel{1,bad}}} } def arithChk2{F, w:T, x:T, i & match{F,__mul} & isvec{T} & i32==eltype{T}} = { - max:= broadcast{[8]u32, 0x4efffffe} + max:= cast_v{[8]f32, broadcast{[8]u32, 0x4efffffe}} def cf32{x} = emit{[8]f32, '_mm256_cvtepi32_ps', x} f32mul:= cf32{w} * cf32{x} - tup{w*x, any{emit{[8]u32, '_mm256_cmp_ps', abs{f32mul}, max, 29}}} + tup{w*x, any{cast_v{[8]u32, emit{[8]f32, '_mm256_cmp_ps', abs{f32mul}, max, 29}}}} # TODO fallback to the below if the above fails # def wp = unpackQ{w, broadcast{T, 0}} # def xp = unpackQ{x, broadcast{T, 0}}