61 lines
2.9 KiB
Plaintext
61 lines
2.9 KiB
Plaintext
# compact casting for the annoying intrinsic type system
|
|
def v2i{x:T=[_]E if w256{T}} = if(isint{E}) x else [32]u8 ~~ x
|
|
def v2f{x:T if w256{T}} = [8]f32 ~~ x
|
|
def v2d{x:T if w256{T}} = [4]f64 ~~ x
|
|
|
|
def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
|
|
def zeroPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}}
|
|
|
|
# load & store
|
|
def load_low{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, load_low{*n_h{V} ~~ ptr, w}}
|
|
def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr}
|
|
|
|
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}}
|
|
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x}
|
|
|
|
|
|
# float comparison
|
|
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
|
local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
|
|
def unord{a:T, b:T==[8]f32} = f32cmpAVX{a,b,3}
|
|
def unord{a:T, b:T==[4]f64} = f64cmpAVX{a,b,3}
|
|
|
|
# f32 arith
|
|
def rsqrtE{a:T==[8]f32} = emit{T, '_mm256_rsqrt_ps', a}
|
|
def rcpE{a:T==[8]f32} = emit{T, '_mm256_rcp_ps', a}
|
|
|
|
# conversion
|
|
def half{x:T, i if w256{T} and knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
|
|
def half{x:T, 0 if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
|
|
def pair{a:T,b:T if w128{T}} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
|
|
|
|
def widen{T==[4]f64, x:([4]i32)} = emit{T, '_mm256_cvtepi32_pd', x}
|
|
def widen{T==[4]f64, x:([4]f32)} = emit{T, '_mm256_cvtps_pd', x}
|
|
def widen{T==[4]f64, x:X=[_]U if w128i{X} and width{U}<32} = widen{T, widen{[4]i32, x}}
|
|
def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}}
|
|
|
|
|
|
# structural operations
|
|
def blend_top{f:T, t:T, m:M if w256i{T,32} and w256i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, v2f{m}}
|
|
def blend_top{f:T, t:T, m:M if w256i{T,64} and w256i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, v2d{m}}
|
|
|
|
# shuffles
|
|
# TODO rename to shuf, move shuf to shuf_imm or something
|
|
def shufInd{a:T, b:T=[8]E, {...is} if width{E}==32 and length{is}==8 and ({l,h} => same{l,h-4} and x86_shufps_range{l, 8}){slice{is,0,4}, slice{is,4,8}}} = {
|
|
vec_shuffle{[4]f32, tup{a, b}, slice{is,0,4} & 3}
|
|
}
|
|
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==64 and length{is}==4} = T~~shufInd{re_el{u32,a}, re_el{u32,b}, merge{...each{{i} => tup{i*2, i*2+1}, is}}}
|
|
|
|
# mask stuff
|
|
def andAllZero{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
|
|
|
|
def top_to_int{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
|
|
def top_to_int{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
|
|
def hom_to_int{x:T if w256{T}} = top_to_int{x}
|
|
|
|
def any_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} != 0
|
|
def all_hom{x:T if w256i{T} and elwidth{T}>=32} = hom_to_int{[8]u32 ~~ x} == 0xff
|
|
|
|
def any_top{x:T=[_]E if w256i{T} and width{E}>=32} = top_to_int{x} != 0
|
|
def all_top{x:T=[k]E if w256i{T} and width{E}>=32} = top_to_int{x} == (1<<k)-1
|