Use Singeli select include for x86 blend, byteshift, zip
This commit is contained in:
parent
6140290813
commit
27af1c0558
@ -1,6 +1,3 @@
|
||||
def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
# questionable pack; these work in 128-bit lanes and saturate elements
|
||||
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
|
||||
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
|
||||
@ -19,14 +16,11 @@ def mul32{a:T,b:T==[ 4]u64} = emit{T, '_mm256_mul_epu32', a, b} # reads only l
|
||||
|
||||
# structural operations
|
||||
|
||||
def shl{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i16~~a, [16]i16~~b, m}
|
||||
def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m}
|
||||
def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m}
|
||||
|
||||
def blend{([8]u16), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
|
||||
def blend{([8]u32), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
|
||||
def blend{([4]u64), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
|
||||
|
||||
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M if w256{T, 16} and w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
|
||||
|
||||
|
||||
@ -175,7 +175,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
|
||||
|
||||
# base cases
|
||||
def {
|
||||
absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half,
|
||||
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
|
||||
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
|
||||
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
|
||||
pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow,
|
||||
@ -209,13 +209,15 @@ if_inline (hasarch{'X86_64'}) {
|
||||
include 'arch/neon_intrin/basic'
|
||||
include 'arch/neon_intrin/select'
|
||||
include './neon'
|
||||
def {vec_shuffle16_lo}
|
||||
} else {
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make,vec_shuffle16_lo}
|
||||
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
|
||||
}
|
||||
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
if_inline (not hasarch{'X86_64'}) {
|
||||
def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
|
||||
}
|
||||
|
||||
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
|
||||
def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k}
|
||||
|
||||
@ -130,9 +130,9 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
|
||||
def V16 = re_el{u16,V}
|
||||
|
||||
def b = new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
|
||||
def c = blend{[8]u16, b, b <<{u64} 4, 2b01100110}
|
||||
def d = (V16~~c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
|
||||
def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
|
||||
def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}}
|
||||
def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
|
||||
homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
|
||||
}
|
||||
|
||||
|
||||
@ -75,7 +75,7 @@ export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', sca
|
||||
|
||||
# Assumes identity is 0
|
||||
def scan_assoc{op} = {
|
||||
def shl0{v, k} = shl{[16]u8, v, k/8} # Lanewise
|
||||
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
|
||||
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
|
||||
# Broadcast end of lane 0 to entire lane 1
|
||||
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
|
||||
@ -107,7 +107,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
|
||||
m := V**mark
|
||||
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
|
||||
p := clmul{a, m, i}
|
||||
t := shr{[16]u8, p, 8}
|
||||
t := vec_shift_left_128{p, 1}
|
||||
s := p ^ carry
|
||||
carry = s ^ t
|
||||
s
|
||||
@ -342,7 +342,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
||||
r in tup{V, rp} over promote{u64,nw}) {
|
||||
s := scan_words{x, m}
|
||||
pc:= c; c = vec_shuffle{-(s>>63), 3,0,1,2}
|
||||
r = apply_carry{s, blend{V, c, pc, 2b0001}, (m-V**1)&~m}
|
||||
r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m}
|
||||
m = m>>d4 | m<<(l-d4)
|
||||
}
|
||||
}
|
||||
|
||||
@ -47,7 +47,7 @@ def make_scan_idem{T, op, up} = {
|
||||
def w=width{T}; def c = k/w
|
||||
def merger{a,b} = if (up) merge{a,b} else merge{b,a}
|
||||
def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}}
|
||||
(if (up) shl else shr){[16]u8, v, k/8} | id
|
||||
(if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id
|
||||
}
|
||||
def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
|
||||
def shb{v, k if k>=32} = vec_shuffle{[4]u32, v, shift{k/32,4}}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
### SSSE3 ###
|
||||
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
|
||||
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)}
|
||||
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n}
|
||||
|
||||
|
||||
### SSE4.1 ###
|
||||
@ -21,12 +21,11 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
|
||||
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m}
|
||||
# assumes all bits are the same in each mask item
|
||||
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m}
|
||||
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m}
|
||||
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M}} = blend_hom{f, t, m}
|
||||
|
||||
|
||||
|
||||
|
||||
@ -23,10 +23,6 @@ def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
|
||||
|
||||
|
||||
|
||||
# shift
|
||||
def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
|
||||
def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
|
||||
|
||||
# integer arith
|
||||
def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
|
||||
def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
|
||||
@ -68,12 +64,6 @@ def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
|
||||
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
|
||||
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
|
||||
|
||||
def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}}
|
||||
def x86_postfix{(f32)} = '_ps'
|
||||
def x86_postfix{(f64)} = '_pd'
|
||||
def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
|
||||
def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
|
||||
|
||||
def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4}
|
||||
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = T~~vec_shuffle{[4]f32, tup{a, b}, is&3}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user