Use Singeli select include for x86 blend, byteshift, zip

This commit is contained in:
Marshall Lochbaum 2024-10-12 15:41:48 -04:00
parent 6140290813
commit 27af1c0558
7 changed files with 23 additions and 38 deletions

View File

@ -1,6 +1,3 @@
def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b}
def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b}
# questionable pack; these work in 128-bit lanes and saturate elements
def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
@ -19,14 +16,11 @@ def mul32{a:T,b:T==[ 4]u64} = emit{T, '_mm256_mul_epu32', a, b} # reads only l
# structural operations
def shl{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i16~~a, [16]i16~~b, m}
def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m}
def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m}
def blend{([8]u16), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
def blend{([8]u32), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
def blend{([4]u64), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m}
def homBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M if w256{T, 16} and w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}

View File

@ -175,7 +175,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 }
# base cases
def {
absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half,
absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half,
homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit,
loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep,
pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow,
@ -209,13 +209,15 @@ if_inline (hasarch{'X86_64'}) {
include 'arch/neon_intrin/basic'
include 'arch/neon_intrin/select'
include './neon'
def {vec_shuffle16_lo}
} else {
def {__adds,__subs,__sqrt,vec_broadcast,vec_make,vec_shuffle16_lo}
def {__adds,__subs,__sqrt,vec_broadcast,vec_make}
}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
if_inline (not hasarch{'X86_64'}) {
def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k}
}
def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k}
def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k}

View File

@ -130,9 +130,9 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = {
def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = {
def V16 = re_el{u16,V}
def b = new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
def c = blend{[8]u16, b, b <<{u64} 4, 2b01100110}
def d = (V16~~c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)}
def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}}
def d = (c * make{V16, cycle{k/2, 1<<tup{2,0,2,0}}}) >> 2 # TODO ofence constant for clang
homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f
}

View File

@ -75,7 +75,7 @@ export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', sca
# Assumes identity is 0
def scan_assoc{op} = {
def shl0{v, k} = shl{[16]u8, v, k/8} # Lanewise
def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
# Broadcast end of lane 0 to entire lane 1
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
@ -107,7 +107,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
m := V**mark
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
p := clmul{a, m, i}
t := shr{[16]u8, p, 8}
t := vec_shift_left_128{p, 1}
s := p ^ carry
carry = s ^ t
s
@ -342,7 +342,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
r in tup{V, rp} over promote{u64,nw}) {
s := scan_words{x, m}
pc:= c; c = vec_shuffle{-(s>>63), 3,0,1,2}
r = apply_carry{s, blend{V, c, pc, 2b0001}, (m-V**1)&~m}
r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m}
m = m>>d4 | m<<(l-d4)
}
}

View File

@ -47,7 +47,7 @@ def make_scan_idem{T, op, up} = {
def w=width{T}; def c = k/w
def merger{a,b} = if (up) merge{a,b} else merge{b,a}
def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}}
(if (up) shl else shr){[16]u8, v, k/8} | id
(if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id
}
def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
def shb{v, k if k>=32} = vec_shuffle{[4]u32, v, shift{k/32,4}}

View File

@ -1,6 +1,6 @@
### SSSE3 ###
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)}
def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n}
### SSE4.1 ###
@ -21,12 +21,11 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16
# mask stuff
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m}
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M}} = blend_hom{f, t, m}

View File

@ -23,10 +23,6 @@ def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
# shift
def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
# integer arith
def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
@ -68,12 +64,6 @@ def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}}
def x86_postfix{(f32)} = '_ps'
def x86_postfix{(f64)} = '_pd'
def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b}
def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b}
def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4}
def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = T~~vec_shuffle{[4]f32, tup{a, b}, is&3}