From 27af1c055816a4d5385864b5d420d5f84cd65e61 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 12 Oct 2024 15:41:48 -0400 Subject: [PATCH] Use Singeli select include for x86 blend, byteshift, zip --- src/singeli/src/avx2.singeli | 14 ++++---------- src/singeli/src/base.singeli | 12 +++++++----- src/singeli/src/bitalign.singeli | 6 +++--- src/singeli/src/scan.singeli | 6 +++--- src/singeli/src/scan_common.singeli | 2 +- src/singeli/src/sse.singeli | 11 +++++------ src/singeli/src/sse2.singeli | 10 ---------- 7 files changed, 23 insertions(+), 38 deletions(-) diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index e569de6b..5ec66dcc 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -1,6 +1,3 @@ -def zip128{a:T, b:T, 0 if w256{T}} = emit{T, merge{'_mm256_unpacklo',x86_postfix{eltype{T}}}, a, b} -def zip128{a:T, b:T, 1 if w256{T}} = emit{T, merge{'_mm256_unpackhi',x86_postfix{eltype{T}}}, a, b} - # questionable pack; these work in 128-bit lanes and saturate elements def packQ{a:T,b:T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b} def packQ{a:T,b:T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b} @@ -19,14 +16,11 @@ def mul32{a:T,b:T==[ 4]u64} = emit{T, '_mm256_mul_epu32', a, b} # reads only l # structural operations -def shl{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} -def shr{([16]u8), x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} +def blend{([8]u16), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[16]i16~~a, [16]i16~~b, m} +def blend{([8]u32), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 8]i32~~a, [ 8]i32~~b, m} +def blend{([4]u64), a:T, b:T, m if w256{T} and ktup{m}} = T ~~ blend_units{[ 4]f64~~a, [ 4]f64~~b, m} -def blend{([8]u16), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m} -def blend{([8]u32), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} -def blend{([4]u64), a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m} - -def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} +def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = blend_top{f, t, m} def homBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = topBlend{f, t, m} def homBlend{f:T, t:T, m:M if w256{T, 16} and w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 1dd99ab4..603a27eb 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -175,7 +175,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 } # base cases def { - absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half, + absu,andAllZero,andnz,b_getBatch,blend,blend_units,clmul,cvt,extract,fold_addw,half, homAll,homAny,bitAll,bitAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowTrunc,narrowPair,packQ,pair,pdep, pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow, @@ -209,13 +209,15 @@ if_inline (hasarch{'X86_64'}) { include 'arch/neon_intrin/basic' include 'arch/neon_intrin/select' include './neon' - def {vec_shuffle16_lo} } else { - def {__adds,__subs,__sqrt,vec_broadcast,vec_make,vec_shuffle16_lo} + def {__adds,__subs,__sqrt,vec_broadcast,vec_make} } -def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} -def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} +if_inline (not hasarch{'X86_64'}) { + def {vec_shift_left_128,vec_shift_right_128,vec_shuffle16_lo} + def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} + def zip128{a:T, b:T, k if width{T}==128} = zip{a, b, k} +} def mzip {a:T, b:T, k} = el_m{T} ~~ zip {a, b, k} def mzip128{a:T, b:T, k} = el_m{T} ~~ zip128{a, b, k} diff --git a/src/singeli/src/bitalign.singeli b/src/singeli/src/bitalign.singeli index c914353c..bdd9fc1b 100644 --- a/src/singeli/src/bitalign.singeli +++ b/src/singeli/src/bitalign.singeli @@ -130,9 +130,9 @@ def bitalign{s, 8 if s<7, a:V=[k](u8) if hasarch{'X86_64'}} = { def bitalign{7, 8, a:V=[k](u8) if hasarch{'X86_64'}} = { def V16 = re_el{u16,V} - def b = new_shuf{a, range{k} - (((range{k}+2)/8)>>0)} - def c = blend{[8]u16, b, b <<{u64} 4, 2b01100110} - def d = (V16~~c * make{V16, cycle{k/2, 1<> 2 # TODO ofence constant for clang + def b = V16~~new_shuf{a, range{k} - (((range{k}+2)/8)>>0)} + def c = blend_units{b, b <<{u64} 4, tup{0,1,1,0}} + def d = (c * make{V16, cycle{k/2, 1<> 2 # TODO ofence constant for clang homBlend{V~~d, V~~(d+d), make{V, cycle{k, tup{0, 0xff}}}} & V**0x7f } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 78e3f45f..254239c0 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -75,7 +75,7 @@ export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', sca # Assumes identity is 0 def scan_assoc{op} = { - def shl0{v, k} = shl{[16]u8, v, k/8} # Lanewise + def shl0{v:[_]T, k} = vec_shift_right_128{v, k/width{T}} # Lanewise def shl0{v:V, k==128 if hasarch{'AVX2'}} = { # Broadcast end of lane 0 to entire lane 1 l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v} @@ -107,7 +107,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64 m := V**mark def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total p := clmul{a, m, i} - t := shr{[16]u8, p, 8} + t := vec_shift_left_128{p, 1} s := p ^ carry carry = s ^ t s @@ -342,7 +342,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { r in tup{V, rp} over promote{u64,nw}) { s := scan_words{x, m} pc:= c; c = vec_shuffle{-(s>>63), 3,0,1,2} - r = apply_carry{s, blend{V, c, pc, 2b0001}, (m-V**1)&~m} + r = apply_carry{s, blend{c, pc, 1,0,0,0}, (m-V**1)&~m} m = m>>d4 | m<<(l-d4) } } diff --git a/src/singeli/src/scan_common.singeli b/src/singeli/src/scan_common.singeli index fa29e611..21f6b045 100644 --- a/src/singeli/src/scan_common.singeli +++ b/src/singeli/src/scan_common.singeli @@ -47,7 +47,7 @@ def make_scan_idem{T, op, up} = { def w=width{T}; def c = k/w def merger{a,b} = if (up) merge{a,b} else merge{b,a} def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}} - (if (up) shl else shr){[16]u8, v, k/8} | id + (if (up) vec_shift_right_128 else vec_shift_left_128){v, c} | id } def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} def shb{v, k if k>=32} = vec_shuffle{[4]u32, v, shift{k/32,4}} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index dc347da2..1af2dfb6 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -1,6 +1,6 @@ ### SSSE3 ### def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} -def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)} +def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = vec_merge_shift_right{a, b, n} ### SSE4.1 ### @@ -21,12 +21,11 @@ def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16 # mask stuff def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} -def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ blend_top{v2f{f}, v2f{t}, m} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ blend_top{v2d{f}, v2d{t}, m} +def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = blend_top{ f , t , m} # assumes all bits are the same in each mask item -def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m} -def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m} +def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M}} = blend_hom{f, t, m} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index c4aa1a5e..997c3f17 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -23,10 +23,6 @@ def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} -# shift -def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} -def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} - # integer arith def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} @@ -68,12 +64,6 @@ def packs{a:T,b:T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b} def packs{a:T,b:T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b} def packQ{a:T,b:T if w128i{T}} = packs{a,b} -def x86_postfix{T if isint{T}} = merge{'_epi',fmtnat{width{T}}} -def x86_postfix{(f32)} = '_ps' -def x86_postfix{(f64)} = '_pd' -def zip{a:T, b:T, 0 if w128{T}} = emit{T, merge{'_mm_unpacklo',x86_postfix{eltype{T}}}, a, b} -def zip{a:T, b:T, 1 if w128{T}} = emit{T, merge{'_mm_unpackhi',x86_postfix{eltype{T}}}, a, b} - def x86_shufps_range{is, hi} = inrange{is,0,2, 0,4} and inrange{is,2,4, hi,hi+4} def shufInd{a:T, b:T=[4]E, {...is} if width{E}==32 and length{is}==4 and x86_shufps_range{is, 4}} = T~~vec_shuffle{[4]f32, tup{a, b}, is&3}