SSE2 equal.singeli
This commit is contained in:
parent
b0a757eaba
commit
a62c21733e
@ -590,7 +590,7 @@ cachedBin‿linkerCache ← {
|
||||
⟩
|
||||
cbqnSrc ↩ cbqnSrc clangd.Files "src"
|
||||
singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨
|
||||
"xa."‿"src/builtins/arithm.c"‿"monarith", "2a."‿"src/core/stuff.c"‿"equal",
|
||||
"xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal",
|
||||
"xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp",
|
||||
"2a."‿"src/builtins/squeeze.c"‿"squeeze", "2a."‿"src/utils/mut.c"‿"copy",
|
||||
"xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose",
|
||||
|
||||
@ -420,8 +420,8 @@ u8 eqFnData[] = { // for the main diagonal, amount to shift length by; otherwise
|
||||
n,n,n,n,n,1,1,2,
|
||||
};
|
||||
|
||||
#if SINGELI_SIMD
|
||||
#define F(X) avx2_equal_##X
|
||||
#if SINGELI_SIMD || SINGELI_X86_64
|
||||
#define F(X) simd_equal_##X
|
||||
#define SINGELI_FILE equal
|
||||
#include "../utils/includeSingeli.h"
|
||||
#else
|
||||
|
||||
@ -164,6 +164,7 @@ def topMaskStore{...x} = assert{'topMaskStore not supported', show{...x}}
|
||||
def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}}
|
||||
def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}}
|
||||
def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}}
|
||||
def widen{...x} = assert{'widen not supported', show{...x}}
|
||||
|
||||
def widen{T, x:X & T==X} = x
|
||||
def narrow{T, x:X & T==eltype{X}} = x
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
include './base'
|
||||
include './cbqnDefs'
|
||||
include './f64'
|
||||
if (hasarch{'X86_64'}) {
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
} else if (hasarch{'X86_64'}) {
|
||||
include './sse2'
|
||||
} else if (hasarch{'AARCH64'}) {
|
||||
include './neon'
|
||||
}
|
||||
@ -36,9 +38,9 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
f0:= TF**0.0
|
||||
f1:= TF**1.0
|
||||
maskedLoopPositive{bulk, l, {i, M} => {
|
||||
wu:= (if (hasarch{'X86_64'}) {
|
||||
wu:= (if (hasarch{'AVX2'}) {
|
||||
cw:= b_getBatchLo{bulk, *u64~~w, i}
|
||||
topBlend{f0, f1, TU**cw << make{TU,63,62,61,60}}
|
||||
topBlend{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
|
||||
} else {
|
||||
cw:= loadBatchBit{TU, *u64~~w, i}
|
||||
homBlend{f0, f1, cw}
|
||||
@ -77,23 +79,23 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
1
|
||||
}
|
||||
|
||||
export{'avx2_equal_1_1', equal{u1, u1}}
|
||||
export{'avx2_equal_1_8', equal{u1, u8}}
|
||||
export{'avx2_equal_1_16', equal{u1, u16}}
|
||||
export{'avx2_equal_1_32', equal{u1, u32}}
|
||||
export{'avx2_equal_1_f64', equal{u1, f64}}
|
||||
export{'simd_equal_1_1', equal{u1, u1}}
|
||||
export{'simd_equal_1_8', equal{u1, u8}}
|
||||
export{'simd_equal_1_16', equal{u1, u16}}
|
||||
export{'simd_equal_1_32', equal{u1, u32}}
|
||||
export{'simd_equal_1_f64', equal{u1, f64}}
|
||||
|
||||
export{'avx2_equal_8_8', equal{i8, i8}}
|
||||
export{'simd_equal_8_8', equal{i8, i8}}
|
||||
|
||||
export{'avx2_equal_s8_16', equal{i8, i16}}
|
||||
export{'avx2_equal_s8_32', equal{i8, i32}}
|
||||
export{'avx2_equal_s16_32', equal{i16, i32}}
|
||||
export{'simd_equal_s8_16', equal{i8, i16}}
|
||||
export{'simd_equal_s8_32', equal{i8, i32}}
|
||||
export{'simd_equal_s16_32', equal{i16, i32}}
|
||||
|
||||
export{'avx2_equal_s8_f64', equal{i8, f64}}
|
||||
export{'avx2_equal_s16_f64', equal{i16, f64}}
|
||||
export{'avx2_equal_s32_f64', equal{i32, f64}}
|
||||
export{'avx2_equal_f64_f64', equal{f64, f64}}
|
||||
export{'simd_equal_s8_f64', equal{i8, f64}}
|
||||
export{'simd_equal_s16_f64', equal{i16, f64}}
|
||||
export{'simd_equal_s32_f64', equal{i32, f64}}
|
||||
export{'simd_equal_f64_f64', equal{f64, f64}}
|
||||
|
||||
export{'avx2_equal_u8_16', equal{u8, u16}}
|
||||
export{'avx2_equal_u8_32', equal{u8, u32}}
|
||||
export{'avx2_equal_u16_32', equal{u16, u32}}
|
||||
export{'simd_equal_u8_16', equal{u8, u16}}
|
||||
export{'simd_equal_u8_32', equal{u8, u32}}
|
||||
export{'simd_equal_u16_32', equal{u16, u32}}
|
||||
|
||||
@ -41,7 +41,8 @@ def widen{T==[2]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}
|
||||
def narrow{T, x:X & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
|
||||
def narrow{T, x:X & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
|
||||
|
||||
# mask
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
|
||||
def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
|
||||
|
||||
@ -159,6 +159,7 @@ def abs{a:[4]f32} = emit{[4]f32, '_mm_and_ps', a, v2f{[4]u32 ** 0x7FFFFFFF}}
|
||||
def abs{a:[2]f64} = emit{[2]f64, '_mm_and_pd', a, v2d{[2]u64 ** ((cast{u64,1}<<63)-1)}}
|
||||
|
||||
# mask stuff
|
||||
def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0}
|
||||
|
||||
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
|
||||
def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
|
||||
@ -196,7 +197,8 @@ def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
|
||||
def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
|
||||
def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
|
||||
def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X}
|
||||
def widen{T==[4]i32, x:[16]i8} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24}
|
||||
def widen{T, x:X & w128i{T,32} & w128i{X,8} & w128s{T}==w128s{X}} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24}
|
||||
def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
|
||||
|
||||
def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m))
|
||||
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user