From 9209ca5d6d8696819583cdbef6402618ea264741 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 2 Apr 2023 01:53:16 +0300 Subject: [PATCH] SSE2 copy.singeli --- build/src/build.bqn | 2 +- src/singeli/src/avx2.singeli | 2 +- src/singeli/src/base.singeli | 1 + src/singeli/src/copy.singeli | 4 +++- src/singeli/src/sse2.singeli | 28 ++++++++++++++++++++++------ src/utils/mut.c | 4 ++-- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index a4269346..ec32a0eb 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -592,7 +592,7 @@ cachedBin‿linkerCache ← { singeliMap ← 1↓¨ ({∨´ ((po.avx2/"2")∾⊑po.arch) ∊ 𝕩}¨ ⊑¨)⊸/ ⟨ "xa."‿"src/builtins/arithm.c"‿"monarith", "xa."‿"src/core/stuff.c"‿"equal", "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", - "2a."‿"src/builtins/squeeze.c"‿"squeeze", "2a."‿"src/utils/mut.c"‿"copy", + "2a."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index f676d80a..58e717e1 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -152,7 +152,7 @@ def narrow{T, x:X & w256i{X,32} & width{T}==16} = to_el{T, shuf{[4]u64, packQ{x, def narrow{T, x:X & w256i{X,16} & width{T}== 8} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} def narrow{T, x:X & w256f{X,64} & T>1<<2) | (iota{32}&1)}}} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 931c5cae..c84e7c59 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -164,6 +164,7 @@ def homMaskStore{...x} = assert{'homMaskStore not supported', show{...x}} def topMaskStoreF{...x} = assert{'topMaskStoreF not supported', show{...x}} def homMaskStoreF{...x} = assert{'homMaskStoreF not supported', show{...x}} def widen{...x} = assert{'widen not supported', show{...x}} +def narrow{...x} = assert{'narrow not supported', show{...x}} def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 7529a96c..f6bc3452 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -1,8 +1,10 @@ include './base' -if (hasarch{'X86_64'}) { +if (hasarch{'AVX2'}) { include './sse' include './avx' include './avx2' +} else if (hasarch{'X86_64'}) { + include './sse2' } else if (hasarch{'AARCH64'}) { include './neon' } diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 7c2d53e3..411ae389 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -195,11 +195,27 @@ def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n} def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n} -def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} -def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} -def widen{T, x:X & w128i{T} & w128i{X,elwidth{T}/2}} = unpackLo{x, x} >> elwidth{X} -def widen{T, x:X & w128i{T,32} & w128i{X,8} & w128s{T}==w128s{X}} = { def u=unpackLo{x, x}; unpackLo{u, u} >> 24} -def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} - def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f & ~m)) def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} + +def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = { + def s{v} = s{unpackLo{v, v}} + def s{v:V & V==T} = v + s{x} >> (elwidth{T} - elwidth{X}) +} +def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} +def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x} + +def narrow{T==i16, x:[4]i32} = packs{x,x} +def narrow{T==i8, x:[8]i16} = packs{x,x} +def narrow{T==u8, x:[8]u16} = packs{x,x} +def narrow{T==u16, x:[4]u32} = [8]u16~~shuf{[4]i32, shuf16Hi{shuf16Lo{x, 4b3320}, 4b3320}, 4b3320} +def narrow{T==i8, x:[4]i32} = narrow{T, narrow{i16, x}} +def narrow{T==u8, x:[4]u32} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{x}}} +def narrow{T==u8, x:[2]u64} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{f{x}}}} +def narrow{T==u16, x:[2]u64} = shuf16Lo{[8]u16~~shuf{[4]i32, x, 4b3320}, 4b3320} +def narrow{T==u32, x:[2]u64} = [4]u32~~shuf{[4]i32, x, 4b3320} + +def narrow{T, x:X & w128f{X,64} & T