NEON squeeze

This commit is contained in:
dzaima 2023-01-25 01:25:28 +02:00
parent 88410c02e5
commit 6baa96c1e3
8 changed files with 116 additions and 49 deletions

View File

@ -514,7 +514,7 @@ cachedBin‿linkerCache ← {
"src/jit/", "nvm.c"
"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"
singeliMap {po.arch"aarch64"? 𝕩/˜(1¨𝕩)"cmp""bits""equal""dyarith""monarith"; 𝕩}
singeliMap {po.arch"aarch64"? 𝕩/˜(1¨𝕩)"cmp""bits""equal""dyarith""monarith""squeeze"; 𝕩}
"src/builtins/arithm.c""monarith",
"src/core/stuff.c""equal", "src/utils/mut.c""copy", "src/utils/bits.c""bits"
"src/builtins/arithd.c""dyarith", "src/builtins/cmp.c""cmp", "src/builtins/squeeze.c""squeeze"

View File

@ -1,6 +1,6 @@
#include "../core.h"
#if SINGELI_X86_64
#if SINGELI
#define SINGELI_FILE squeeze
#include "../utils/includeSingeli.h"
#endif
@ -36,14 +36,14 @@ B num_squeeze(B x) {
goto r_bit;
}
#if !SINGELI_X86_64
#if !SINGELI
usz i = 0;
#endif
u32 or = 0; // using bitwise or as an approximate ⌈´
switch (xe) { default: UD;
case el_bit: goto r_x;
#if SINGELI_X86_64
#if SINGELI
case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; }
case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; }
case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; }
@ -69,7 +69,7 @@ B num_squeeze(B x) {
B* xp = arr_bptr(x);
if (xp==NULL) goto r_f;
#if SINGELI_X86_64
#if SINGELI
or = avx2_squeeze_numB(xp, ia);
if (-2==(i32)or) goto r_x;
if (-1==(i32)or) goto r_f64;
@ -109,11 +109,15 @@ B num_squeeze(B x) {
B chr_squeeze(B x) {
usz ia = IA(x);
u8 xe = TI(x,elType);
if (ia==0) {
if (xe==el_c8) return x;
goto r_c8;
}
usz i = 0;
i32 or = 0;
switch(xe) { default: UD;
case el_c8: goto r_x;
#if SINGELI_X86_64
#if SINGELI
case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; }
case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; }
#else
@ -138,7 +142,7 @@ B chr_squeeze(B x) {
B* xp = arr_bptr(x);
if (xp!=NULL) {
#if SINGELI_X86_64
#if SINGELI
u32 t = avx2_squeeze_chrB(xp, ia);
if (t==0) goto r_c8;
else if (t==1) goto r_c16;

View File

@ -98,7 +98,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
def half{x:T, i & w256{T} & knum{i}} = v_half{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
def half{x:T, i==0 & w256{T}} = v_half{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def pair{a:T,b:T & width{T}==128} = v_dbl{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
def pair{x & tuplen{x}==2} = pair{tupsel{0,x},tupsel{1,x}}
def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x}
def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x}

View File

@ -124,6 +124,8 @@ def fold_max {...x} = assert{'fold_max not supported', show{...x}}
def fold_min {...x} = assert{'fold_min not supported', show{...x}}
def fold_add {...x} = assert{'fold_add not supported', show{...x}}
def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
def pair{...x} = assert{'pair not supported', show{...x}}
def cvt{...x} = assert{'cvt not supported', show{...x}}
def cvt{T, x:X & T==eltype{X}} = x
def shuf{...x} = assert{'shuf not supported', show{...x}}

View File

@ -123,9 +123,11 @@ def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}}
def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{ty_half{v_half{X}}, ntyp0{'vmovn', X}, x}
def narrow{T, x:X & n128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
def narrowUpper{lowRes:L, x:X & n64{L} & n128{X} & isint{eltype{X}} & ty_dbl{v_dbl{L}}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
def narrowPair{a, b} = narrowUpper{narrow{ty_half{eltype{type{a}}}, a}, b}
def narrowPair{a:T, b:T} = narrowUpper{narrow{ty_half{eltype{T}}, a}, b}
def narrowPair{a:T, b:T & isint{eltype{T}}} = packLo{a, b}
def widenUpper{x:T & n128{T} & isint{eltype{T}}} = emit{ty_dbl{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T & n128{T}} = tup{widen{ty_dbl{T}, x}, widenUpper{x}}

View File

@ -31,10 +31,10 @@ def fold{F, x:T} = {
show{'WARNING: using fallback fold'}
def E = eltype{T}
r:E = 0
each{{i} => { r = F{r, cast_i{E, extract{x, i}}, iota{vcount{T}}} }}
promote{u32, r}
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
r
}
def fold{F, x:T & w128{T}} = {
def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
@ -43,7 +43,7 @@ def fold{F, x:T & w128{T}} = {
if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
extract{c, 0}
}
def fold{F, x:T & w256{T}} = fold{F, F{half{x, 0}, half{x, 1}}}
def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
def makeOptBranch{enable, F} = {
if (enable) {
@ -57,11 +57,29 @@ def makeOptBranch{enable, F} = {
}
}
def cvtNarrow{T, x:X & width{T}==elwidth{X}} = cvt{T, x}
def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
assert{len>0}
def bulk = vw / width{X}
def XV = [bulk]X
def E = tern{X==f64, u32, ty_u{X}}
def EV = [bulk]E
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
if (elwidth{T}==64) {
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
else fold_max{narrow{TE, x}}
} else {
fold_max{x}
}
}
# show{XV, EV, CHR, B}
xp:= *X~~x0
r1:= EV**0
@ -76,7 +94,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
}
}}
r2:= cast_i{u32, fold{|, r1}}
r2:= foldTotal{u32, r1}
if (X>u32 and r2>=65536) return{2}
if (X>u16 and r2>=256) return{1}
0
@ -98,37 +116,73 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
return{0xffff_ffff} # float
}}
maskedLoop{bulk, len, {i, M} => {
v0:= loadBatch{xp, i, XV}
def toint{x:T & isint{eltype{T}}} = x
def toint{flt:T & isfloat{eltype{T}}} = {
int:= narrow{i32, flt}
assert{width{type{int}} == width{XV}/2} # we'll be doing operations over it
def conv{x} = tern{preserve_negative_zero, ty_u{x}, x}
if (anynePositive{conv{flt}, conv{widen{T, int}}, M}) { # is any not an integer
if (B) { iCont=i; goto{case_B} } # if B, need to give an even more special result
else return{0xffff_ffff} # float
}
int
}
assert{XV == type{v0}}
v1:= toint{v0} # TODO do with an if instead
r1|= M{((EV ** ~E~~1) & EV~~v1) ^ EV~~(v1 >> (width{X}-1))}
}}
def getAcc{EV, x:T} = {
((EV ** ~(eltype{EV})~~1) & EV~~x) ^ EV~~(x >> (elwidth{T}-1))
}
promote{u32, fold{|, r1}}
if (isint{X}) { # i16, i32
muLoop{bulk, 1, len, {is, M} => {
def v0 = loadBatch{xp, is, XV}
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
}}
} else { # f64
def EV2 = v_dbl{EV}
r2:= EV2**0
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
def v0 = loadBatch{xp, is, XV}
def int = {
def tmp = {
if (hasarch{'AARCH64'} and tuplen{is}==2) {
def intp = narrowPair{...each{{v}=>cvt{i64,v}, v0}}
def wdn = each{{v}=>cvt{f64,v}, widen{intp}}
tup{intp, wdn}
} else {
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it
def wdn = each{{v} => cvtWiden{XV, v}, ints}
def intp = {
if (tuplen{ints}==1) tupsel{0, ints}
else pair{ints}
}
tup{intp, wdn}
}
}
def int = tupsel{0, tmp}
def wdn = tupsel{1, tmp}
def conv{x} = tern{preserve_negative_zero, ty_u{x}, x}
def as = each{conv, v0}
def bs = each{conv, wdn}
def cond = {
if (tuplen{is}==1) anynePositive{...as, ...bs, M}
else ~homAll{tree_fold{&, each{==, as, bs}}}
}
if (cond) { # is any not an integer
if (B) { iCont=tupsel{0, is}; goto{case_B} } # if B, need to give an even more special result
else return{0xffff_ffff} # else, not integer => float
}
int
}
def acc = { if (tuplen{is}==2) r2; else r1 }
acc|= M{getAcc{type{acc}, int}}
}, {} => { r1 = half{r2,0}|half{r2,1} }}
}
def f = foldTotal{E, r1}
cast_i{u32, f}
}
}
}
'avx2_squeeze_i8' = squeeze{256, i8, 0, 0}
'avx2_squeeze_i16' = squeeze{256, i16, 0, 0}
'avx2_squeeze_i32' = squeeze{256, i32, 0, 0}
'avx2_squeeze_f64' = squeeze{256, f64, 0, 0}
'avx2_squeeze_numB'= squeeze{256, f64, 0, 1}
'avx2_squeeze_i8' = squeeze{arch_defvw, i8, 0, 0}
'avx2_squeeze_i16' = squeeze{arch_defvw, i16, 0, 0}
'avx2_squeeze_i32' = squeeze{arch_defvw, i32, 0, 0}
'avx2_squeeze_f64' = squeeze{arch_defvw, f64, 0, 0}
'avx2_squeeze_numB'= squeeze{arch_defvw, f64, 0, 1}
'avx2_squeeze_c16' = squeeze{256, u16, 1, 0}
'avx2_squeeze_c32' = squeeze{256, u32, 1, 0}
'avx2_squeeze_chrB'= squeeze{256, u64, 1, 1}
'avx2_squeeze_c16' = squeeze{arch_defvw, u16, 1, 0}
'avx2_squeeze_c32' = squeeze{arch_defvw, u32, 1, 0}
'avx2_squeeze_chrB'= squeeze{arch_defvw, u64, 1, 1}

View File

@ -18,7 +18,7 @@ spec ← specF64∾{⇐}∾"a⍉𝕨"∾(@+1114111-↕5)∾<"foo"
RandEl r.Range3
{𝕊: r.Range spec} # some special values
{𝕊: -(r.Range 2) × (¯5+r.Range 10) + 2r.Range 64} # random near-power-of-two number
{𝕊: -(r.Range 2) (¯5+r.Range 10) + 2r.Range 64} # random near-power-of-two number
{𝕊: @+01114111(¯5+r.Range 10) + 22+r.Range 20} # random near-power-of-two character
@ -59,11 +59,11 @@ Do ← { 𝕊:
}
¬ (arrv EEqual arr) (arrv EEqual sq) ((<sqt) tOut)?
•Out "Fail:"
•Out "Array: "Fmt arr
•Out "Squeezed: "Fmt sq
•Out "Testing variation "Fmt 𝕩
•Out "Got: "•Fmt Type sq
•Out "Expected one of: "Fmt tOut
•Out "Array: "Repr arr
•Out "Squeezed: "Repr sq
•Out "Testing variation "Repr 𝕩
•Out "Got: "(Type sq)" / "sqt
•Out "Expected one of: "Repr tOut
•Exit 1
;@
}¨ tIn

View File

@ -22,7 +22,12 @@ Do ← { 𝕊:
F { 𝕊:
p {r.Range 2? r.Range l; (¯1+l-)(r.Range 2) r.Range (2r.Range 10)l}
b 𝕩˙(p) a
! b EEqual Squeeze b # won't be acceptable with in-place squeezing, but that doesn't exist yet
¬b EEqual Squeeze b? # won't be acceptable with in-place squeezing, but that doesn't exist yet
•Out "Fail:"
•Out "Array: "•Repr b
•Out "Squeezed: "•Repr Squeeze b
•Exit 1
;@
}
F¨ (at5)specSpl