NEON squeeze
This commit is contained in:
parent
88410c02e5
commit
6baa96c1e3
@ -514,7 +514,7 @@ cachedBin‿linkerCache ← {
|
||||
⟨"src/jit/", "nvm.c"⟩
|
||||
⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩
|
||||
⟩
|
||||
singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"‿"monarith"; 𝕩} ⟨
|
||||
singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"‿"monarith"‿"squeeze"; 𝕩} ⟨
|
||||
"src/builtins/arithm.c"‿"monarith",
|
||||
"src/core/stuff.c"‿"equal", "src/utils/mut.c"‿"copy", "src/utils/bits.c"‿"bits"
|
||||
"src/builtins/arithd.c"‿"dyarith", "src/builtins/cmp.c"‿"cmp", "src/builtins/squeeze.c"‿"squeeze"
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#include "../core.h"
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
#define SINGELI_FILE squeeze
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
@ -36,14 +36,14 @@ B num_squeeze(B x) {
|
||||
goto r_bit;
|
||||
}
|
||||
|
||||
#if !SINGELI_X86_64
|
||||
#if !SINGELI
|
||||
usz i = 0;
|
||||
#endif
|
||||
|
||||
u32 or = 0; // using bitwise or as an approximate ⌈´
|
||||
switch (xe) { default: UD;
|
||||
case el_bit: goto r_x;
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; }
|
||||
case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; }
|
||||
case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; }
|
||||
@ -69,7 +69,7 @@ B num_squeeze(B x) {
|
||||
B* xp = arr_bptr(x);
|
||||
if (xp==NULL) goto r_f;
|
||||
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
or = avx2_squeeze_numB(xp, ia);
|
||||
if (-2==(i32)or) goto r_x;
|
||||
if (-1==(i32)or) goto r_f64;
|
||||
@ -109,11 +109,15 @@ B num_squeeze(B x) {
|
||||
B chr_squeeze(B x) {
|
||||
usz ia = IA(x);
|
||||
u8 xe = TI(x,elType);
|
||||
if (ia==0) {
|
||||
if (xe==el_c8) return x;
|
||||
goto r_c8;
|
||||
}
|
||||
usz i = 0;
|
||||
i32 or = 0;
|
||||
switch(xe) { default: UD;
|
||||
case el_c8: goto r_x;
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; }
|
||||
case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; }
|
||||
#else
|
||||
@ -138,7 +142,7 @@ B chr_squeeze(B x) {
|
||||
|
||||
B* xp = arr_bptr(x);
|
||||
if (xp!=NULL) {
|
||||
#if SINGELI_X86_64
|
||||
#if SINGELI
|
||||
u32 t = avx2_squeeze_chrB(xp, ia);
|
||||
if (t==0) goto r_c8;
|
||||
else if (t==1) goto r_c16;
|
||||
|
||||
@ -98,7 +98,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
|
||||
def half{x:T, i & w256{T} & knum{i}} = v_half{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
|
||||
def half{x:T, i==0 & w256{T}} = v_half{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
|
||||
def pair{a:T,b:T & width{T}==128} = v_dbl{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
|
||||
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
|
||||
def pair{x & tuplen{x}==2} = pair{tupsel{0,x},tupsel{1,x}}
|
||||
|
||||
def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x}
|
||||
def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x}
|
||||
|
||||
@ -124,6 +124,8 @@ def fold_max {...x} = assert{'fold_max not supported', show{...x}}
|
||||
def fold_min {...x} = assert{'fold_min not supported', show{...x}}
|
||||
def fold_add {...x} = assert{'fold_add not supported', show{...x}}
|
||||
def fold_addw{...x} = assert{'fold_addw not supported', show{...x}}
|
||||
def narrowPair{...x} = assert{'narrowPair not supported', show{...x}}
|
||||
def pair{...x} = assert{'pair not supported', show{...x}}
|
||||
def cvt{...x} = assert{'cvt not supported', show{...x}}
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
def shuf{...x} = assert{'shuf not supported', show{...x}}
|
||||
|
||||
@ -123,9 +123,11 @@ def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
|
||||
|
||||
def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}}
|
||||
def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{ty_half{v_half{X}}, ntyp0{'vmovn', X}, x}
|
||||
def narrow{T, x:X & n128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
|
||||
|
||||
def narrowUpper{lowRes:L, x:X & n64{L} & n128{X} & isint{eltype{X}} & ty_dbl{v_dbl{L}}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
|
||||
def narrowPair{a, b} = narrowUpper{narrow{ty_half{eltype{type{a}}}, a}, b}
|
||||
def narrowPair{a:T, b:T} = narrowUpper{narrow{ty_half{eltype{T}}, a}, b}
|
||||
def narrowPair{a:T, b:T & isint{eltype{T}}} = packLo{a, b}
|
||||
|
||||
def widenUpper{x:T & n128{T} & isint{eltype{T}}} = emit{ty_dbl{T}, ntyp0{'vmovl_high', T}, x}
|
||||
def widen{x:T & n128{T}} = tup{widen{ty_dbl{T}, x}, widenUpper{x}}
|
||||
|
||||
@ -31,10 +31,10 @@ def fold{F, x:T} = {
|
||||
show{'WARNING: using fallback fold'}
|
||||
def E = eltype{T}
|
||||
r:E = 0
|
||||
each{{i} => { r = F{r, cast_i{E, extract{x, i}}, iota{vcount{T}}} }}
|
||||
promote{u32, r}
|
||||
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
|
||||
r
|
||||
}
|
||||
def fold{F, x:T & w128{T}} = {
|
||||
def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = {
|
||||
c:= x
|
||||
def EW = elwidth{T}
|
||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||
@ -43,7 +43,7 @@ def fold{F, x:T & w128{T}} = {
|
||||
if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
|
||||
extract{c, 0}
|
||||
}
|
||||
def fold{F, x:T & w256{T}} = fold{F, F{half{x, 0}, half{x, 1}}}
|
||||
def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
|
||||
|
||||
def makeOptBranch{enable, F} = {
|
||||
if (enable) {
|
||||
@ -57,11 +57,29 @@ def makeOptBranch{enable, F} = {
|
||||
}
|
||||
}
|
||||
|
||||
def cvtNarrow{T, x:X & width{T}==elwidth{X}} = cvt{T, x}
|
||||
def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
|
||||
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
|
||||
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
|
||||
|
||||
squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
assert{len>0}
|
||||
def bulk = vw / width{X}
|
||||
def XV = [bulk]X
|
||||
def E = tern{X==f64, u32, ty_u{X}}
|
||||
def EV = [bulk]E
|
||||
|
||||
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
|
||||
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
|
||||
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
|
||||
if (elwidth{T}==64) {
|
||||
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
|
||||
else fold_max{narrow{TE, x}}
|
||||
} else {
|
||||
fold_max{x}
|
||||
}
|
||||
}
|
||||
|
||||
# show{XV, EV, CHR, B}
|
||||
xp:= *X~~x0
|
||||
r1:= EV**0
|
||||
@ -76,7 +94,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
|
||||
}
|
||||
}}
|
||||
r2:= cast_i{u32, fold{|, r1}}
|
||||
r2:= foldTotal{u32, r1}
|
||||
if (X>u32 and r2>=65536) return{2}
|
||||
if (X>u16 and r2>=256) return{1}
|
||||
0
|
||||
@ -98,37 +116,73 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
return{0xffff_ffff} # float
|
||||
}}
|
||||
|
||||
maskedLoop{bulk, len, {i, M} => {
|
||||
v0:= loadBatch{xp, i, XV}
|
||||
def toint{x:T & isint{eltype{T}}} = x
|
||||
def toint{flt:T & isfloat{eltype{T}}} = {
|
||||
int:= narrow{i32, flt}
|
||||
assert{width{type{int}} == width{XV}/2} # we'll be doing operations over it
|
||||
|
||||
def conv{x} = tern{preserve_negative_zero, ty_u{x}, x}
|
||||
|
||||
if (anynePositive{conv{flt}, conv{widen{T, int}}, M}) { # is any not an integer
|
||||
if (B) { iCont=i; goto{case_B} } # if B, need to give an even more special result
|
||||
else return{0xffff_ffff} # float
|
||||
}
|
||||
int
|
||||
}
|
||||
assert{XV == type{v0}}
|
||||
v1:= toint{v0} # TODO do with an if instead
|
||||
r1|= M{((EV ** ~E~~1) & EV~~v1) ^ EV~~(v1 >> (width{X}-1))}
|
||||
}}
|
||||
def getAcc{EV, x:T} = {
|
||||
((EV ** ~(eltype{EV})~~1) & EV~~x) ^ EV~~(x >> (elwidth{T}-1))
|
||||
}
|
||||
|
||||
promote{u32, fold{|, r1}}
|
||||
if (isint{X}) { # i16, i32
|
||||
muLoop{bulk, 1, len, {is, M} => {
|
||||
def v0 = loadBatch{xp, is, XV}
|
||||
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
|
||||
}}
|
||||
} else { # f64
|
||||
def EV2 = v_dbl{EV}
|
||||
r2:= EV2**0
|
||||
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
|
||||
def v0 = loadBatch{xp, is, XV}
|
||||
|
||||
def int = {
|
||||
def tmp = {
|
||||
if (hasarch{'AARCH64'} and tuplen{is}==2) {
|
||||
def intp = narrowPair{...each{{v}=>cvt{i64,v}, v0}}
|
||||
def wdn = each{{v}=>cvt{f64,v}, widen{intp}}
|
||||
tup{intp, wdn}
|
||||
} else {
|
||||
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
|
||||
assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it
|
||||
def wdn = each{{v} => cvtWiden{XV, v}, ints}
|
||||
def intp = {
|
||||
if (tuplen{ints}==1) tupsel{0, ints}
|
||||
else pair{ints}
|
||||
}
|
||||
tup{intp, wdn}
|
||||
}
|
||||
}
|
||||
def int = tupsel{0, tmp}
|
||||
def wdn = tupsel{1, tmp}
|
||||
|
||||
def conv{x} = tern{preserve_negative_zero, ty_u{x}, x}
|
||||
def as = each{conv, v0}
|
||||
def bs = each{conv, wdn}
|
||||
def cond = {
|
||||
if (tuplen{is}==1) anynePositive{...as, ...bs, M}
|
||||
else ~homAll{tree_fold{&, each{==, as, bs}}}
|
||||
}
|
||||
if (cond) { # is any not an integer
|
||||
if (B) { iCont=tupsel{0, is}; goto{case_B} } # if B, need to give an even more special result
|
||||
else return{0xffff_ffff} # else, not integer => float
|
||||
}
|
||||
int
|
||||
}
|
||||
def acc = { if (tuplen{is}==2) r2; else r1 }
|
||||
|
||||
acc|= M{getAcc{type{acc}, int}}
|
||||
}, {} => { r1 = half{r2,0}|half{r2,1} }}
|
||||
}
|
||||
|
||||
|
||||
def f = foldTotal{E, r1}
|
||||
cast_i{u32, f}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
'avx2_squeeze_i8' = squeeze{256, i8, 0, 0}
|
||||
'avx2_squeeze_i16' = squeeze{256, i16, 0, 0}
|
||||
'avx2_squeeze_i32' = squeeze{256, i32, 0, 0}
|
||||
'avx2_squeeze_f64' = squeeze{256, f64, 0, 0}
|
||||
'avx2_squeeze_numB'= squeeze{256, f64, 0, 1}
|
||||
'avx2_squeeze_i8' = squeeze{arch_defvw, i8, 0, 0}
|
||||
'avx2_squeeze_i16' = squeeze{arch_defvw, i16, 0, 0}
|
||||
'avx2_squeeze_i32' = squeeze{arch_defvw, i32, 0, 0}
|
||||
'avx2_squeeze_f64' = squeeze{arch_defvw, f64, 0, 0}
|
||||
'avx2_squeeze_numB'= squeeze{arch_defvw, f64, 0, 1}
|
||||
|
||||
'avx2_squeeze_c16' = squeeze{256, u16, 1, 0}
|
||||
'avx2_squeeze_c32' = squeeze{256, u32, 1, 0}
|
||||
'avx2_squeeze_chrB'= squeeze{256, u64, 1, 1}
|
||||
'avx2_squeeze_c16' = squeeze{arch_defvw, u16, 1, 0}
|
||||
'avx2_squeeze_c32' = squeeze{arch_defvw, u32, 1, 0}
|
||||
'avx2_squeeze_chrB'= squeeze{arch_defvw, u64, 1, 1}
|
||||
@ -18,7 +18,7 @@ spec ← specF64∾{⇐}∾"a⍉𝕨"∾(@+1114111-↕5)∾<"foo"
|
||||
|
||||
RandEl ← r.Range∘3◶⟨
|
||||
{𝕊: r.Range∘≠⊸⊑ spec} # some special values
|
||||
{𝕊: -⍟(r.Range 2) × (¯5+r.Range 10) + 2⋆r.Range 64} # random near-power-of-two number
|
||||
{𝕊: -⍟(r.Range 2) (¯5+r.Range 10) + 2⋆r.Range 64} # random near-power-of-two number
|
||||
{𝕊: @+0⌈1114111⌊(¯5+r.Range 10) + 2⋆2+r.Range 20} # random near-power-of-two character
|
||||
⟩
|
||||
|
||||
@ -59,11 +59,11 @@ Do ← { 𝕊:
|
||||
}
|
||||
¬ (arrv EEqual arr) ∧ (arrv EEqual sq) ∧ (⊑(<sqt) ∊ tOut)?
|
||||
•Out "Fail:"
|
||||
•Out "Array: "∾•Fmt arr
|
||||
•Out "Squeezed: "∾•Fmt sq
|
||||
•Out "Testing variation "∾•Fmt 𝕩
|
||||
•Out "Got: "∾•Fmt Type sq
|
||||
•Out "Expected one of: "∾•Fmt tOut
|
||||
•Out "Array: "∾•Repr arr
|
||||
•Out "Squeezed: "∾•Repr sq
|
||||
•Out "Testing variation "∾•Repr 𝕩
|
||||
•Out "Got: "∾(Type sq)∾" / "∾sqt
|
||||
•Out "Expected one of: "∾•Repr tOut
|
||||
•Exit 1
|
||||
;@
|
||||
}¨ tIn
|
||||
|
||||
@ -22,7 +22,12 @@ Do ← { 𝕊:
|
||||
F ← { 𝕊:
|
||||
p ← {r.Range 2? r.Range l; (¯1+l-⊢)⍟(r.Range 2) r.Range (2⋆r.Range 10)⌊l}
|
||||
b ← 𝕩˙⌾(p⊸⊑) a
|
||||
! b EEqual Squeeze b # won't be acceptable with in-place squeezing, but that doesn't exist yet
|
||||
¬b EEqual Squeeze b? # won't be acceptable with in-place squeezing, but that doesn't exist yet
|
||||
•Out "Fail:"
|
||||
•Out "Array: "∾•Repr b
|
||||
•Out "Squeezed: "∾•Repr Squeeze b
|
||||
•Exit 1
|
||||
;@
|
||||
}
|
||||
|
||||
F¨ (at≥5)⊑specSpl
|
||||
|
||||
Loading…
Reference in New Issue
Block a user