From 6baa96c1e340c697610e5c6deba333c05fa1dcbb Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 25 Jan 2023 01:25:28 +0200 Subject: [PATCH] NEON squeeze --- build/src/build.bqn | 2 +- src/builtins/squeeze.c | 16 +++-- src/singeli/src/avx.singeli | 2 +- src/singeli/src/base.singeli | 2 + src/singeli/src/neon.singeli | 4 +- src/singeli/src/squeeze.singeli | 120 +++++++++++++++++++++++--------- test/squeezeExact.bqn | 12 ++-- test/squeezeValid.bqn | 7 +- 8 files changed, 116 insertions(+), 49 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index b92184b6..7317a88f 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -514,7 +514,7 @@ cachedBin‿linkerCache ← { ⟨"src/jit/", "nvm.c"⟩ ⟨"src/utils/", "ryu.c", "utf.c", "hash.c", "file.c", "mut.c", "each.c", "bits.c"⟩ ⟩ - singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"‿"monarith"; 𝕩} ⟨ + singeliMap ← {po.arch≡"aarch64"? 𝕩/˜(1⊑¨𝕩)∊"cmp"‿"bits"‿"equal"‿"dyarith"‿"monarith"‿"squeeze"; 𝕩} ⟨ "src/builtins/arithm.c"‿"monarith", "src/core/stuff.c"‿"equal", "src/utils/mut.c"‿"copy", "src/utils/bits.c"‿"bits" "src/builtins/arithd.c"‿"dyarith", "src/builtins/cmp.c"‿"cmp", "src/builtins/squeeze.c"‿"squeeze" diff --git a/src/builtins/squeeze.c b/src/builtins/squeeze.c index 38837a47..6a8b9d8d 100644 --- a/src/builtins/squeeze.c +++ b/src/builtins/squeeze.c @@ -1,6 +1,6 @@ #include "../core.h" -#if SINGELI_X86_64 +#if SINGELI #define SINGELI_FILE squeeze #include "../utils/includeSingeli.h" #endif @@ -36,14 +36,14 @@ B num_squeeze(B x) { goto r_bit; } - #if !SINGELI_X86_64 + #if !SINGELI usz i = 0; #endif u32 or = 0; // using bitwise or as an approximate ⌈´ switch (xe) { default: UD; case el_bit: goto r_x; - #if SINGELI_X86_64 + #if SINGELI case el_i8: { or = avx2_squeeze_i8 (i8any_ptr (x), ia); if(or> 1) goto r_x; else goto mostBit; } case el_i16: { or = avx2_squeeze_i16(i16any_ptr(x), ia); if(or> I8_MAX) goto r_x; else goto mostI8; } case el_i32: { or = avx2_squeeze_i32(i32any_ptr(x), ia); if(or> I16_MAX) goto r_x; else goto mostI16; } @@ -69,7 +69,7 @@ B num_squeeze(B x) { B* xp = arr_bptr(x); if (xp==NULL) goto r_f; - #if SINGELI_X86_64 + #if SINGELI or = avx2_squeeze_numB(xp, ia); if (-2==(i32)or) goto r_x; if (-1==(i32)or) goto r_f64; @@ -109,11 +109,15 @@ B num_squeeze(B x) { B chr_squeeze(B x) { usz ia = IA(x); u8 xe = TI(x,elType); + if (ia==0) { + if (xe==el_c8) return x; + goto r_c8; + } usz i = 0; i32 or = 0; switch(xe) { default: UD; case el_c8: goto r_x; - #if SINGELI_X86_64 + #if SINGELI case el_c16: { u32 t = avx2_squeeze_c16(c16any_ptr(x), ia); if (t==0) goto r_c8; else goto r_x; } case el_c32: { u32 t = avx2_squeeze_c32(c32any_ptr(x), ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; else if (t==2) goto r_x; else UD; } #else @@ -138,7 +142,7 @@ B chr_squeeze(B x) { B* xp = arr_bptr(x); if (xp!=NULL) { - #if SINGELI_X86_64 + #if SINGELI u32 t = avx2_squeeze_chrB(xp, ia); if (t==0) goto r_c8; else if (t==1) goto r_c16; diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 816ae970..171526c9 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -98,7 +98,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} def half{x:T, i & w256{T} & knum{i}} = v_half{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} def half{x:T, i==0 & w256{T}} = v_half{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} def pair{a:T,b:T & width{T}==128} = v_dbl{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} -def pair{x} = pair{tupsel{0,x},tupsel{1,x}} +def pair{x & tuplen{x}==2} = pair{tupsel{0,x},tupsel{1,x}} def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x} def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 248fd138..93832ee4 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -124,6 +124,8 @@ def fold_max {...x} = assert{'fold_max not supported', show{...x}} def fold_min {...x} = assert{'fold_min not supported', show{...x}} def fold_add {...x} = assert{'fold_add not supported', show{...x}} def fold_addw{...x} = assert{'fold_addw not supported', show{...x}} +def narrowPair{...x} = assert{'narrowPair not supported', show{...x}} +def pair{...x} = assert{'pair not supported', show{...x}} def cvt{...x} = assert{'cvt not supported', show{...x}} def cvt{T, x:X & T==eltype{X}} = x def shuf{...x} = assert{'shuf not supported', show{...x}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 8f8b7b09..2d24abf4 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -123,9 +123,11 @@ def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}} def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{ty_half{v_half{X}}, ntyp0{'vmovn', X}, x} +def narrow{T, x:X & n128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T} { r = F{r, cast_i{E, extract{x, i}}, iota{vcount{T}}} }} - promote{u32, r} + each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} + r } -def fold{F, x:T & w128{T}} = { +def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { c:= x def EW = elwidth{T} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} @@ -43,7 +43,7 @@ def fold{F, x:T & w128{T}} = { if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} extract{c, 0} } -def fold{F, x:T & w256{T}} = fold{F, F{half{x, 0}, half{x, 1}}} +def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} def makeOptBranch{enable, F} = { if (enable) { @@ -57,11 +57,29 @@ def makeOptBranch{enable, F} = { } } +def cvtNarrow{T, x:X & width{T}==elwidth{X}} = cvt{T, x} +def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x} +def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x} +def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x} + squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { + assert{len>0} def bulk = vw / width{X} def XV = [bulk]X def E = tern{X==f64, u32, ty_u{X}} def EV = [bulk]E + + # fold with either Max or Bitwise Or, truncating/zero-extending to TE + def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}} + def foldTotal{TE, x:T & hasarch{'AARCH64'}} = { + if (elwidth{T}==64) { + if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} + else fold_max{narrow{TE, x}} + } else { + fold_max{x} + } + } + # show{XV, EV, CHR, B} xp:= *X~~x0 r1:= EV**0 @@ -76,7 +94,7 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { if (anynePositive{EV**((1<u32 and r2>=65536) return{2} if (X>u16 and r2>=256) return{1} 0 @@ -98,37 +116,73 @@ squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { return{0xffff_ffff} # float }} - maskedLoop{bulk, len, {i, M} => { - v0:= loadBatch{xp, i, XV} - def toint{x:T & isint{eltype{T}}} = x - def toint{flt:T & isfloat{eltype{T}}} = { - int:= narrow{i32, flt} - assert{width{type{int}} == width{XV}/2} # we'll be doing operations over it - - def conv{x} = tern{preserve_negative_zero, ty_u{x}, x} - - if (anynePositive{conv{flt}, conv{widen{T, int}}, M}) { # is any not an integer - if (B) { iCont=i; goto{case_B} } # if B, need to give an even more special result - else return{0xffff_ffff} # float - } - int - } - assert{XV == type{v0}} - v1:= toint{v0} # TODO do with an if instead - r1|= M{((EV ** ~E~~1) & EV~~v1) ^ EV~~(v1 >> (width{X}-1))} - }} + def getAcc{EV, x:T} = { + ((EV ** ~(eltype{EV})~~1) & EV~~x) ^ EV~~(x >> (elwidth{T}-1)) + } - promote{u32, fold{|, r1}} + if (isint{X}) { # i16, i32 + muLoop{bulk, 1, len, {is, M} => { + def v0 = loadBatch{xp, is, XV} + r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}} + }} + } else { # f64 + def EV2 = v_dbl{EV} + r2:= EV2**0 + muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => { + def v0 = loadBatch{xp, is, XV} + + def int = { + def tmp = { + if (hasarch{'AARCH64'} and tuplen{is}==2) { + def intp = narrowPair{...each{{v}=>cvt{i64,v}, v0}} + def wdn = each{{v}=>cvt{f64,v}, widen{intp}} + tup{intp, wdn} + } else { + def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0} + assert{vcount{type{tupsel{0,ints}}} == bulk} # we'll be doing operations over it + def wdn = each{{v} => cvtWiden{XV, v}, ints} + def intp = { + if (tuplen{ints}==1) tupsel{0, ints} + else pair{ints} + } + tup{intp, wdn} + } + } + def int = tupsel{0, tmp} + def wdn = tupsel{1, tmp} + + def conv{x} = tern{preserve_negative_zero, ty_u{x}, x} + def as = each{conv, v0} + def bs = each{conv, wdn} + def cond = { + if (tuplen{is}==1) anynePositive{...as, ...bs, M} + else ~homAll{tree_fold{&, each{==, as, bs}}} + } + if (cond) { # is any not an integer + if (B) { iCont=tupsel{0, is}; goto{case_B} } # if B, need to give an even more special result + else return{0xffff_ffff} # else, not integer => float + } + int + } + def acc = { if (tuplen{is}==2) r2; else r1 } + + acc|= M{getAcc{type{acc}, int}} + }, {} => { r1 = half{r2,0}|half{r2,1} }} + } + + + def f = foldTotal{E, r1} + cast_i{u32, f} } } } -'avx2_squeeze_i8' = squeeze{256, i8, 0, 0} -'avx2_squeeze_i16' = squeeze{256, i16, 0, 0} -'avx2_squeeze_i32' = squeeze{256, i32, 0, 0} -'avx2_squeeze_f64' = squeeze{256, f64, 0, 0} -'avx2_squeeze_numB'= squeeze{256, f64, 0, 1} +'avx2_squeeze_i8' = squeeze{arch_defvw, i8, 0, 0} +'avx2_squeeze_i16' = squeeze{arch_defvw, i16, 0, 0} +'avx2_squeeze_i32' = squeeze{arch_defvw, i32, 0, 0} +'avx2_squeeze_f64' = squeeze{arch_defvw, f64, 0, 0} +'avx2_squeeze_numB'= squeeze{arch_defvw, f64, 0, 1} -'avx2_squeeze_c16' = squeeze{256, u16, 1, 0} -'avx2_squeeze_c32' = squeeze{256, u32, 1, 0} -'avx2_squeeze_chrB'= squeeze{256, u64, 1, 1} \ No newline at end of file +'avx2_squeeze_c16' = squeeze{arch_defvw, u16, 1, 0} +'avx2_squeeze_c32' = squeeze{arch_defvw, u32, 1, 0} +'avx2_squeeze_chrB'= squeeze{arch_defvw, u64, 1, 1} \ No newline at end of file diff --git a/test/squeezeExact.bqn b/test/squeezeExact.bqn index 00f92f74..123616a8 100644 --- a/test/squeezeExact.bqn +++ b/test/squeezeExact.bqn @@ -18,7 +18,7 @@ spec ← specF64∾{⇐}∾"a⍉𝕨"∾(@+1114111-↕5)∾<"foo" RandEl ← r.Range∘3◶⟨ {𝕊: r.Range∘≠⊸⊑ spec} # some special values - {𝕊: -⍟(r.Range 2) × (¯5+r.Range 10) + 2⋆r.Range 64} # random near-power-of-two number + {𝕊: -⍟(r.Range 2) (¯5+r.Range 10) + 2⋆r.Range 64} # random near-power-of-two number {𝕊: @+0⌈1114111⌊(¯5+r.Range 10) + 2⋆2+r.Range 20} # random near-power-of-two character ⟩ @@ -59,11 +59,11 @@ Do ← { 𝕊: } ¬ (arrv EEqual arr) ∧ (arrv EEqual sq) ∧ (⊑(