From 464a53463d9165773f7f56fb82c7300d03886e6a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 6 Jun 2023 20:04:39 -0400 Subject: [PATCH 01/34] Generic Singeli version of 32-bit binary search --- build/src/build.bqn | 1 + src/builtins/grade.h | 6 +++++- src/builtins/sort.c | 5 +++++ src/singeli/src/bins.singeli | 26 ++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/singeli/src/bins.singeli diff --git a/build/src/build.bqn b/build/src/build.bqn index 431342de..13a2818f 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -634,6 +634,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/sort.c"‿"bins" "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", diff --git a/src/builtins/grade.h b/src/builtins/grade.h index d3878017..fffbe7f0 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -370,13 +370,17 @@ B GRADE_CAT(c2)(B t, B w, B x) { for (i64 i = 0; i < (i64)wia-1; i++) if (wi[i] GRADE_UD(>,<) wi[i+1]) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); FL_SET(w, fl); } - + + #if SINGELI + si_bins[2*2 + GRADE_UD(0,1)](wi, wia, xi, xia, rp); + #else for (usz i = 0; i < xia; i++) { i32 c = xi[i]; i32 *s = wi-1; for (usz l = wia+1, h; (h=l/2)>0; l-=h) { i32* m = s+h; if (!(c LT *m)) s = m; } rp[i] = s - (wi-1); } + #endif } else { gen:; SGetU(x) diff --git a/src/builtins/sort.c b/src/builtins/sort.c index 2684600d..e803ba21 100644 --- a/src/builtins/sort.c +++ b/src/builtins/sort.c @@ -3,6 +3,11 @@ // Defines Sort, Grade, and Bins +#if SINGELI + #define SINGELI_FILE bins + #include "../utils/includeSingeli.h" +#endif + #define CAT0(A,B) A##_##B #define CAT(A,B) CAT0(A,B) typedef struct BI32p { B k; i32 v; } BI32p; diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli new file mode 100644 index 00000000..a659f31e --- /dev/null +++ b/src/singeli/src/bins.singeli @@ -0,0 +1,26 @@ +include './base' +include 'util/tup' + +def bin_search{lt, w, wn, x, n, res} = { + ws := w - 1 + l0 := wn + 1 + @for (x, res over n) { + s := ws + l := l0; h := undefined{u64} + while ((h=l/2) > 0) { + m := s + h + if (not lt{x, load{m}}) s = m + l -= h + } + res = cast_i{i32, s - ws} + } +} + +fn bins_branchless{T, up}(w:*void, wn:u64, x:*void, xn:u64, r:*i32) : void = { + bin_search{if (up) <; else >, *T~~w, wn, *T~~x, xn, r} +} + +exportT{ + 'si_bins', + join{table{bins_branchless, tup{i8,i16,i32,f64}, tup{1,0}}} +} From ee84f9fe1faef1a8abe40134fabe0da1b8c1cc41 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 7 Jun 2023 17:30:17 -0400 Subject: [PATCH 02/34] 4-way branchless binary search unrolling --- src/singeli/src/bins.singeli | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index a659f31e..f7a24b53 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -4,16 +4,24 @@ include 'util/tup' def bin_search{lt, w, wn, x, n, res} = { ws := w - 1 l0 := wn + 1 - @for (x, res over n) { - s := ws - l := l0; h := undefined{u64} + # Take a list of indices in x/res to allow unrolling + def search{inds} = { + xs:= each{bind{load,x}, inds} # Values + ss:= each{{_}=>ws, inds} # Initial lower bound + l := l0; h := undefined{u64} # Interval size l, same for all values while ((h=l/2) > 0) { - m := s + h - if (not lt{x, load{m}}) s = m + # Branchless update + def bin1{s, x, m} = { if (not lt{x, load{m}}) s = m } + each{bin1, ss, xs, each{bind{+,h}, ss}} l -= h } - res = cast_i{i32, s - ws} + each{{r,s} => store{res, r, cast_i{i32, s - ws}}, inds, ss} } + # Unroll by 4 then 1 + def search{i, k} = search{each{bind{+,i}, iota{k}}} + j:u64 = 0 + def searches{k} = { while (j+k <= n) { search{j, k}; j+=k } } + each{searches, tup{4, 1}} } fn bins_branchless{T, up}(w:*void, wn:u64, x:*void, xn:u64, r:*i32) : void = { From 5424c57b7ef5edded9951f2ce9738573ec621080 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 7 Jun 2023 17:48:17 -0400 Subject: [PATCH 03/34] Use Singeli float binary search --- src/builtins/grade.h | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index fffbe7f0..b6ff9bf4 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -350,7 +350,22 @@ B GRADE_CAT(c2)(B t, B w, B x) { if (LIKELY(we,<) wi[i+1]) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); + FL_SET(w, fl); + } + si_bins[3*2 + GRADE_UD(0,1)](wi, wia, xi, xia, rp); + goto done; + } + #endif + goto gen; + } w=toI32Any(w); x=toI32Any(x); } else { for (u64 i=0; i Date: Mon, 3 Jul 2023 16:01:52 -0400 Subject: [PATCH 04/34] AVX2 binary search on one lane of i8 (unused for now) --- src/singeli/src/bins.singeli | 53 +++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index f7a24b53..b5c7dfa8 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,7 +1,46 @@ include './base' +if (hasarch{'AVX2'}) { + include './sse' + include './avx' + include './avx2' +} +include './mask' include 'util/tup' -def bin_search{lt, w, wn, x, n, res} = { +def ceil_log2{n:u64} = 64 - clz{n+1} + +def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8 & hasarch{'AVX2'}} = { + def T = i8; def I = u8 + def lt = if (up) <; else > + def pre = (if (up) minvalue else maxvalue){T} + def vl = 32 + def V = [vl]T; def H = [vl/2]T + def U = [vl]I + log := ceil_log2{wn-1} + l := 1<> 1) # Type doesn't matter but u8 would fail + } + store{*U~~(res+j), 0, s - off} + j += vl + } + if (j != n) { j = n-vl; goto{tail} } +} + +def bin_search_branchless{up, w, wn, x, n, res} = { + def lt = if (up) <; else > ws := w - 1 l0 := wn + 1 # Take a list of indices in x/res to allow unrolling @@ -24,11 +63,17 @@ def bin_search{lt, w, wn, x, n, res} = { each{searches, tup{4, 1}} } -fn bins_branchless{T, up}(w:*void, wn:u64, x:*void, xn:u64, r:*i32) : void = { - bin_search{if (up) <; else >, *T~~w, wn, *T~~x, xn, r} +fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, r:*i32) : void = { + if (hasarch{'AVX2'} and T==i8 and wn<16 and xn>=32) { + bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~r} + # Slow and useless: need to allocate i8 result + j:=xn; while (j > 0) { --j; store{r, j, cast_i{i32, load{*i8~~r, j}}} } + } else { + bin_search_branchless{up, *T~~w, wn, *T~~x, xn, r} + } } exportT{ 'si_bins', - join{table{bins_branchless, tup{i8,i16,i32,f64}, tup{1,0}}} + join{table{bins, tup{i8,i16,i32,f64}, tup{1,0}}} } From ec9b875503b60c467b27a9f0387b1f66d32cbdde Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 3 Jul 2023 17:25:26 -0400 Subject: [PATCH 05/34] Separate is-sorted check from Bins implementation --- src/builtins/grade.h | 56 +++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index b6ff9bf4..d161938a 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -326,6 +326,42 @@ B GRADE_CAT(c1)(B t, B x) { } +bool CAT(isSorted,GRADE_UD(Up,Down))(B x) { + assert(isArr(x) && RNK(x)==1); // TODO extend to >=1 + usz xia = IA(x); + if (xia <= 1) return 1; + + #define CMP(TEST) \ + for (usz i=1; i,<) xp[i]) } + switch (TI(x,elType)) { default: UD; + CASE(i8) CASE(i16) CASE(i32) CASE(f64) + CASE(c8) CASE(c16) CASE(c32) + case el_bit: { + #define HI GRADE_UD(1,0) + u64* xp = bitarr_ptr(x); + u64 i = bit_find(xp, xia, HI); + usz iw = i/64; + u64 m = ~(u64)0; + u64 d = GRADE_UD(,~)xp[iw] ^ (m<<(i%64)); + if (iw == xia/64) return (d &~ (m<<(xia%64))) == 0; + if (d) return 0; + usz o = iw + 1; + usz l = xia - 64*o; + return (bit_find(xp+o, l, !HI) == l); + #undef HI + } + case el_B: { + B* xp = TO_BPTR(x); + CMP(compare(xp[i-1], xp[i]) GRADE_UD(>,<) 0) + } + } + #undef CASE + #undef CMP +} + B GRADE_CAT(c2)(B t, B w, B x) { if (isAtm(w) || RNK(w)==0) thrM(GRADE_CHR": 𝕨 must have rank≥1"); if (isAtm(x)) x = m_unit(x); @@ -343,9 +379,14 @@ B GRADE_CAT(c2)(B t, B w, B x) { u8 xe = TI(x,elType); usz xia = IA(x); if (wia>I32_MAX-10) thrM(GRADE_CHR": 𝕨 too big"); - i32* rp; B r = m_i32arrc(&rp, x); u8 fl = GRADE_UD(fl_asc,fl_dsc); + if (CHECK_VALID && !FL_HAS(w,fl)) { + if (!CAT(isSorted,GRADE_UD(Up,Down))(w)) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); + FL_SET(w, fl); + } + + i32* rp; B r = m_i32arrc(&rp, x); if (LIKELY(we,<) wi[i+1]) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); - FL_SET(w, fl); - } si_bins[3*2 + GRADE_UD(0,1)](wi, wia, xi, xia, rp); goto done; } @@ -381,10 +418,6 @@ B GRADE_CAT(c2)(B t, B w, B x) { } i32* wi = tyany_ptr(w); i32* xi = tyany_ptr(x); - if (CHECK_VALID && !FL_HAS(w,fl)) { - for (i64 i = 0; i < (i64)wia-1; i++) if (wi[i] GRADE_UD(>,<) wi[i+1]) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); - FL_SET(w, fl); - } #if SINGELI si_bins[2*2 + GRADE_UD(0,1)](wi, wia, xi, xia, rp); @@ -402,11 +435,6 @@ B GRADE_CAT(c2)(B t, B w, B x) { SLOW2("𝕨"GRADE_CHR"𝕩", w, x); B* wp = TO_BPTR(w); - if (CHECK_VALID && !FL_HAS(w,fl)) { - for (i64 i = 0; i < (i64)wia-1; i++) if (compare(wp[i], wp[i+1]) GRADE_UD(>,<) 0) thrM(GRADE_CHR": 𝕨 must be sorted"GRADE_UD(," in descending order")); - FL_SET(w, fl); - } - for (usz i = 0; i < xia; i++) { B c = GetU(x,i); usz s = 0, e = wia+1; From 4f93b5849b0a54972436c22b88d7fc9f2462c78a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 3 Jul 2023 18:14:30 -0400 Subject: [PATCH 06/34] =?UTF-8?q?Fix=20shape=20of=20char=E2=8D=8Bnum=20res?= =?UTF-8?q?ult=20(was=20deshaped)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/grade.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index d161938a..4b905100 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -387,7 +387,7 @@ B GRADE_CAT(c2)(B t, B w, B x) { } i32* rp; B r = m_i32arrc(&rp, x); - + if (LIKELY(we Date: Mon, 3 Jul 2023 17:51:05 -0400 Subject: [PATCH 07/34] In bins, promote to smallest common int type instead of i32 --- src/builtins/grade.h | 35 ++++++++++++++++++++--------------- src/singeli/src/bins.singeli | 1 + 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 4b905100..10e17f13 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -378,6 +378,10 @@ B GRADE_CAT(c2)(B t, B w, B x) { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); + if (wia==0 | xia==0) { + Arr* ra=allZeroes(xia); arr_shCopy(ra, x); + decG(w); decG(x); return taga(ra); + } if (wia>I32_MAX-10) thrM(GRADE_CHR": 𝕨 too big"); u8 fl = GRADE_UD(fl_asc,fl_dsc); @@ -391,19 +395,16 @@ B GRADE_CAT(c2)(B t, B w, B x) { if (LIKELY(wexe? we : xe; + if (ze==el_bit) ze = el_i8; + if (ze > we) { switch (ze) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; case el_f64:w=toF64Any(w);break; } } + if (ze > xe) { switch (ze) { default:UD; case el_i8:x=toI8Any(x);break; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; case el_f64:x=toF64Any(x);break; } } + we = ze; + #else + if (!elInt(we) | !elInt(xe)) goto gen; w=toI32Any(w); x=toI32Any(x); + #endif } else { for (u64 i=0; i 0} def T = i8; def I = u8 def lt = if (up) <; else > def pre = (if (up) minvalue else maxvalue){T} From 302d637129a4e80f2503fce7f218f6b1200b8ce9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 4 Jul 2023 11:15:58 -0400 Subject: [PATCH 08/34] Allocate binary search result in Singeli, using i8 for vectors --- src/builtins/grade.h | 12 ++++++++---- src/builtins/sort.c | 4 ++++ src/singeli/src/bins.singeli | 24 +++++++++++++++++------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 10e17f13..1021f477 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -390,8 +390,7 @@ B GRADE_CAT(c2)(B t, B w, B x) { FL_SET(w, fl); } - i32* rp; B r = m_i32arrc(&rp, x); - + B r; if (LIKELY(we=32) { - bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~r} - # Slow and useless: need to allocate i8 result - j:=xn; while (j > 0) { --j; store{r, j, cast_i{i32, load{*i8~~r, j}}} } - } else { - bin_search_branchless{up, *T~~w, wn, *T~~x, xn, r} +def B = u64 +fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, xb:B) : B = { + def alloc{T,ts} = { + u:*u64 = tup{0} + rpp:* *T = (* *T)~~u + r := emit{B, merge{'m_',ts,'arrc_wrapu'}, rpp, xb} + rp := load{rpp,0} + tup{r, rp} } + r := undefined{B} + if (hasarch{'AVX2'} and T==i8 and wn<16 and xn>=32) { + def {rt, rp} = alloc{i8, 'i8'}; r = rt + bin_search_vec{up, *T~~w, wn, *T~~x, xn, rp} + } else { + def {rt, rp} = alloc{i32, 'i32'}; r = rt + bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp} + } + r } exportT{ From 7161689196c24f17c1c436e1a31deca9ec70055d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 4 Jul 2023 15:51:47 -0400 Subject: [PATCH 09/34] Fix bin_search_vec over-writing --- src/singeli/src/bins.singeli | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 332aa52e..73b7a96a 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -25,8 +25,9 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8 & hasarch{'AVX2'}} = { wv := double{homBlend{load{*H~~(w-gap), 0}, H**pre, maskOf{H,gap}}} h0 := U**(l/2) j:u64 = 0 - def tail = setlabel{} - while (j < n) { + def end = makelabel{} + assert{n >= vl} + while (1) { xv:= load{*V~~(x+j), 0} s := U**0 h := h0 @@ -36,8 +37,9 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8 & hasarch{'AVX2'}} = { } store{*U~~(res+j), 0, s - off} j += vl + if (j > n-vl) { if (j == n) goto{end}; j = n-vl } } - if (j != n) { j = n-vl; goto{tail} } + setlabel{end} } def bin_search_branchless{up, w, wn, x, n, res} = { From ed9e8b4057d2dc5166ee77d10e48d6c90fef2da6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 4 Jul 2023 21:38:08 -0400 Subject: [PATCH 10/34] Table-based 1-byte Bins implementations, including AVX2 --- src/singeli/src/bins.singeli | 123 ++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 73b7a96a..1f2ad32d 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -7,9 +7,31 @@ if (hasarch{'AVX2'}) { include './mask' include 'util/tup' +def for_backwards{vars,begin,end,iter} = { + i:u64 = end + while (i > begin) { + --i + iter{i, vars} + } +} +def for_vec_overlap{vl}{vars,begin==0,n,iter} = { + assert{n >= vl} + def end = makelabel{} + j:u64 = 0 + while (1) { + iter{j, vars} + j += vl + if (j > n-vl) { if (j == n) goto{end}; j = n-vl } + } + setlabel{end} +} + def ceil_log2{n:u64} = 64 - clz{n+1} -def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8 & hasarch{'AVX2'}} = { +# Shift as u16, since x86 is missing 8-bit shifts +def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) + +def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { assert{wn > 0} def T = i8; def I = u8 def lt = if (up) <; else > @@ -17,29 +39,82 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8 & hasarch{'AVX2'}} = { def vl = 32 def V = [vl]T; def H = [vl/2]T def U = [vl]I - log := ceil_log2{wn-1} - l := 1<= vl} - while (1) { - xv:= load{*V~~(x+j), 0} - s := U**0 - h := h0 - @for (promote{u64,log}) { - s |= h &~ lt{xv, sel{H, wv, s | h}} - h = U~~(([width{V}/16]u16~~h) >> 1) # Type doesn't matter but u8 would fail - } - store{*U~~(res+j), 0, s - off} - j += vl - if (j > n-vl) { if (j == n) goto{end}; j = n-vl } + def getsel{h:(H)} = { + v := pair{h,h} + {i} => sel{H, v, i} + } + if (hasarch{'AVX2'} and wn < 16) { + log := ceil_log2{wn-1} + l := 1< V**0} + } + # Popcount on 8-bit values + def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } + def sum4 = getsel{make{H, sums{vl/2}}} + bot4 := U**0x0f + def vpopc{v} = { + def s{b} = sum4{b&bot4} + s{shr16{v,4}} + s{v} + } + # 32-byte select + vtop := U**(vl/2) + def getsel{v & width{type{v}}==256} = { + hs := each{bind{shuf, [4]u64, v}, tup{4b3232, 4b1010}} + {i} => homBlend{...each{{h}=>sel{H,h,i}, hs}, i127), 8**0}}} + # Exact values for multiples of 8 + store{*U~~t0, 0, vpopc{vb}} + plus_scan{t0, 256/8} + def sel_c = getsel{swap{load{*V~~t0, 0}}} + # Top 5 bits select bytes from tables; bottom 3 select from mask + bot3 := U**0x07 + @for_vec_overlap{vl} (j to n) { + xv := load{*U~~(x+j), 0} + xb := xv & bot3 + xt := shr16{xv &~ bot3, 3} + ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} + store{*U~~(res+j), 0, ind} + } + } else { + plus_scan{t0, 256} + @for (res, x over n) res = load{t, x} + } } - setlabel{end} } def bin_search_branchless{up, w, wn, x, n, res} = { @@ -76,7 +151,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, xb:B) : B = { tup{r, rp} } r := undefined{B} - if (hasarch{'AVX2'} and T==i8 and wn<16 and xn>=32) { + if (T==i8 and wn<128 and xn>=32) { def {rt, rp} = alloc{i8, 'i8'}; r = rt bin_search_vec{up, *T~~w, wn, *T~~x, xn, rp} } else { From 38671e3fe9f5264455cbed8bcdee36b12dbd7d36 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 5 Jul 2023 07:32:49 -0400 Subject: [PATCH 11/34] =?UTF-8?q?Fast=20handling=20for=20length-1=20?= =?UTF-8?q?=F0=9D=95=A8=20in=20Bins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/grade.h | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 1021f477..8b78b522 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -362,6 +362,8 @@ bool CAT(isSorted,GRADE_UD(Up,Down))(B x) { #undef CMP } +extern B CAT(GRADE_UD(le,ge),c2)(B,B,B); + B GRADE_CAT(c2)(B t, B w, B x) { if (isAtm(w) || RNK(w)==0) thrM(GRADE_CHR": 𝕨 must have rank≥1"); if (isAtm(x)) x = m_unit(x); @@ -378,9 +380,29 @@ B GRADE_CAT(c2)(B t, B w, B x) { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); + B r; if (wia==0 | xia==0) { Arr* ra=allZeroes(xia); arr_shCopy(ra, x); - decG(w); decG(x); return taga(ra); + r=taga(ra); goto done; + } + if (wia==1) { + B c = IGet(w, 0); + if (LIKELY(weI32_MAX-10) thrM(GRADE_CHR": 𝕨 too big"); @@ -390,7 +412,6 @@ B GRADE_CAT(c2)(B t, B w, B x) { FL_SET(w, fl); } - B r; if (LIKELY(we Date: Wed, 5 Jul 2023 09:57:30 -0400 Subject: [PATCH 12/34] Unroll vector binary searches for different search depths --- src/singeli/src/bins.singeli | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 1f2ad32d..44da08a2 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -26,7 +26,7 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = { setlabel{end} } -def ceil_log2{n:u64} = 64 - clz{n+1} +def ceil_log2{n:u64} = 64 - clz{n-1} # Shift as u16, since x86 is missing 8-bit shifts def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) @@ -44,22 +44,24 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { {i} => sel{H, v, i} } if (hasarch{'AVX2'} and wn < 16) { - log := ceil_log2{wn-1} + log := ceil_log2{wn+1} l := 1< Date: Wed, 5 Jul 2023 15:50:27 -0400 Subject: [PATCH 13/34] =?UTF-8?q?Use=20type=20of=20=E2=89=A0=F0=9D=95=A8?= =?UTF-8?q?=20for=20Singeli=20Bins=20result=20and=20allocate=20outside=20S?= =?UTF-8?q?ingeli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/grade.h | 4 ++- src/builtins/sort.c | 4 --- src/singeli/src/bins.singeli | 48 ++++++++++++++++++++++-------------- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 8b78b522..58466ac5 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -443,7 +443,9 @@ B GRADE_CAT(c2)(B t, B w, B x) { #if SINGELI u8 k = elWidthLogBits(we) - 3; - r = b(si_bins[k*2 + GRADE_UD(0,1)](tyany_ptr(w), wia, tyany_ptr(x), xia, x.u)); + u8 rl = wia<128 ? 0 : wia<(1<<15) ? 1 : wia<(1<<31) ? 2 : 3; + void *rp = m_tyarrc(&r, 1<maxvalue{T}) { ++t; c{...ts} } + def c{T==f64} = {} + c{...rtypes} + t +} +fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = { + each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs} +} +def make_wr{k} = { + def w = each{{T} => write{T,k}, rtypes} + a:*(type{tupsel{0,w}}) = w +} +def wr_arrs = each{make_wr, unroll_sizes} + +def bin_search_branchless{up, w, wn, x, n, res, rtype} = { def lt = if (up) <; else > ws := w - 1 l0 := wn + 1 @@ -134,33 +153,24 @@ def bin_search_branchless{up, w, wn, x, n, res} = { each{bin1, ss, xs, each{bind{+,h}, ss}} l -= h } - each{{r,s} => store{res, r, cast_i{i32, s - ws}}, inds, ss} + each{{s} => u64~~(s - ws), ss} } # Unroll by 4 then 1 def search{i, k} = search{each{bind{+,i}, iota{k}}} j:u64 = 0 - def searches{k} = { while (j+k <= n) { search{j, k}; j+=k } } - each{searches, tup{4, 1}} + def searches{k, wr_arr} = { + wr := load{wr_arr, rtype} + while (j+k <= n) { wr(res, j, ...search{j, k}); j+=k } + } + each{searches, unroll_sizes, wr_arrs} } -def B = u64 -fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, xb:B) : B = { - def alloc{T,ts} = { - u:*u64 = tup{0} - rpp:* *T = (* *T)~~u - r := emit{B, merge{'m_',ts,'arrc_wrapu'}, rpp, xb} - rp := load{rpp,0} - tup{r, rp} - } - r := undefined{B} +fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { if (T==i8 and wn<128 and xn>=32) { - def {rt, rp} = alloc{i8, 'i8'}; r = rt - bin_search_vec{up, *T~~w, wn, *T~~x, xn, rp} + bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~rp} } else { - def {rt, rp} = alloc{i32, 'i32'}; r = rt - bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp} + bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp, rty} } - r } exportT{ From 81da9c586f631677d94d2c8432a04100a08f4163 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 5 Jul 2023 16:41:50 -0400 Subject: [PATCH 14/34] Table-based 1-byte Bins with >1-byte result --- src/singeli/src/bins.singeli | 75 +++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 30b15a6c..a7926ee3 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -31,6 +31,58 @@ def ceil_log2{n:u64} = 64 - clz{n-1} # Shift as u16, since x86 is missing 8-bit shifts def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) +def rtypes = tup{i8, i16, i32, f64} +# Return index of smallest possible result type given max result value +def get_rtype{len} = { + t:u8 = 0 + def c{T, ...ts} = if (len>maxvalue{T}) { ++t; c{...ts} } + def c{T==f64} = {} + c{...rtypes} + t +} +def rtype_arr{gen} = { + def t = each{gen, rtypes} + a:*(type{tupsel{0,t}}) = t +} + +# Write the last index of v at t+v, for each unique v in w +fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = { + def break = makelabel{} + i:u64 = 0; while (1) { + d:u64 = 16 + id := i+d + wi := undefined{T} + if (id >= n) { + @for (w over j from i to n) store{t, w, j+1} + goto{break} + } else if ((wi = load{w, i}) == load{w, id}) { + md := n - i + d2 := undefined{u64} + while ((d2=d+d) < md and wi == load{w, i + d2}) d = d2 + i += d + l := n - i; if (l > d) l = d + # Last instance of wi in [i,i+l); shrink l + while (l > 8) { + h := l/2 + m := i + h + if (wi == load{w, m}) i = m + l -= h + } + } else { + @unroll (j to 8) store{t, load{w, i+j}, i+j+1} + i += 8 + } + } + setlabel{break} +} +fn lookup_indices{I,T,R}(tab:*I, x:*T, rp:*void, xn:u64) : void = { + def m = 1< 0} def T = i8; def I = u8 @@ -120,23 +172,10 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { } def unroll_sizes = tup{4,1} -def rtypes = tup{i8, i16, i32, f64} -# Return index of smallest possible result type given max result value -def get_rtype{len} = { - t:u8 = 0 - def c{T, ...ts} = if (len>maxvalue{T}) { ++t; c{...ts} } - def c{T==f64} = {} - c{...rtypes} - t -} fn write{T,k}(r:*void, i:u64, ...vs:k**u64) : void = { each{{j,v} => store{*T~~r, i+j, cast_i{T,v}}, iota{k}, vs} } -def make_wr{k} = { - def w = each{{T} => write{T,k}, rtypes} - a:*(type{tupsel{0,w}}) = w -} -def wr_arrs = each{make_wr, unroll_sizes} +def wr_arrs = each{{k} => rtype_arr{{T} => write{T,k}}, unroll_sizes} def bin_search_branchless{up, w, wn, x, n, res, rtype} = { def lt = if (up) <; else > @@ -168,6 +207,14 @@ def bin_search_branchless{up, w, wn, x, n, res, rtype} = { fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { if (T==i8 and wn<128 and xn>=32) { bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~rp} + } else if (T==i8 and xn>=64 and (xn>=1024 or (xn-32) >= cast_i{u64,1}<<(ceil_log2{wn}/2+1))) { + def T = i8; def I = u64 + t0:*I = copy{256,0} + t:*I = t0 + 128 + write_indices{I,T}(t, *T~~w, wn) + def for_dir = if (up) for else for_backwards + s:I=0; @for_dir (t0 over 256) { if (t0 > s) s = t0; t0 = s } + load{lookup_i8_arr,rty}(t0, *T~~x, rp, xn) } else { bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp, rty} } From 2c9e07f33d34513fd75109fe0be4c4b1cbe96f93 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 5 Jul 2023 21:32:41 -0400 Subject: [PATCH 15/34] 1-byte Bins up to 32 unique eleemnts by unique lookup then index --- src/singeli/src/bins.singeli | 42 ++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index a7926ee3..0d5bf26a 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -95,7 +95,7 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { v := pair{h,h} {i} => sel{H, v, i} } - if (hasarch{'AVX2'} and wn < 16) { + if (hasarch{'AVX2'} and wn < 8) { log := ceil_log2{wn+1} l := 1< V**0} + @collect (t in *V~~t0 over nb) addu{homMask{t > V**0}} + } + dup := promote{u64,nu} < wn + # Unique index to w index conversion + ui := undefined{V}; ui1 := undefined{V} + if (dup) { + if (nu > vl) goto{no_bittab} + # We'll subtract 1 when indexing so the initial 0 isn't needed + tui:*i8 = copy{vl, 0}; i:T = 0 + @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } + ui = load{*V~~tui, 0} + if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} + ui = shuf{[4]u64, ui, 4b1010} } # Popcount on 8-bit values def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } @@ -140,10 +153,10 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { s{shr16{v,4}} + s{v} } # 32-byte select - vtop := U**(vl/2) + vtop := V**(vl/2) def getsel{v & width{type{v}}==256} = { hs := each{bind{shuf, [4]u64, v}, tup{4b3232, 4b1010}} - {i} => homBlend{...each{{h}=>sel{H,h,i}, hs}, i homBlend{...each{{h}=>sel{H,h,i}, hs}, V~~i 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 Date: Thu, 6 Jul 2023 11:21:45 -0400 Subject: [PATCH 16/34] Cleanup --- src/singeli/src/bins.singeli | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 0d5bf26a..60545c48 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -14,6 +14,7 @@ def for_backwards{vars,begin,end,iter} = { iter{i, vars} } } +def for_dir{up} = if (up) for else for_backwards def for_vec_overlap{vl}{vars,begin==0,n,iter} = { assert{n >= vl} def end = makelabel{} @@ -31,6 +32,20 @@ def ceil_log2{n:u64} = 64 - clz{n-1} # Shift as u16, since x86 is missing 8-bit shifts def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) +def getsel{...x} = assert{'shuffling not supported', show{...x}} +if (hasarch{'AVX2'}) { + def getsel{h:H & lvec{H, 16, 8}} = { + v := pair{h,h} + {i} => sel{H, v, i} + } + def getsel{v:V & lvec{V, 32, 8}} = { + def H = v_half{V} + vtop := V**(vcount{V}/2) + hs := each{bind{shuf, [4]u64, v}, tup{4b3232, 4b1010}} + {i} => homBlend{...each{{h}=>sel{H,h,i}, hs}, V~~i sel{H, v, i} - } if (hasarch{'AVX2'} and wn < 8) { log := ceil_log2{wn+1} l := 1< homBlend{...each{{h}=>sel{H,h,i}, hs}, V~~i 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 16) ind = homBlend{sel{H,ui1,i0}, ind, i0=32) { bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~rp} } else if (T==i8 and xn>=64 and (xn>=1024 or (xn-32) >= cast_i{u64,1}<<(ceil_log2{wn}/2+1))) { - def T = i8; def I = u64 + def I = u64 t0:*I = copy{256,0} t:*I = t0 + 128 write_indices{I,T}(t, *T~~w, wn) - def for_dir = if (up) for else for_backwards - s:I=0; @for_dir (t0 over 256) { if (t0 > s) s = t0; t0 = s } + s:I=0; @for_dir{up} (t0 over 256) { if (t0 > s) s = t0; t0 = s } load{lookup_i8_arr,rty}(t0, *T~~x, rp, xn) } else { bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp, rty} From a711eb72eba2dd0f24067de810ee5e629d4321ca Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 6 Jul 2023 17:54:43 -0400 Subject: [PATCH 17/34] Table-based 2-byte Bins, using max-scan --- src/singeli/src/bins.singeli | 85 ++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 60545c48..b2c73415 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -32,6 +32,50 @@ def ceil_log2{n:u64} = 64 - clz{n-1} # Shift as u16, since x86 is missing 8-bit shifts def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) +# Forward or backwards in-place max-scan +# Assumes a whole number of vectors and minimum 0 +fn max_scan{T, up}(x:*T, len:u64) : void = { + def w = width{T} + if (hasarch{'AVX2'} and T!=u64) { + def op = max + # TODO unify with scan.singeli avx2_scan_idem + def rev{a} = if (up) a else (tuplen{a}-1)-reverse{a} + def maker{T, l} = make{T, rev{l}} + def sel8{v, t} = sel{[16]u8, v, maker{[32]i8, t}} + def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} + def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,rev{n}}} + def spread{a:VT} = { + def w = elwidth{VT} + def b = w/8 + if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}}; else a + } + def shift{k,l} = merge{iota{k},iota{l-k}} + def c8 {k, a} = op{a, shuf{[4]u32, a, shift{k,4}}} + def c32{k, a} = (if (w<=8*k) op{a, sel8{a, shift{k,16}}}; else a) + def pre{a} = { + b:= c8{2, c8{1, c32{2, c32{1, a}}}} + op{b, sel{[8]i32, spread{b}, maker{[8]i32, 3*(3 m) m = x; x = m } + } +} + +def fmt_type{T} = { + def w = width{T} + merge{quality{T}, if (w==8) '8' else if (w==16) '16' else if (w==32) '32' else '64'} +} +def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len} +def tfree{ptr} = emit{void, 'TFREE', ptr} + def getsel{...x} = assert{'shuffling not supported', show{...x}} if (hasarch{'AVX2'}) { def getsel{h:H & lvec{H, 16, 8}} = { @@ -90,13 +134,16 @@ fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = { } setlabel{break} } -fn lookup_indices{I,T,R}(tab:*I, x:*T, rp:*void, xn:u64) : void = { - def m = 1< 0} @@ -190,7 +237,7 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { setlabel{no_bittab} } plus_scan{t0, 256} - lookup_indices{T,T,T}(t0, x, *void~~res, n) + @for (res, x over n) res = load{t, x} if (hasarch{'AVX2'}) setlabel{done} } } @@ -229,17 +276,21 @@ def bin_search_branchless{up, w, wn, x, n, res, rtype} = { } fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { - if (T==i8 and wn<128 and xn>=32) { - bin_search_vec{up, *T~~w, wn, *T~~x, xn, *i8~~rp} - } else if (T==i8 and xn>=64 and (xn>=1024 or (xn-32) >= cast_i{u64,1}<<(ceil_log2{wn}/2+1))) { - def I = u64 - t0:*I = copy{256,0} - t:*I = t0 + 128 - write_indices{I,T}(t, *T~~w, wn) - s:I=0; @for_dir{up} (t0 over 256) { if (t0 > s) s = t0; t0 = s } - load{lookup_i8_arr,rty}(t0, *T~~x, rp, xn) + def param = tup{up, *T~~w, wn, *T~~x, xn, rp} + def lookup{k} = { + if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param} + else if (k+1 < tuplen{rtypes}) lookup{k+1} + } + # Lookup table threshold has to account for cost of + # populating the table (proportional to wn until it's large), and + # initializing the table (constant, much higher for i16) + if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) { + if (rty==0) bin_search_vec{...slice{param,0,-1}, *i8~~rp} + else lookup{1} + } else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/promote{u64,ceil_log2{wn}+2})) { + lookup{0} } else { - bin_search_branchless{up, *T~~w, wn, *T~~x, xn, rp, rty} + bin_search_branchless{...param, rty} } } From d665b90bbfe1add540f202690705276c6a2fa6b6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 6 Jul 2023 22:00:10 -0400 Subject: [PATCH 18/34] Slightly faster binary search pattern with a blend --- src/singeli/src/bins.singeli | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index b2c73415..c475e46d 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -167,7 +167,8 @@ def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { s := U**0 h := h0 @unroll (klog) { - s |= h &~ lt{xv, selw{s | h}} + m := s | h + s = homBlend{m, s, lt{xv, selw{m}}} h = shr16{h, 1} } store{*U~~(res+j), 0, s - off} From 1080236433f8c66c1a1ce5769f838e65a200864b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 08:01:55 -0400 Subject: [PATCH 19/34] 2-byte vector binary searches --- src/singeli/src/bins.singeli | 197 +++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 92 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index c475e46d..7244f470 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -145,101 +145,113 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { tfree{t0} } -def bin_search_vec{up, w:*i8, wn, x:*i8, n, res:*i8} = { - assert{wn > 0} - def T = i8; def I = u8 - def lt = if (up) <; else > - def pre = (if (up) minvalue else maxvalue){T} +def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { + assert{wn < 128} # Total must fit in i8 + def T = i8 def vl = 32 - def V = [vl]T; def H = [vl/2]T - def U = [vl]I - if (hasarch{'AVX2'} and wn < 8) { - log := ceil_log2{wn+1} - l := 1< V**0}} } - } else { - assert{wn < 128} # Total must fit in i8 - t0:*i8 = copy{256,0} - t:*i8 = t0 + 128 - @for (w over wn) store{t, w, 1+load{t, w}} - def plus_scan{tab, len} = { - s:i8=0; @for_dir{up} (tab over len) { s += tab; tab = s } + dup := promote{u64,nu} < wn + # Unique index to w index conversion + ui := undefined{V}; ui1 := undefined{V} + if (dup) { + if (nu > vl) goto{no_bittab} + # We'll subtract 1 when indexing so the initial 0 isn't needed + tui:*i8 = copy{vl, 0}; i:T = 0 + @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } + ui = load{*V~~tui, 0} + if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} + ui = shuf{[4]u64, ui, 4b1010} } - def no_bittab = makelabel{}; def done = makelabel{} - if (hasarch{'AVX2'}) { - # Convert to bit table - def nb = 256/vl - nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques - vb := U~~make{[nb](ty_u{vl}), - @collect (t in *V~~t0 over nb) addu{homMask{t > V**0}} - } - dup := promote{u64,nu} < wn - # Unique index to w index conversion - ui := undefined{V}; ui1 := undefined{V} + # Popcount on 8-bit values + def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } + def sum4 = getsel{make{H, sums{vl/2}}} + bot4 := U**0x0f + def vpopc{v} = { + def s{b} = sum4{b&bot4} + s{shr16{v,4}} + s{v} + } + # Bit table + def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness + def sel_b = getsel{swap{vb}} + # Masks for filtering bit table + def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} + # Exact values for multiples of 8 + store{*U~~t0, 0, vpopc{vb}} + plus_scan{t0, 256/8} + def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} + # Top 5 bits select bytes from tables; bottom 3 select from mask + bot3 := U**0x07 + @for_vec_overlap{vl} (j to xn) { + xv := load{*U~~(x+j), 0} + xb := xv & bot3 + xt := shr16{xv &~ bot3, 3} + ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} if (dup) { - if (nu > vl) goto{no_bittab} - # We'll subtract 1 when indexing so the initial 0 isn't needed - tui:*i8 = copy{vl, 0}; i:T = 0 - @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } - ui = load{*V~~tui, 0} - if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} - ui = shuf{[4]u64, ui, 4b1010} + i0 := V~~ind # Can contain -1 + ind = sel{H, ui, i0} + if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0127), 8**0}}} - # Exact values for multiples of 8 - store{*U~~t0, 0, vpopc{vb}} - plus_scan{t0, 256/8} - def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} - # Top 5 bits select bytes from tables; bottom 3 select from mask - bot3 := U**0x07 - @for_vec_overlap{vl} (j to n) { - xv := load{*U~~(x+j), 0} - xb := xv & bot3 - xt := shr16{xv &~ bot3, 3} - ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} - if (dup) { - i0 := V~~ind # Can contain -1 - ind = sel{H, ui, i0} - if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 1} + def wd = width{T} + def bytes = wd/8; def bb = bind{base,256} + def vl = 256/wd + def V = [vl]T; def H = v_half{V} + def U = [vl](ty_u{T}) + def lt = if (up) <; else > + # Number of steps + log := ceil_log2{wn+1} + l := 1<>(lb{bytes}+wd-8)}, 0} + store{*[vl]i8~~(*i8~~rp+j), 0, r - off} } - plus_scan{t0, 256} - @for (res, x over n) res = load{t, x} - if (hasarch{'AVX2'}) setlabel{done} } } @@ -282,12 +294,13 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param} else if (k+1 < tuplen{rtypes}) lookup{k+1} } + if (hasarch{'AVX2'} and T<=i16 and wn < 8 and xn >= 256/width{T}) { + bin_search_vec{T, ...param, 8} # Lookup table threshold has to account for cost of # populating the table (proportional to wn until it's large), and # initializing the table (constant, much higher for i16) - if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) { - if (rty==0) bin_search_vec{...slice{param,0,-1}, *i8~~rp} - else lookup{1} + } else if (T==i8 and xn>=32 and (xn>=512 or xn >= wn>>6 + 32)) { + lookup{0} } else if (T==i16 and xn>=512 and (xn>=1<<14 or xn >= wn>>6 + (u64~~3<<(12+rty))/promote{u64,ceil_log2{wn}+2})) { lookup{0} } else { From 7f5ba961c28a8a1642feb1659a76c621cae8f473 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 08:05:34 -0400 Subject: [PATCH 20/34] Use default bins_lookup if there's no AVX2 --- src/singeli/src/bins.singeli | 110 +++++++++++++++++------------------ 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 7244f470..16d66846 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -145,7 +145,7 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { tfree{t0} } -def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { +def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch{'AVX2'}} = { assert{wn < 128} # Total must fit in i8 def T = i8 def vl = 32 @@ -159,64 +159,64 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { def plus_scan{tab, len} = { s:i8=0; @for_dir{up} (tab over len) { s += tab; tab = s } } + + # Convert to bit table def no_bittab = makelabel{}; def done = makelabel{} - if (hasarch{'AVX2'}) { - # Convert to bit table - def nb = 256/vl - nu:u8 = 0; def addu{b} = { nu+=popc{b}; b } # Number of uniques - vb := U~~make{[nb](ty_u{vl}), - @collect (t in *V~~t0 over nb) addu{homMask{t > V**0}} - } - dup := promote{u64,nu} < wn - # Unique index to w index conversion - ui := undefined{V}; ui1 := undefined{V} - if (dup) { - if (nu > vl) goto{no_bittab} - # We'll subtract 1 when indexing so the initial 0 isn't needed - tui:*i8 = copy{vl, 0}; i:T = 0 - @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } - ui = load{*V~~tui, 0} - if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} - ui = shuf{[4]u64, ui, 4b1010} - } - # Popcount on 8-bit values - def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } - def sum4 = getsel{make{H, sums{vl/2}}} - bot4 := U**0x0f - def vpopc{v} = { - def s{b} = sum4{b&bot4} - s{shr16{v,4}} + s{v} - } - # Bit table - def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness - def sel_b = getsel{swap{vb}} - # Masks for filtering bit table - def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} - # Exact values for multiples of 8 - store{*U~~t0, 0, vpopc{vb}} - plus_scan{t0, 256/8} - def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} - # Top 5 bits select bytes from tables; bottom 3 select from mask - bot3 := U**0x07 - @for_vec_overlap{vl} (j to xn) { - xv := load{*U~~(x+j), 0} - xb := xv & bot3 - xt := shr16{xv &~ bot3, 3} - ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} - if (dup) { - i0 := V~~ind # Can contain -1 - ind = sel{H, ui, i0} - if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 V**0}} } + dup := promote{u64,nu} < wn + # Unique index to w index conversion + ui := undefined{V}; ui1 := undefined{V} + if (dup) { + if (nu > vl) goto{no_bittab} + # We'll subtract 1 when indexing so the initial 0 isn't needed + tui:*i8 = copy{vl, 0}; i:T = 0 + @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } + ui = load{*V~~tui, 0} + if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} + ui = shuf{[4]u64, ui, 4b1010} + } + # Popcount on 8-bit values + def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } + def sum4 = getsel{make{H, sums{vl/2}}} + bot4 := U**0x0f + def vpopc{v} = { + def s{b} = sum4{b&bot4} + s{shr16{v,4}} + s{v} + } + # Bit table + def swap{v} = shuf{[4]u64, v, 4b1032} # For signedness + def sel_b = getsel{swap{vb}} + # Masks for filtering bit table + def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} + # Exact values for multiples of 8 + store{*U~~t0, 0, vpopc{vb}} + plus_scan{t0, 256/8} + def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} + # Top 5 bits select bytes from tables; bottom 3 select from mask + bot3 := U**0x07 + @for_vec_overlap{vl} (j to xn) { + xv := load{*U~~(x+j), 0} + xb := xv & bot3 + xt := shr16{xv &~ bot3, 3} + ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} + if (dup) { + i0 := V~~ind # Can contain -1 + ind = sel{H, ui, i0} + if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 Date: Fri, 7 Jul 2023 09:47:02 -0400 Subject: [PATCH 21/34] Precomputed midpoint bits in vector binary search Clang already optimized to this but it's more explicit --- src/singeli/src/bins.singeli | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 16d66846..01562e7a 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -220,7 +220,7 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch } def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { - assert{wn > 1} + assert{wn > 1}; assert{wn < maxwn} def wd = width{T} def bytes = wd/8; def bb = bind{base,256} def vl = 256/wd @@ -229,24 +229,22 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def lt = if (up) <; else > # Number of steps log := ceil_log2{wn+1} - l := 1< U**(lowbits << j), iota{maxwn-1}} @unroll (klog from 2 to lb{maxwn}+1) { if (log==klog) @for_vec_overlap{vl} (j to xn) { xv:= load{*V~~(x+j), 0} s := U**bb{iota{bytes}} # Select sequential bytes within each U - h := h0 - @unroll (klog) { - m := s | h + @unroll (j to klog) { + m := s | tupsel{klog-1-j, bits} s = homBlend{m, s, lt{xv, V~~selw{to_el{u8,m}}}} - h = shr16{h, 1} } r := if (T==i8) s else half{narrow{u8, s>>(lb{bytes}+wd-8)}, 0} From 8054597004dbba3e0246d3a39733566135aa1ae8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 15:23:51 -0400 Subject: [PATCH 22/34] Handle up to 64 unique values in bit-table 1-byte Bins --- src/singeli/src/bins.singeli | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 01562e7a..06dae16d 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -169,15 +169,18 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch } dup := promote{u64,nu} < wn # Unique index to w index conversion - ui := undefined{V}; ui1 := undefined{V} + ui := undefined{V}; ui1 := undefined{V}; ui2 := each{undefined,tup{V,V}} if (dup) { - if (nu > vl) goto{no_bittab} + def maxu = 2*vl + if (nu > maxu) goto{no_bittab} # We'll subtract 1 when indexing so the initial 0 isn't needed - tui:*i8 = copy{vl, 0}; i:T = 0 + tui:*i8 = copy{maxu, 0}; i:T = 0 @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } - ui = load{*V~~tui, 0} + def tv = bind{load, *V~~tui} + ui = tv{0} if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} ui = shuf{[4]u64, ui, 4b1010} + if (nu > vl) ui2 = each{bind{shuf, [4]u64, tv{1}}, tup{4b1010, 4b3232}} } # Popcount on 8-bit values def sums{n} = if (n==1) tup{0} else { def s=sums{n/2}; merge{s,s+1} } @@ -206,8 +209,13 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch ind := sel_c{xt} - vpopc{sel_b{xt} & U~~sel_m{xb}} if (dup) { i0 := V~~ind # Can contain -1 - ind = sel{H, ui, i0} - if (nu > 16) ind = homBlend{sel{H,ui1,i0}, ind, i0 16) { + b := V**0 < (i0 & V**(vl/2)) + ind = homBlend{ind, isel{ui1}, b} + if (nu > 32) ind = homBlend{homBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} + } } store{*U~~(res+j), 0, ind} } From f36dc7adba528f9ecd9dd31a257287681b7d12ba Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 15:41:04 -0400 Subject: [PATCH 23/34] Use max-scan instead of plus-scan for 1-byte Bins table --- src/singeli/src/bins.singeli | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 06dae16d..a5fdeab1 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -147,7 +147,6 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch{'AVX2'}} = { assert{wn < 128} # Total must fit in i8 - def T = i8 def vl = 32 def V = [vl]T; def H = v_half{V} def U = [vl]u8 @@ -155,10 +154,7 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch t0:*i8 = copy{256,0} t:*i8 = t0 + 128 - @for (w over wn) store{t, w, 1+load{t, w}} - def plus_scan{tab, len} = { - s:i8=0; @for_dir{up} (tab over len) { s += tab; tab = s } - } + @for (w over j to wn) store{t, w, j+1} # Convert to bit table def no_bittab = makelabel{}; def done = makelabel{} @@ -175,7 +171,7 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch if (nu > maxu) goto{no_bittab} # We'll subtract 1 when indexing so the initial 0 isn't needed tui:*i8 = copy{maxu, 0}; i:T = 0 - @for (tui over promote{u64,nu}) { i += load{t, load{w, i}}; tui = i } + @for (tui over promote{u64,nu}) { i = load{t, load{w, i}}; tui = i } def tv = bind{load, *V~~tui} ui = tv{0} if (nu > 16) ui1 = shuf{[4]u64, ui, 4b3232} @@ -198,7 +194,7 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch def sel_m = getsel{make{H, merge{ms - 256*(ms>127), 8**0}}} # Exact values for multiples of 8 store{*U~~t0, 0, vpopc{vb}} - plus_scan{t0, 256/8} + st:i8=0; @for_dir{up} (t0 over 256/8) { st += t0; t0 = st } def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} # Top 5 bits select bytes from tables; bottom 3 select from mask bot3 := U**0x07 @@ -222,7 +218,7 @@ def bins_lookup{I==i8, T==i8, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void & hasarch goto{done} setlabel{no_bittab} - plus_scan{t0, 256} + max_scan{I, up}(t0, 256) @for (res, x over xn) res = load{t, x} setlabel{done} } From d19df2693aa2ddb4e68119410b5a27ff1a88353d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 15:57:43 -0400 Subject: [PATCH 24/34] Merge 1-byte table code with normal bins_lookup --- src/singeli/src/bins.singeli | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index a5fdeab1..505d91ad 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -134,30 +134,36 @@ fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = { } setlabel{break} } +fn write_indices{I,T & width{I}==8}(t:*I, w:*T, n:u64) : void = { + @for (w over j to n) store{t, w, j+1} +} def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = { + # Build table def tc = 1< 32) ind = homBlend{homBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} } } - store{*U~~(res+j), 0, ind} + store{*U~~(*T~~rp+j), 0, ind} } goto{done} setlabel{no_bittab} - - max_scan{I, up}(t0, 256) - @for (res, x over xn) res = load{t, x} - setlabel{done} } def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { From fe92f91ca12e057be0bbbb6e2b74e9651b35467f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 20:27:58 -0400 Subject: [PATCH 25/34] 2-byte vector binary search on 2 lanes --- src/singeli/src/bins.singeli | 46 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 505d91ad..0a47afe9 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -90,6 +90,15 @@ if (hasarch{'AVX2'}) { } } +# Move evens to half 0 and odds to half 1 +def uninterleave{x:V & hasarch{'AVX2'}} = { + def vl = vcount{V}; def bytes = width{eltype{V}}/8 + def i = 2*iota{vl/4} + def i2= join{table{+, bytes*merge{i,i+1}, iota{bytes}}} + t := V~~sel{[16]u8, to_el{u8,x}, make{[32]u8, merge{i2,i2}}} + shuf{[4]u64, t, 4b3120} +} + def rtypes = tup{i8, i16, i32, f64} # Return index of smallest possible result type given max result value def get_rtype{len} = { @@ -239,18 +248,35 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { off := [vl]u8**(gap - 1) # Fill with minimum value at the beginning def pre = (if (up) minvalue else maxvalue){T} - wv := homBlend{load{*H~~(w-gap)}, H**pre, maskOf{H,gap}} - def selw = getsel{[16]u8~~wv} + wv := homBlend{load{*V~~(w-gap)}, V**pre, maskOf{V,gap}} + # Separate even/odd elements if double width + def maxstep = lb{maxwn} + def lstep = lb{vl/2} + def has_w1 = maxstep > lstep + if (has_w1 and wn >= vl/2) wv = uninterleave{wv} + def ms{h} = getsel{[16]u8~~half{wv,h}} + def selw = ms{0}; def selw1 = if (has_w1) ms{1} else 'undef' # Midpoint bits for each step def lowbits = bb{copy{bytes,bytes}} - bits := each{{j} => U**(lowbits << j), iota{maxwn-1}} - @unroll (klog from 2 to lb{maxwn}+1) { - if (log==klog) @for_vec_overlap{vl} (j to xn) { + bits := each{{j} => U**(lowbits << j), iota{lstep}} + # Unroll sizes up to a full lane, handling extra lanes conditionally + # in the largest one + @unroll (klog from 2 to min{maxstep,lstep}+1) { + def last = klog==lstep + def this = if (not last) log==klog else log>=klog + if (this) @for_vec_overlap{vl} (j to xn) { + def as_u8{T,op, ...par} = T~~op{...each{bind{to_el,u8},par}} xv:= load{*V~~(x+j), 0} s := U**bb{iota{bytes}} # Select sequential bytes within each U @unroll (j to klog) { - m := s | tupsel{klog-1-j, bits} - s = homBlend{m, s, lt{xv, V~~selw{to_el{u8,m}}}} + m := s | tupsel{klog-1-j,bits} + s = homBlend{m, s, lt{xv, as_u8{V,selw, m}}} + } + # Extra selection lanes + if (last and has_w1 and log>klog) { + c := lt{xv, as_u8{V,selw1, s}} + assert{T==i16} # Otherwise position of c is different + s = as_u8{U,+, s,c}; s += s } r := if (T==i8) s else half{narrow{u8, s>>(lb{bytes}+wd-8)}, 0} @@ -298,8 +324,10 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param} else if (k+1 < tuplen{rtypes}) lookup{k+1} } - if (hasarch{'AVX2'} and T<=i16 and wn < 8 and xn >= 256/width{T}) { - bin_search_vec{T, ...param, 8} + # For >=8 i8 values, vector bit-table is as good as binary search + def wn_vec = if (T==i8) 8 else 16 + if (hasarch{'AVX2'} and T<=i16 and wn < wn_vec and xn >= 256/width{T}) { + bin_search_vec{T, ...param, wn_vec} # Lookup table threshold has to account for cost of # populating the table (proportional to wn until it's large), and # initializing the table (constant, much higher for i16) From c8d20fbf2634a4ac1bdc80a89ea49a9b59d60f5f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 21:11:31 -0400 Subject: [PATCH 26/34] And 2-byte vector binary search on 4 lanes --- src/singeli/src/bins.singeli | 40 ++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 0a47afe9..c7efdf79 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,4 +1,5 @@ include './base' +def shufHalves{...x} = assert{'shufHalves not supported', show{...x}} if (hasarch{'AVX2'}) { include './sse' include './avx' @@ -248,14 +249,24 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { off := [vl]u8**(gap - 1) # Fill with minimum value at the beginning def pre = (if (up) minvalue else maxvalue){T} - wv := homBlend{load{*V~~(w-gap)}, V**pre, maskOf{V,gap}} - # Separate even/odd elements if double width + wg := *V~~(w-gap) + wv := homBlend{load{wg}, V**pre, maskOf{V,gap}} + # Separate even/odd elements to double width, like Eytzinger def maxstep = lb{maxwn} def lstep = lb{vl/2} - def has_w1 = maxstep > lstep - if (has_w1 and wn >= vl/2) wv = uninterleave{wv} - def ms{h} = getsel{[16]u8~~half{wv,h}} - def selw = ms{0}; def selw1 = if (has_w1) ms{1} else 'undef' + def ex = maxstep - lstep + wv2 := wv + if (ex>=1 and wn >= vl/2) { + wv = uninterleave{wv} + if (ex>=2 and wn >= vl) { + t := uninterleave{load{wg, 1}} + wv2= shufHalves{wv, t, 16b31} + wv = uninterleave{shufHalves{wv, t, 16b20}} + } + } + def ms{h} = getsel{to_el{u8,half{wv,h}}} + def selw = ms{0}; def selw1 = if (ex>=1) ms{1} else 'undef' + def selw2 = if (ex>=2) getsel{to_el{u8,wv2}} else 'undef' # Midpoint bits for each step def lowbits = bb{copy{bytes,bytes}} bits := each{{j} => U**(lowbits << j), iota{lstep}} @@ -273,10 +284,17 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { s = homBlend{m, s, lt{xv, as_u8{V,selw, m}}} } # Extra selection lanes - if (last and has_w1 and log>klog) { - c := lt{xv, as_u8{V,selw1, s}} - assert{T==i16} # Otherwise position of c is different - s = as_u8{U,+, s,c}; s += s + if (last and ex>=1 and log>=klog+1) { + def addbit{sel} = { + c := lt{xv, as_u8{V,sel, s}} + assert{T==i16} # Otherwise position of c is different + s = as_u8{U,+, s,c}; s = as_u8{U,+, s,s} + } + addbit{selw1} + if (ex>=2 and log>=klog+2) { + s = as_u8{U,+, s, U**bb{bytes-iota{bytes}}} # Undo part of last step + addbit{selw2} + } } r := if (T==i8) s else half{narrow{u8, s>>(lb{bytes}+wd-8)}, 0} @@ -325,7 +343,7 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { else if (k+1 < tuplen{rtypes}) lookup{k+1} } # For >=8 i8 values, vector bit-table is as good as binary search - def wn_vec = if (T==i8) 8 else 16 + def wn_vec = if (T==i8) 8 else 32 if (hasarch{'AVX2'} and T<=i16 and wn < wn_vec and xn >= 256/width{T}) { bin_search_vec{T, ...param, wn_vec} # Lookup table threshold has to account for cost of From 64ae8f9afd7939fbb46b3a0c754d129802ab63d0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 7 Jul 2023 21:37:49 -0400 Subject: [PATCH 27/34] Slightly better blend pattern for >16-byte bit table --- src/singeli/src/bins.singeli | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index c7efdf79..89fa999d 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -31,7 +31,7 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = { def ceil_log2{n:u64} = 64 - clz{n-1} # Shift as u16, since x86 is missing 8-bit shifts -def shr16{v, n} = type{v}~~(([width{type{v}}/16]u16~~v) >> n) +def shr16{v:V, n} = V~~(to_el{u16, v} >> n) # Forward or backwards in-place max-scan # Assumes a whole number of vectors and minimum 0 @@ -224,9 +224,9 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { def isel{u} = sel{H, u, i0} ind = isel{ui} if (nu > 16) { - b := V**0 < (i0 & V**(vl/2)) - ind = homBlend{ind, isel{ui1}, b} - if (nu > 32) ind = homBlend{homBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} + b := V~~(to_el{u16, i0} << (7 - lb{vl/2})) + ind = topBlend{ind, isel{ui1}, b} + if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} } } store{*U~~(*T~~rp+j), 0, ind} From af5ba0a2d2632f6d2ec4df5c41967e285f1dd28e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 8 Jul 2023 10:32:03 -0400 Subject: [PATCH 28/34] Switch from Eytzinger to linear search on extra lanes --- src/singeli/src/bins.singeli | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 89fa999d..d46d7f08 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -246,27 +246,30 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { # Number of steps log := ceil_log2{wn+1} gap := 1<=1 and wn >= vl/2) { wv = uninterleave{wv} + --gap # Allows subtracting < instead of adding <= if (ex>=2 and wn >= vl) { t := uninterleave{load{wg, 1}} - wv2= shufHalves{wv, t, 16b31} + wv2= uninterleave{shufHalves{wv, t, 16b31}} wv = uninterleave{shufHalves{wv, t, 16b20}} + gap -= 2 } } - def ms{h} = getsel{to_el{u8,half{wv,h}}} - def selw = ms{0}; def selw1 = if (ex>=1) ms{1} else 'undef' - def selw2 = if (ex>=2) getsel{to_el{u8,wv2}} else 'undef' + def ms{v}{h} = getsel{to_el{u8,half{v,h}}} + def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef' + def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' + # Offset at end + off := U~~V**i8~~(gap - 1) # Midpoint bits for each step def lowbits = bb{copy{bytes,bytes}} bits := each{{j} => U**(lowbits << j), iota{lstep}} @@ -279,26 +282,25 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def as_u8{T,op, ...par} = T~~op{...each{bind{to_el,u8},par}} xv:= load{*V~~(x+j), 0} s := U**bb{iota{bytes}} # Select sequential bytes within each U + def ltx{se,ind} = lt{xv, as_u8{V,se, ind}} @unroll (j to klog) { m := s | tupsel{klog-1-j,bits} - s = homBlend{m, s, lt{xv, as_u8{V,selw, m}}} + s = homBlend{m, s, ltx{selw, m}} } + r := if (T==i8) s else s>>(lb{bytes}+wd-8) # Extra selection lanes if (last and ex>=1 and log>=klog+1) { - def addbit{sel} = { - c := lt{xv, as_u8{V,sel, s}} - assert{T==i16} # Otherwise position of c is different - s = as_u8{U,+, s,c}; s = as_u8{U,+, s,s} - } - addbit{selw1} + r += r + c := ltx{selw1,s} if (ex>=2 and log>=klog+2) { - s = as_u8{U,+, s, U**bb{bytes-iota{bytes}}} # Undo part of last step - addbit{selw2} + r += r + each{{se} => c += ltx{se,s}, selw2} } + r += c } - r := if (T==i8) s - else half{narrow{u8, s>>(lb{bytes}+wd-8)}, 0} - store{*[vl]i8~~(*i8~~rp+j), 0, r - off} + r -= off + rn := if (T==i8) r else half{narrow{u8, r}, 0} + store{*[vl]i8~~(*i8~~rp+j), 0, rn} } } } From fc57e0012dfb799990d65fbc8690ee55a8304bdf Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 8 Jul 2023 13:28:26 -0400 Subject: [PATCH 29/34] Shuffle-based 4-byte vector binary search --- src/singeli/src/bins.singeli | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index d46d7f08..d4205fae 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -299,8 +299,10 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { r += c } r -= off - rn := if (T==i8) r else half{narrow{u8, r}, 0} - store{*[vl]i8~~(*i8~~rp+j), 0, rn} + rn := if (T==i8) r + else if (T==i16) half{narrow{u8, r}, 0} + else extract{to_el{i64, narrow{u8, r}}, 0} + store{*type{rn}~~(*i8~~rp+j), 0, rn} } } } @@ -345,8 +347,8 @@ fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = { else if (k+1 < tuplen{rtypes}) lookup{k+1} } # For >=8 i8 values, vector bit-table is as good as binary search - def wn_vec = if (T==i8) 8 else 32 - if (hasarch{'AVX2'} and T<=i16 and wn < wn_vec and xn >= 256/width{T}) { + def wn_vec = if (T==i8) 8 else 2*256/width{T} + if (hasarch{'AVX2'} and T<=i32 and wn < wn_vec and xn >= 256/width{T}) { bin_search_vec{T, ...param, wn_vec} # Lookup table threshold has to account for cost of # populating the table (proportional to wn until it's large), and From 46c6d4705570ba7f6f1b1f3b0227bf1cee4cd037 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 8 Jul 2023 14:23:59 -0400 Subject: [PATCH 30/34] Permutevar instead of shuffle for 4-byte vector binary search --- src/singeli/src/bins.singeli | 40 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index d4205fae..5f47e52e 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -89,6 +89,7 @@ if (hasarch{'AVX2'}) { hs := each{bind{shuf, [4]u64, v}, tup{4b3232, 4b1010}} {i} => homBlend{...each{{h}=>sel{H,h,i}, hs}, V~~i sel{V, v, i} } } # Move evens to half 0 and odds to half 1 @@ -238,9 +239,11 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { assert{wn > 1}; assert{wn < maxwn} def wd = width{T} - def bytes = wd/8; def bb = bind{base,256} - def vl = 256/wd - def V = [vl]T; def H = v_half{V} + def I = if (wd<32) u8 else u32; def wi = width{I} + def lanes = hasarch{'AVX2'} & (I==u8) + def isub = wd/wi; def bb = bind{base,1<>lanes + def V = [vl]T def U = [vl](ty_u{T}) def lt = if (up) <; else > # Number of steps @@ -249,29 +252,33 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { # Fill with minimum value at the beginning def pre = (if (up) minvalue else maxvalue){T} wg := *V~~(w-gap) - wv := homBlend{load{wg}, V**pre, maskOf{V,gap}} + wv0:= homBlend{load{wg}, V**pre, maskOf{V,gap}} # For multiple lanes, interleave like transpose def maxstep = lb{maxwn} - def lstep = lb{vl/2} + def lstep = lb{svl} def ex = maxstep - lstep + wv := if (lanes) wv0 else tup{wv0,wv0} wv2 := wv # Compiler complains if uninitialized - if (ex>=1 and wn >= vl/2) { - wv = uninterleave{wv} + if (ex>=1 and wn >= svl) { --gap # Allows subtracting < instead of adding <= - if (ex>=2 and wn >= vl) { - t := uninterleave{load{wg, 1}} - wv2= uninterleave{shufHalves{wv, t, 16b31}} - wv = uninterleave{shufHalves{wv, t, 16b20}} + def un = uninterleave + def tr_half{a, b} = each{bind{shufHalves,a,b}, tup{16b20, 16b31}} + def un{{a,b}} = tr_half{un{a},un{b}} + if (not lanes) tupsel{1,wv} = load{wg, 1} + wv = un{wv} + if (ex>=2 and wn >= 2*svl) { + assert{lanes} # Different transpose pattern needed gap -= 2 + tup{wv, wv2} = each{un, tr_half{wv, un{load{wg, 1}}}} } } - def ms{v}{h} = getsel{to_el{u8,half{v,h}}} + def ms{v}{h} = getsel{to_el{I, if (lanes) half{v,h} else tupsel{h,v}}} def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' # Offset at end off := U~~V**i8~~(gap - 1) # Midpoint bits for each step - def lowbits = bb{copy{bytes,bytes}} + def lowbits = bb{copy{isub,isub}} bits := each{{j} => U**(lowbits << j), iota{lstep}} # Unroll sizes up to a full lane, handling extra lanes conditionally # in the largest one @@ -279,15 +286,14 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { def last = klog==lstep def this = if (not last) log==klog else log>=klog if (this) @for_vec_overlap{vl} (j to xn) { - def as_u8{T,op, ...par} = T~~op{...each{bind{to_el,u8},par}} xv:= load{*V~~(x+j), 0} - s := U**bb{iota{bytes}} # Select sequential bytes within each U - def ltx{se,ind} = lt{xv, as_u8{V,se, ind}} + s := U**bb{iota{isub}} # Select sequential bytes within each U + def ltx{se,ind} = lt{xv, V~~se{to_el{I,ind}}} @unroll (j to klog) { m := s | tupsel{klog-1-j,bits} s = homBlend{m, s, ltx{selw, m}} } - r := if (T==i8) s else s>>(lb{bytes}+wd-8) + r := if (isub==1) s else s>>(lb{isub}+wd-wi) # Extra selection lanes if (last and ex>=1 and log>=klog+1) { r += r From 1fa00c9c6f589b6f363821f1080f34023abb5987 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 8 Jul 2023 17:20:31 -0400 Subject: [PATCH 31/34] Update bins implementation comments --- src/builtins/grade.h | 28 ++++++++++++++++++++++------ src/singeli/src/bins.singeli | 2 ++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 58466ac5..47f80e79 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -19,15 +19,31 @@ // SHOULD widen odd cell sizes under 8 bytes in sort and grade // Bins +// Length 0 or 1 𝕨: trivial, or comparison +// Stand-alone 𝕨 sortedness check +// SHOULD vectorize sortedness check on lists of numbers // Mixed integer and character arguments gives all 0 or ≠𝕨 -// Integers and characters: 4-byte branchless binary search +// Non-Singeli, integers and characters: +// 4-byte branchless binary search, 4-byte output +// SHOULD support fast character searches +// SHOULD special-case boolean 𝕨 or 𝕩 +// Different widths: widen narrower argument +// SHOULD narrow wider-type 𝕩 if it isn't much shorter +// SHOULD trim wider-type 𝕨 and possibly narrow +// Same-width numbers: +// Output type based on ≠𝕨 +// Short 𝕨: vector binary search (then linear on extra lanes) +// 1- or 2-byte type, long enough 𝕩: lookup table from ⌈` +// Binary gallops to skip long repeated elements of 𝕨 +// 1-byte, no duplicates or few uniques: vector bit-table lookup +// General: interleaved branchless binary search +// COULD start interleaved search with a vector binary round // General case: branching binary search -// SHOULD implement f64 branchless binary search -// SHOULD interleave multiple branchless binary searches -// SHOULD specialize bins on equal types at least -// SHOULD implement table-based ⍋⍒ for small-range 𝕨 -// SHOULD special-case short 𝕨 +// COULD trim 𝕨 based on range of 𝕩 +// COULD optimize small-range 𝕨 with small-type methods // SHOULD partition 𝕩 when 𝕨 is large +// COULD interpolation search for large 𝕩 and short 𝕨 +// COULD use linear search and galloping for sorted 𝕩 #define GRADE_CAT(N) CAT(GRADE_UD(gradeUp,gradeDown),N) #define GRADE_NEG GRADE_UD(,-) diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 5f47e52e..ab04ddbb 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -126,6 +126,7 @@ fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = { @for (w over j from i to n) store{t, w, j+1} goto{break} } else if ((wi = load{w, i}) == load{w, id}) { + # Gallop md := n - i d2 := undefined{u64} while ((d2=d+d) < md and wi == load{w, i + d2}) d = d2 @@ -236,6 +237,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { setlabel{no_bittab} } +# Binary search within vector registers def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { assert{wn > 1}; assert{wn < maxwn} def wd = width{T} From 9511598aa384ef9a275d4c45e2c8fe8296d91ae1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 9 Jul 2023 11:21:31 -0400 Subject: [PATCH 32/34] Fast Bins when either argument is boolean --- src/builtins/grade.h | 45 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 47f80e79..35efb726 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -26,7 +26,7 @@ // Non-Singeli, integers and characters: // 4-byte branchless binary search, 4-byte output // SHOULD support fast character searches -// SHOULD special-case boolean 𝕨 or 𝕩 +// Boolean 𝕨 or 𝕩: lookup table (single binary search on boolean 𝕨) // Different widths: widen narrower argument // SHOULD narrow wider-type 𝕩 if it isn't much shorter // SHOULD trim wider-type 𝕨 and possibly narrow @@ -378,7 +378,23 @@ bool CAT(isSorted,GRADE_UD(Up,Down))(B x) { #undef CMP } +// Location of first 1 (ascending) or 0 (descending), by binary search +static u64 CAT(bit_boundary,GRADE_UD(up,dn))(u64* x, u64 n) { + u64 c = GRADE_UD(,~)(u64)0; + u64 *s = x-1; + for (usz l = BIT_N(n)+1, h; (h=l/2)>0; l-=h) { + u64* m = s+h; if (!(c LT *m)) s = m; + } + ++s; // Word containing boundary + u64 b = 64*(s-x); + if (b >= n) return n; + u64 v = GRADE_UD(~,) *s; + if (b+63 >= n) v &= ~(u64)0 >> ((-n)%64); + return b + POPC(v); +} + extern B CAT(GRADE_UD(le,ge),c2)(B,B,B); +extern B select_c2(B t, B w, B x); B GRADE_CAT(c2)(B t, B w, B x) { if (isAtm(w) || RNK(w)==0) thrM(GRADE_CHR": 𝕨 must have rank≥1"); @@ -431,11 +447,32 @@ B GRADE_CAT(c2)(B t, B w, B x) { if (LIKELY(wexe? we : xe; - if (ze==el_bit) ze = el_i8; - if (ze > we) { switch (ze) { default:UD; case el_i8:w=toI8Any(w);break; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; case el_f64:w=toF64Any(w);break; } } - if (ze > xe) { switch (ze) { default:UD; case el_i8:x=toI8Any(x);break; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; case el_f64:x=toF64Any(x);break; } } + if (ze > we) { switch (ze) { default:UD; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; case el_f64:w=toF64Any(w);break; } } + if (ze > xe) { switch (ze) { default:UD; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; case el_f64:x=toF64Any(x);break; } } we = ze; #else if (!elInt(we) | !elInt(xe)) goto gen; From 492e97e2ca6db129c7a75103a4bd7cbf5af83f5e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 9 Jul 2023 19:31:04 -0400 Subject: [PATCH 33/34] Right argument narrowing for numeric Bins --- src/builtins/grade.h | 47 ++++++++++++++++++++++++++++-------- src/singeli/src/bins.singeli | 23 ++++++++++++++++++ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 35efb726..7891bc53 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -27,8 +27,8 @@ // 4-byte branchless binary search, 4-byte output // SHOULD support fast character searches // Boolean 𝕨 or 𝕩: lookup table (single binary search on boolean 𝕨) -// Different widths: widen narrower argument -// SHOULD narrow wider-type 𝕩 if it isn't much shorter +// Different widths: generally widen narrower argument +// Narrow wider-type 𝕩 instead if it isn't much shorter // SHOULD trim wider-type 𝕨 and possibly narrow // Same-width numbers: // Output type based on ≠𝕨 @@ -393,8 +393,10 @@ static u64 CAT(bit_boundary,GRADE_UD(up,dn))(u64* x, u64 n) { return b + POPC(v); } -extern B CAT(GRADE_UD(le,ge),c2)(B,B,B); +#define LE_C2 CAT(GRADE_UD(le,ge),c2) +extern B LE_C2(B,B,B); extern B select_c2(B t, B w, B x); +extern B mul_c2(B, B, B); B GRADE_CAT(c2)(B t, B w, B x) { if (isAtm(w) || RNK(w)==0) thrM(GRADE_CHR": 𝕨 must have rank≥1"); @@ -421,7 +423,7 @@ B GRADE_CAT(c2)(B t, B w, B x) { B c = IGet(w, 0); if (LIKELY(wexe? we : xe; - if (ze > we) { switch (ze) { default:UD; case el_i16:w=toI16Any(w);break; case el_i32:w=toI32Any(w);break; case el_f64:w=toF64Any(w);break; } } - if (ze > xe) { switch (ze) { default:UD; case el_i16:x=toI16Any(x);break; case el_i32:x=toI32Any(x);break; case el_f64:x=toF64Any(x);break; } } - we = ze; + #define WIDEN(E, X) switch (E) { default:UD; case el_i16:X=toI16Any(X);break; case el_i32:X=toI32Any(X);break; case el_f64:X=toF64Any(X);break; } + if (xe > we) { + if (xia/4 < wia) { // Narrow x + assert(el_i8 <=we && we<=el_i32); + assert(el_i16<=xe && xe<=el_f64); + i32 pre = -1; pre<<=(8<<(we-el_i8))-1; + pre = GRADE_UD(pre,-1-pre); // Smallest value of w's type + i32 w0 = o2iG(IGetU(w,0)); + // Saturation is correct except it can move low values past + // pre. Post-adjust with mult×r + if (w0 == pre) mult = LE_C2(m_f64(0), m_i32(pre), incG(x)); + // Narrow x with saturating conversion + B xn; void *xp = m_tyarrc(&xn, elWidth(we), x, el2t(we)); + u8 ind = xe xe) WIDEN(we, x) + #undef WIDEN #else if (!elInt(we) | !elInt(xe)) goto gen; w=toI32Any(w); x=toI32Any(x); @@ -484,7 +509,7 @@ B GRADE_CAT(c2)(B t, B w, B x) { for (u64 i=0; ibf) d = b + } +} + +exportT{ + 'si_saturate', + each{{a}=>saturate{...a}, merge{ + tup{tup{i16,i8}, tup{i32,i8}, tup{i32,i16}}, + join{table{bind{tup,f64}, tup{i8,i16,i32}, tup{1,0}}} + }} +} From f6d1f9fcabf45036511e29cd93b3dca261c9730e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 9 Jul 2023 19:57:34 -0400 Subject: [PATCH 34/34] Move utilities from bins to appropriate files --- src/singeli/src/base.singeli | 13 +++++++++++++ src/singeli/src/bins.singeli | 20 +++----------------- src/singeli/src/cbqnDefs.singeli | 8 ++++++++ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index d9905189..d7111202 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -24,6 +24,11 @@ def ctz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_ctzll', x} def ctz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_ctz', x} def clz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_clzll', x} def clz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_clz', x} +# count-leading-zeros complement, less type-dependent +def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} +def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} + +def ceil_log2{n} = clzc{n-1} def truncBits{n, v & n<=8} = cast_i{u8, v} def truncBits{n, v & n==16} = cast_i{u16, v} @@ -175,6 +180,7 @@ def cvt{...x} = assert{'cvt not supported', show{...x}} def shuf{...x} = assert{'shuf not supported', show{...x}} def shuf16Lo{...x} = assert{'shuf16Lo not supported', show{...x}} def shuf16Hi{...x} = assert{'shuf16Hi not supported', show{...x}} +def shufHalves{...x} = assert{'shufHalves not supported', show{...x}} def homAll{...x} = assert{'homAll not supported', show{...x}} def homAny{...x} = assert{'homAny not supported', show{...x}} def homMask{...x} = assert{'homMask not supported', show{...x}} @@ -281,6 +287,13 @@ def forNZ{vars,begin,end,iter} = { ++i } } +def for_backwards{vars,begin,end,iter} = { + i:u64 = end + while (i > begin) { + --i + iter{i, vars} + } +} def forUnroll{exp,unr}{vars,begin,end,iter} = { i:u64 = begin while ((i+unr) <= end) { diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 062c3db5..16e0558a 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,5 +1,5 @@ include './base' -def shufHalves{...x} = assert{'shufHalves not supported', show{...x}} +include './cbqnDefs' if (hasarch{'AVX2'}) { include './sse' include './avx' @@ -8,14 +8,8 @@ if (hasarch{'AVX2'}) { include './mask' include 'util/tup' -def for_backwards{vars,begin,end,iter} = { - i:u64 = end - while (i > begin) { - --i - iter{i, vars} - } -} def for_dir{up} = if (up) for else for_backwards + def for_vec_overlap{vl}{vars,begin==0,n,iter} = { assert{n >= vl} def end = makelabel{} @@ -28,8 +22,6 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = { setlabel{end} } -def ceil_log2{n:u64} = 64 - clz{n-1} - # Shift as u16, since x86 is missing 8-bit shifts def shr16{v:V, n} = V~~(to_el{u16, v} >> n) @@ -70,13 +62,6 @@ fn max_scan{T, up}(x:*T, len:u64) : void = { } } -def fmt_type{T} = { - def w = width{T} - merge{quality{T}, if (w==8) '8' else if (w==16) '16' else if (w==32) '32' else '64'} -} -def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len} -def tfree{ptr} = emit{void, 'TFREE', ptr} - def getsel{...x} = assert{'shuffling not supported', show{...x}} if (hasarch{'AVX2'}) { def getsel{h:H & lvec{H, 16, 8}} = { @@ -103,6 +88,7 @@ def uninterleave{x:V & hasarch{'AVX2'}} = { def rtypes = tup{i8, i16, i32, f64} # Return index of smallest possible result type given max result value +# (Unused; done in C for now) def get_rtype{len} = { t:u8 = 0 def c{T, ...ts} = if (len>maxvalue{T}) { ++t; c{...ts} } diff --git a/src/singeli/src/cbqnDefs.singeli b/src/singeli/src/cbqnDefs.singeli index 4a89a523..b76ff5f7 100644 --- a/src/singeli/src/cbqnDefs.singeli +++ b/src/singeli/src/cbqnDefs.singeli @@ -31,3 +31,11 @@ def cbqn_elType{T & T==u16} = 6 def cbqn_elType{T & T==u32} = 7 def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'} + +def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len} +def tfree{ptr} = emit{void, 'TFREE', ptr} +def fmt_type{T} = { + def w = match (width{T}) { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } + merge{quality{T}, w} +} +def fmt_type{T & isptr{T}} = merge{'*',fmt_type{eltype{T}}}