From e681f3c09a71343c65fa8c7ab6214bbf80264776 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 29 Apr 2024 21:04:27 -0400 Subject: [PATCH 01/16] =?UTF-8?q?Run-based=201-byte=20/=E2=81=BC=20impleme?= =?UTF-8?q?ntation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/count.singeli | 83 ++++++++++++++++++++++++++--------- src/singeli/src/neon.singeli | 2 + 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 546b0ffc..88b38c8d 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -28,36 +28,31 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { r0:u64 = 0 # Elements actually handled by vector case # Find range to check for suitability; return a negative if found + # Also record number of differences dc + # (double-counts at index vec but it doesn't need to be exact) xv := *V~~x - jv := load{xv}; mv := jv - @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } + jv := load{xv}; mv := jv; dc := -(jv != load{*V~~(x+1)}) + @for (xv, xp in *V~~(x-1) over _ from 1 to b) { + jv = min{jv, xv}; mv = max{mv, xv} + dc -= xp != xv + } @for (x over _ from rv to r) { if (xmx) mx=x } jt := vfold{min, jv} mt := vfold{max, mv} if (jt < min_allowed) return{jt} if (mt > mx) mx = mt + # Fast cases + dt := promote{u64, fold_addw{dc}} nc := uT~~(mt - jt) # Number of counts to perform: last is implicit - if (nc <= 24*vbits/128) { + if (dt < b * (vec/2) and dt*8 < b * promote{u64,nc}) { + r0 = count_with_runs{V, vec, x, tab, r} + } else if (nc <= 24*vbits/128) { r0 = rv - j0 := promote{u64, uT~~jt} # Starting count - m := promote{u64, nc} # Number of iterations - total := trunc{usz, r0} # To compute last count - def count_each{js, num} = { - j := @collect (k to num) trunc{T, js+k} - c := copy{length{j}, [vec]uT ** 0} - e := each{{j}=>V**j, j} - @for (xv over b) each{{c,e} => c -= xv == e, c, e} - def add_sum{c, j} = { - s := promote{usz, fold_addw{c}} - total -= s; inc{tab, j, s} - } - each{add_sum, c, j} + count_by_sum{T, V, [vec]uT, xv, b, tab, r0, + promote{u64, uT~~jt}, # Starting count + promote{u64, nc} # Number of iterations } - m4 := m / 4 - @for (j4 to m4) count_each{j0 + 4*j4, 4} - @for (j from 4*m4 to m) count_each{j0 + j, 1} - inc{tab, trunc{T, j0 + m}, trunc{usz,total}} } # Scalar fallback and cleanup @@ -68,4 +63,52 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { mx } +# Sum comparisons against each value (except one) in the range +def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { + total := trunc{usz, r0} # To compute last count + def count_each{js, num} = { + j := @collect (k to num) trunc{T, js+k} + c := copy{length{j}, U**0} + e := each{{j}=>V**j, j} + @for (xv over b) each{{c,e} => c -= xv == e, c, e} + def add_sum{c, j} = { + s := promote{usz, fold_addw{c}} + total -= s; inc{tab, j, s} + } + each{add_sum, c, j} + } + m4 := m / 4 + @for (j4 to m4) count_each{j0 + 4*j4, 4} + @for (j from 4*m4 to m) count_each{j0 + j, 1} + inc{tab, trunc{T, j0 + m}, trunc{usz,total}} +} + +# Count adjacent equal elements at once, breaking at w-element groups +# May read up to index r from x, hitting one element that's not counted +def count_with_runs{V, vec, x, tab, r} = { + def w = width{ux} + m0:ux = 1 << (w-1) # Last element in each chunk ends a run + bw := r / w + @for (i to bw) { + xo := x + i*w + m := m0 + # Mark the end of each run + @unroll (j to w / vec) { + def jv = j*vec + def lv{k} = load{*V~~(xo + k)} + m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv + } + # Iterate over runs + jp:usz = - usz~~1 + while (m > m0) @unroll (2) { + j := trunc{usz, ctz{m}} + inc{tab, load{xo, j}, j - jp} + jp = j; m &= m-1 + } + # One step if popc{m} was odd, reducing branch mispredictions above + inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{usz, m>>(w-1)}} + } + bw * w +} + export{'avx2_count_i8', count{i8}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 94e9195f..9b5c21b7 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -15,6 +15,8 @@ def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}} def ntyp0{S, T} = merge{S, '_', nty{T}} +def __neg{a:T if nvecu{T}} = T~~(-ty_s{T}~~a) + def __lt{a:T, 0 if nvecs{T} or nvecf{T}} = emit{ty_u{T}, ntyp{'vcltz', T}, a} def __le{a:T, 0 if nvecs{T} or nvecf{T}} = emit{ty_u{T}, ntyp{'vclez', T}, a} def __gt{a:T, 0 if nvecs{T} or nvecf{T}} = emit{ty_u{T}, ntyp{'vcgtz', T}, a} From e24d0bac634f1e97b9a1f44960710dd1ce2d69ac Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 14 Nov 2024 11:58:13 -0500 Subject: [PATCH 02/16] Extend SIMD counting to i16 in addition to i8 --- src/builtins/grade.h | 4 ++-- src/builtins/slash.c | 30 ++++++++++++++---------------- src/singeli/src/count.singeli | 8 +++++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index be95c6fc..406a21e6 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -97,9 +97,9 @@ extern void (*const si_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); if (e==n) {break;} k=e; \ } #define WRITE_SPARSE(T) WRITE_SPARSE_##T -extern i8 (*const avx2_count_i8)(usz*, i8*, u64, i8); +extern i8 (*const simd_count_i8)(usz*, i8*, u64, i8); #define SINGELI_COUNT_OR(T) \ - if (1==sizeof(T)) avx2_count_i8(c0o, (i8*)xp, n, -128); else + if (1==sizeof(T)) simd_count_i8(c0o, (i8*)xp, n, -128); else #else #define COUNT_THRESHOLD 16 #define WRITE_SPARSE(T) \ diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 2934f9b1..07128741 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -865,12 +865,10 @@ B slash_im(B t, B x) { usz m=1<m/2) thrM("/⁼: Argument cannot contain negative numbers"); \ + SINGELI_COUNT(N) \ i32* rp; r = m_i32arrv(&rp, ria); vfor (usz i=0; im/2) thrM("/⁼: Argument cannot contain negative numbers"); #endif CASE_SMALL(8) CASE_SMALL(16) #undef CASE_SMALL -#undef SINGELI_COUNT_OR +#undef SINGELI_COUNT_ALLOC +#undef SINGELI_COUNT case el_i32: { i32* xp = i32any_ptr(x); IIND_INT(32) r = num_squeeze(r); break; } #undef IIND_INT case el_f64: { diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 88b38c8d..1888b3f0 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -3,14 +3,14 @@ include './vecfold' if_inline (hasarch{'SSE2'}) { fn sum_vec{T}(v:T) = vfold{+, fold{+, mzip128{v, T**0}}} - def fold_addw{v:T=[_](u8)} = sum_vec{T}(v) + def fold_addw{v:T=[_]E if E<=u16} = sum_vec{T}(v) } def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} # Write counts /⁼x to tab and return ⌈´x -fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { +fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { def vbits = arch_defvw def vec = vbits/width{T} def uT = ty_u{T} @@ -18,6 +18,7 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { def block = (2048*8) / vbits # Target vectors per block def b_max = block + block/4 # Last block max length assert{b_max < 1< Date: Thu, 14 Nov 2024 12:06:54 -0500 Subject: [PATCH 03/16] =?UTF-8?q?More=20precise=20sparse=20check=20for=20i?= =?UTF-8?q?8=20/=E2=81=BC=20result:=20previously=20stopped=20at=2065?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 07128741..0bb765a9 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -837,11 +837,11 @@ B slash_im(B t, B x) { if (xia < ria/8) { \ u8 maxcount = 0; \ TALLOC(u8, tab, ria); \ - for (usz i=0; i=16 && maxcount<127) { \ + if (maxcount<=1) a=xia; \ + else if (N>=16 && maxcount<128) { \ i8* rp; r = m_i8arrv(&rp, ria); for (usz i=0; i Date: Thu, 14 Nov 2024 15:07:20 -0500 Subject: [PATCH 04/16] =?UTF-8?q?Allow=20i16=20/=E2=81=BC=20to=20jump=20to?= =?UTF-8?q?=20Singeli=20code=20after=20range=20checking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 0bb765a9..1ecb4937 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -826,7 +826,7 @@ B slash_im(B t, B x) { rp[sum>0] = sum; rp[0] = xia - sum; r = num_squeeze(r); break; } -#define IIND_INT(N) \ +#define IIND_INT(N, CHECK_RIA) \ if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \ usz a=1; while (axp[a-1]) a++; \ u##N max=xp[a-1]; \ @@ -856,32 +856,35 @@ B slash_im(B t, B x) { break; \ } \ usz ria = (usz)max + 1; \ + CHECK_RIA \ i##N* rp; r = m_i##N##arrv(&rp, ria); for (usz i=0; i Date: Thu, 14 Nov 2024 22:18:59 -0500 Subject: [PATCH 05/16] =?UTF-8?q?Run-based=20i32=20/=E2=81=BC=20(leaving?= =?UTF-8?q?=20out=20sum-based)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 37 ++++++++++++++-------- src/singeli/src/count.singeli | 58 +++++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 1ecb4937..69366eab 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -826,7 +826,7 @@ B slash_im(B t, B x) { rp[sum>0] = sum; rp[0] = xia - sum; r = num_squeeze(r); break; } -#define IIND_INT(N, CHECK_RIA) \ +#define TRY_SMALL_OUT(N) \ if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \ usz a=1; while (axp[a-1]) a++; \ u##N max=xp[a-1]; \ @@ -841,11 +841,7 @@ B slash_im(B t, B x) { for (usz i=0; i=16 && maxcount<128) { \ - i8* rp; r = m_i8arrv(&rp, ria); for (usz i=0; i=16 && maxcount<128) { INIT_RES(8) FILL_RES break; } \ } \ } \ if (a==xia) { /* Unique argument */ \ @@ -855,9 +851,11 @@ B slash_im(B t, B x) { for (usz i=0; i vec*b_max) r = vec*block + iter{r} + i += r + } +} + # Write counts /⁼x to tab and return ⌈´x fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { def vbits = arch_defvw def vec = vbits/width{T} def uT = ty_u{T} def V = [vec]T - def block = (2048*8) / vbits # Target vectors per block - def b_max = block + block/4 # Last block max length - assert{b_max < 1< vec*b_max) r = vec*block + mx:T = min_allowed # Maximum of x + block_loop{V, n, {r} => { # Handle r elements b := r / vec # Vector case does b full vectors if it runs rv:= b * vec r0:u64 = 0 # Elements actually handled by vector case @@ -58,9 +65,8 @@ fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { # Scalar fallback and cleanup @for (x over _ from r0 to r) inc{tab, x} - i += r x += r - } + }} mx } @@ -86,7 +92,7 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { # Count adjacent equal elements at once, breaking at w-element groups # May read up to index r from x, hitting one element that's not counted -def count_with_runs{V, vec, x, tab, r} = { +def count_with_runs{V, vec, x, tab:*T, r} = { def w = width{ux} m0:ux = 1 << (w-1) # Last element in each chunk ends a run bw := r / w @@ -100,17 +106,37 @@ def count_with_runs{V, vec, x, tab, r} = { m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv } # Iterate over runs - jp:usz = - usz~~1 + jp:T = - T~~1 while (m > m0) @unroll (2) { - j := trunc{usz, ctz{m}} - inc{tab, load{xo, j}, j - jp} + j := trunc{T, ctz{m}} + inc{tab, load{xo, j}, cast_i{T, j - jp}} jp = j; m &= m-1 } # One step if popc{m} was odd, reducing branch mispredictions above - inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{usz, m>>(w-1)}} + inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}} } bw * w } +# Condensed version without count_by_sum +fn count_i32_i32(tab:*i32, x:*i32, n:u64) : void = { + def T = i32 + def vbits = arch_defvw + def vec = vbits/width{T} + def V = [vec]T + block_loop{V, n, {r} => { + b := r / vec + xv := *V~~x + dc := -(load{xv} != load{*V~~(x+1)}) + @for (xv, xp in *V~~(x-1) over _ from 1 to b) dc -= xp != xv + dt := promote{u64, fold_addw{dc}} + r0:u64 = 0 + if (dt < b * (vec/2)) r0 = count_with_runs{V, vec, x, tab, r} + @for (x over _ from r0 to r) inc{tab, x} + x += r + }} +} + export{'simd_count_i8', count{i8}} export{'simd_count_i16', count{i16}} +export{'simd_count_i32_i32', count_i32_i32} From 11117fcc677dbe568eda24ed2bda44afdea14df5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 15 Nov 2024 17:37:11 -0500 Subject: [PATCH 06/16] Version of count_i32_i32 without large blocks --- src/singeli/src/count.singeli | 76 +++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 476dbce9..aec7e8a0 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -54,7 +54,7 @@ fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { dt := promote{u64, fold_addw{dc}} nc := uT~~(mt - jt) # Number of counts to perform: last is implicit if (dt < b * (vec/2) and dt*8 < b * promote{u64,nc}) { - r0 = count_with_runs{V, vec, x, tab, r} + r0 = count_with_runs{x, tab, r} } else if (nc <= 24*vbits/128) { r0 = rv count_by_sum{T, V, [vec]uT, xv, b, tab, r0, @@ -92,49 +92,55 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { # Count adjacent equal elements at once, breaking at w-element groups # May read up to index r from x, hitting one element that's not counted -def count_with_runs{V, vec, x, tab:*T, r} = { +def count_with_runs{x, tab, r} = { def w = width{ux} m0:ux = 1 << (w-1) # Last element in each chunk ends a run bw := r / w @for (i to bw) { xo := x + i*w - m := m0 - # Mark the end of each run - @unroll (j to w / vec) { - def jv = j*vec - def lv{k} = load{*V~~(xo + k)} - m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv - } - # Iterate over runs - jp:T = - T~~1 - while (m > m0) @unroll (2) { - j := trunc{T, ctz{m}} - inc{tab, load{xo, j}, cast_i{T, j - jp}} - jp = j; m &= m-1 - } - # One step if popc{m} was odd, reducing branch mispredictions above - inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}} + m := m0; mark_run_ends{xo, m} + inc_marked_runs{xo, tab, m, m0} } bw * w } - -# Condensed version without count_by_sum -fn count_i32_i32(tab:*i32, x:*i32, n:u64) : void = { - def T = i32 - def vbits = arch_defvw - def vec = vbits/width{T} +def mark_run_ends{x:*T, m:(ux)} = { + def vec = arch_defvw/width{T} def V = [vec]T - block_loop{V, n, {r} => { - b := r / vec - xv := *V~~x - dc := -(load{xv} != load{*V~~(x+1)}) - @for (xv, xp in *V~~(x-1) over _ from 1 to b) dc -= xp != xv - dt := promote{u64, fold_addw{dc}} - r0:u64 = 0 - if (dt < b * (vec/2)) r0 = count_with_runs{V, vec, x, tab, r} - @for (x over _ from r0 to r) inc{tab, x} - x += r - }} + @unroll (j to width{ux} / vec) { + def jv = j*vec + def lv{k} = load{*V~~(x + k)} + m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv + } +} +def inc_marked_runs{x, tab:*T, m, m0} = { + def w = width{ux} + # Iterate over runs marked in m + jp:T = - T~~1 + while (m > m0) @unroll (2) { + j := trunc{T, ctz{m}} + inc{tab, load{x, j}, cast_i{T, j - jp}} + jp = j; m &= m-1 + } + # One step if popc{m} was odd, reducing branch mispredictions above + inc{tab, load{x, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}} +} + +# No count_by_sum: build each run mask then decide whether to use it +fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = { + def w = width{ux} + m0:ux = 1 << (w-1) + while (n > 0) { + b:usz = w + if (rare{b > n}) { b = n; goto{'skip_runs'} } + m := m0; mark_run_ends{x, m} + if (popc{m} < w/2) { + inc_marked_runs{x, tab, m, m0} + } else { + setlabel{'skip_runs'} + @for (x over b) inc{tab, x} + } + x += b; n -= b + } } export{'simd_count_i8', count{i8}} From e6940e73d0b80b197be9f137e10057ad02661714 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 15 Nov 2024 20:55:15 -0500 Subject: [PATCH 07/16] =?UTF-8?q?Fast=20/=E2=81=BC=20of=20sorted=20argumen?= =?UTF-8?q?ts=20using=20semi-sparse=20representation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 41 ++++++++++++++++- src/singeli/src/count.singeli | 86 ++++++++++++++++++++++++++++------- 2 files changed, 109 insertions(+), 18 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 69366eab..92af67d2 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -812,6 +812,30 @@ B slash_c2(B t, B w, B x) { return c2rt(slash, w, x); } +#if SINGELI_SIMD +static B finish_sorted_count(B r, usz* ov, usz* oc, usz on) { + // Overflow values in ov are sorted but not unique + // Set mo to the greatest sum of oc for equal ov values + usz mo = 0, pv = 0, c = 0; + for (usz i=0; imo) mo=c; + } + // Since mo is a multiple of 128 and all of r is less than 128, + // values in r can't affect the result type + #define RESIZE(T, UT) \ + r = taga(cpy##UT##Arr(r)); T* rp = tyany_ptr(r); \ + for (usz i=0; ixp[a-1]) a++; \ u##N max=xp[a-1]; \ if (amax) max=c; } \ if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ usz ria = max + 1; \ @@ -862,7 +887,7 @@ B slash_im(B t, B x) { i##N* xp = i##N##any_ptr(x); \ usz m=1<m/2) thrM("/⁼: Argument cannot contain negative numbers"); + #define HAS_SINGELI_COUNT_SORTED 0 + #define SINGELI_COUNT_SORTED(N) #endif CASE_SMALL(8) CASE_SMALL(16) #undef CASE_SMALL diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index aec7e8a0..0152ae26 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -91,17 +91,36 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { } # Count adjacent equal elements at once, breaking at w-element groups -# May read up to index r from x, hitting one element that's not counted -def count_with_runs{x, tab, r} = { +# May read up to index n from x, hitting one element that's not counted +def count_with_runs{x, tab, n} = { def w = width{ux} m0:ux = 1 << (w-1) # Last element in each chunk ends a run - bw := r / w + bw := n / w @for (i to bw) { xo := x + i*w m := m0; mark_run_ends{xo, m} inc_marked_runs{xo, tab, m, m0} } - bw * w + bw * w # Number of elements handled +} +# Switch to the normal scalar count if there aren't enough runs +def count_adapt_runs{x0, tab, n} = { + def w = width{ux} + m0:ux = 1 << (w-1) + x := x0; r := n + while (r > 0) { + def skip_runs = makelabel{} + b:usz = w + if (rare{b > r}) { b = r; goto{skip_runs} } + m := m0; mark_run_ends{x, m} + if (popc{m} < w/2) { + inc_marked_runs{x, tab, m, m0} + } else { + setlabel{skip_runs} + @for (x over b) inc{tab, x} + } + x += b; r -= b + } } def mark_run_ends{x:*T, m:(ux)} = { def vec = arch_defvw/width{T} @@ -126,23 +145,56 @@ def inc_marked_runs{x, tab:*T, m, m0} = { } # No count_by_sum: build each run mask then decide whether to use it -fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = { - def w = width{ux} - m0:ux = 1 << (w-1) - while (n > 0) { - b:usz = w - if (rare{b > n}) { b = n; goto{'skip_runs'} } - m := m0; mark_run_ends{x, m} - if (popc{m} < w/2) { - inc_marked_runs{x, tab, m, m0} - } else { - setlabel{'skip_runs'} - @for (x over b) inc{tab, x} +fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = count_adapt_runs{x, tab, n} + +# For i←/⁼x, store r←128|i, and i-r sparsely: x is ∧(/r)∾oc/ov +# ov is sorted but may not be unique, and oc contains multiples of 128 +# Return the shared length of ov and oc +fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = { + def V = [arch_defvw/width{T}]T + def block = 128 + i:usz = 0 + on:usz = 0 + def overflow{xu,c} = { store{ov, on, xu}; store{oc, on, c}; ++on } + while (i < n) { + rem := n - i + xo := x + i + xi := load{xo} + def overflow{c} = overflow{cast_i{usz,xi}, c} + xe := xo-1; def bxi{j} = xi == load{xe, j} + if (block <= rem and bxi{block}) { + # Gallop to find last block ending in xi + d:usz = block + d2 := undefined{usz} + while ((d2=d+d) <= rem and bxi{d2}) d = d2 + l := (rem &~ (block-1)) - d; if (l > d) l = d + # Target is in [d,d+l); shrink l + while (l > block) { + h := (l/2) &~ (block-1) + m := d + h + if (bxi{m}) d = m + l -= h + } + overflow{d} + rem -= d; if (rem == 0) return{on} + i += d; xo += d; xi = load{xo} } - x += b; n -= b + # Count the next block normally + if (rem > block) rem = block + count_adapt_runs{xo, r, rem} + rxi := load{r, xi} + if (rxi >= block) { + store{r, xi, rxi - block} + overflow{block} + } + i += rem } + on } export{'simd_count_i8', count{i8}} export{'simd_count_i16', count{i16}} export{'simd_count_i32_i32', count_i32_i32} +export{'si_count_sorted_i8', count_sorted{i8}} +export{'si_count_sorted_i16', count_sorted{i16}} +export{'si_count_sorted_i32', count_sorted{i32}} From 96b198f6436e78f17be32038e5881d3f5dfb9c4c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 16 Nov 2024 08:17:12 -0500 Subject: [PATCH 08/16] =?UTF-8?q?Condition=20all=20the=20i8=20to=20i32=20/?= =?UTF-8?q?=E2=81=BC=20optimization=20on=20SINGELI=5FSIMD,=20for=20simplic?= =?UTF-8?q?ity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 91 ++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 92af67d2..302320c9 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -850,12 +850,22 @@ B slash_im(B t, B x) { rp[sum>0] = sum; rp[0] = xia - sum; r = num_squeeze(r); break; } -#define TRY_SMALL_OUT(N) \ +#if SINGELI_SIMD + #define TRY_SMALL_OUT(N) \ if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \ usz a=1; while (axp[a-1]) a++; \ u##N max=xp[a-1]; \ if (amax) max=c; } \ if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ usz ria = max + 1; \ @@ -877,79 +887,60 @@ B slash_im(B t, B x) { break; \ } \ usz ria = (usz)max + 1; -#define INIT_RES(N) \ + #define INIT_RES(N) \ i##N* rp; r = m_i##N##arrv(&rp, ria); \ for (usz i=0; im/2) thrM("/⁼: Argument cannot contain negative numbers"); - #define HAS_SINGELI_COUNT_SORTED 0 - #define SINGELI_COUNT_SORTED(N) -#endif CASE_SMALL(8) CASE_SMALL(16) -#undef CASE_SMALL -#undef RIA_SMALL -#undef SINGELI_COUNT_ALLOC -#undef SINGELI_COUNT + #undef CASE_SMALL case el_i32: { i32* xp = i32any_ptr(x); TRY_SMALL_OUT(32) INIT_RES(32) -#if SINGELI_SIMD simd_count_i32_i32(rp, xp, xia); -#else - FILL_RES -#endif r = num_squeeze(r); break; } -#undef TRY_SMALL_OUT -#undef INIT_RES -#undef FILL_RES + #undef TRY_SMALL_OUT + #undef INIT_RES + #undef FILL_RES +#else + #define CASE(N) case el_i##N: { \ + i##N* xp = i##N##any_ptr(x); \ + u##N max=xp[0]; \ + for (usz i=1; imax) max=c; } \ + if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ + usz ria = max + 1; \ + TALLOC(usz, t, ria); \ + for (usz j=0; j Date: Sat, 16 Nov 2024 08:27:36 -0500 Subject: [PATCH 09/16] Squeeze for slash_im generic case instead of converting individual elements to numbers --- src/builtins/slash.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 302320c9..a80703cc 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -842,6 +842,7 @@ B slash_im(B t, B x) { usz xia = IA(x); if (xia==0) { decG(x); return emptyIVec(); } B r; + retry: switch(xe) { default: UD; case el_bit: { usz sum = bit_sum(bitany_ptr(x), xia); @@ -957,21 +958,12 @@ B slash_im(B t, B x) { break; } case el_c8: case el_c16: case el_c32: case el_B: { - SLOW1("/⁼", x); + x = num_squeezeChk(x); + xe = TI(x,elType); + if (elNum(xe)) goto retry; B* xp = TO_BPTR(x); - usz i,j; i64 max=-1; - for (i = 0; i < xia; i++) { i64 c=o2i64(xp[i]); if (c<=max) break; max=c; } - for (j = i; j < xia; j++) { i64 c=o2i64(xp[j]); max=c>max?c:max; if (c<0) thrM("/⁼: Argument cannot contain negative numbers"); } - if (max > USZ_MAX-1) thrOOM(); - usz ria = max+1; - if (i==xia) { - u64* rp; r = m_bitarrv(&rp, ria); for (usz i=0; i Date: Sat, 16 Nov 2024 20:56:25 -0500 Subject: [PATCH 10/16] =?UTF-8?q?Do=20general=20i8=20and=20i16=20/?= =?UTF-8?q?=E2=81=BC=20counts=20to=20i16=20buffer,=20plus=20overflow=20lis?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/grade.h | 32 ++++++++++--- src/builtins/slash.c | 86 ++++++++++++++++++++++------------- src/singeli/src/count.singeli | 59 ++++++++++++++++-------- 3 files changed, 119 insertions(+), 58 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 406a21e6..06eb3ee2 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -97,9 +97,27 @@ extern void (*const si_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); if (e==n) {break;} k=e; \ } #define WRITE_SPARSE(T) WRITE_SPARSE_##T -extern i8 (*const simd_count_i8)(usz*, i8*, u64, i8); -#define SINGELI_COUNT_OR(T) \ - if (1==sizeof(T)) simd_count_i8(c0o, (i8*)xp, n, -128); else +extern i8 (*const simd_count_i8)(u16*, u16*, void*, u64, i8); +#define COUNTING_SORT_i8 \ + usz C=1<<8; \ + TALLOC(u16, c0, C+(n>>15)+1); \ + u16 *c0o=c0+C/2; u16 *ov=c0+C; \ + for (usz j=0; j 0)) { // Overflowed i32! + r = taga(cpyF64Arr(r)); f64* rp = tyany_ptr(r); + for (usz i=0; ixp[a-1]) a++; \ u##N max=xp[a-1]; \ + usz rmax=xia; \ if (a=16 && maxcount<128) { INIT_RES(8) FILL_RES break; } \ + else if (N>=16 && maxcount<128) rmax=127; \ } \ } \ + usz ria = (usz)max + 1; \ if (a==xia) { /* Unique argument */ \ - usz ria = max + 1; \ u64* rp; r = m_bitarrv(&rp, ria); \ for (usz i=0; i>15; \ + TALLOC(u16, ov, os+1); \ + i##N max = simd_count_i##N((u16*)rp, (u16*)ov, xp, xia, 0); \ + if (max < 0) thrM("/⁼: Argument cannot contain negative numbers"); \ + usz ria = (usz)max + 1; \ + if (ria < sa) r = C2(take, m_f64(ria), r); \ + r = finish_small_count(r, ov); \ + TFREE(ov); \ + break; \ } CASE_SMALL(8) CASE_SMALL(16) #undef CASE_SMALL case el_i32: { i32* xp = i32any_ptr(x); TRY_SMALL_OUT(32) - INIT_RES(32) + INIT_RES(32,ria) simd_count_i32_i32(rp, xp, xia); r = num_squeeze(r); break; } #undef TRY_SMALL_OUT #undef INIT_RES - #undef FILL_RES #else #define CASE(N) case el_i##N: { \ i##N* xp = i##N##any_ptr(x); \ diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 0152ae26..7f23fa9e 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -6,31 +6,24 @@ if_inline (hasarch{'SSE2'}) { def fold_addw{v:T=[_]E if E<=u32} = sum_vec{T}(v) } -def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} +def inc{ptr:*T, ind, v} = store{ptr, ind, trunc{T,v} + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} -def block_loop{V=[vec]T, n, iter} = { - def block = (2048*8) / width{V} # Target vectors per block - def b_max = block + block/4 # Last block max length - assert{b_max < 1< vec*b_max) r = vec*block - iter{r} - i += r - } -} - -# Write counts /⁼x to tab and return ⌈´x -fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { +# Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x +fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { def vbits = arch_defvw def vec = vbits/width{T} def uT = ty_u{T} def V = [vec]T + def block = (2048*8) / vbits # Target vectors per block + def b_max = block + block/4 # Last block max length + assert{b_max < 1< { # Handle r elements + i:u64 = 0 + while (i < n) { + # Number of elements to handle in this iteration + r:u64 = n - i; if (r > vec*b_max) r = vec*block b := r / vec # Vector case does b full vectors if it runs rv:= b * vec r0:u64 = 0 # Elements actually handled by vector case @@ -65,11 +58,37 @@ fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = { # Scalar fallback and cleanup @for (x over _ from r0 to r) inc{tab, x} + i += r x += r - }} + + # Keep counts below 1<<15 with the overflow list + # Count from the end to include i==n and handle a long last block nicely + if ((i-n)%(1<<15) < block*vec and i >= 1<<15) { + ov += flush_counts(tab+min_allowed, ov, cast_i{usz,ty_u{mx+min_allowed}} + 1) + } + } + store{ov, 0, maxvalue{u16}} # End marker: note x values fit in i16 mx } +fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = { + def vl = arch_defvw/16 + def V = [vl]u16 + def bot = 1<<15 - 1 + on:usz = 0 + @for (t in *V~~tab over jv to cdiv{n, vl}) if (rare{topAny{t}}) { + o := if (hasarch{'X86_64'}) topMask{t} else homMask{t > V**bot} + if (jv == n/vl) o &= type{o}~~1<<(n%vl) - 1 + while (o > 0) { + jv := jv*vl + cast_i{usz, ctz{o}} + store{tab, jv, load{tab, jv} & bot} + store{ov, on, trunc{u16, jv}}; ++on + o &= o-1 + } + } + on +} + # Sum comparisons against each value (except one) in the range def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { total := trunc{usz, r0} # To compute last count @@ -87,7 +106,7 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { m4 := m / 4 @for (j4 to m4) count_each{j0 + 4*j4, 4} @for (j from 4*m4 to m) count_each{j0 + j, 1} - inc{tab, trunc{T, j0 + m}, trunc{usz,total}} + inc{tab, trunc{T, j0 + m}, total} } # Count adjacent equal elements at once, breaking at w-element groups @@ -137,7 +156,7 @@ def inc_marked_runs{x, tab:*T, m, m0} = { jp:T = - T~~1 while (m > m0) @unroll (2) { j := trunc{T, ctz{m}} - inc{tab, load{x, j}, cast_i{T, j - jp}} + inc{tab, load{x, j}, j - jp} jp = j; m &= m-1 } # One step if popc{m} was odd, reducing branch mispredictions above From 194e8c3f22138ee2498f8ac635a818ef9d08ff57 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 16 Nov 2024 20:57:47 -0500 Subject: [PATCH 11/16] =?UTF-8?q?Update=20/=E2=81=BC=20implementation=20co?= =?UTF-8?q?mments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 15077100..2a0fe97d 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -54,14 +54,21 @@ // Indices inverse (/⁼), a lot like Group // Always gives a squeezed result for integer 𝕩 // Boolean 𝕩: just count 1s -// Long i8 and i16 𝕩: count into zeroed buffer before anything else -// Only zero positive part; if total is too small there were negatives -// Cutoff is set so short 𝕩 gives a result of the same type +// Without SINGELI_SIMD, just write to large-type table and squeeze +// COULD do many /⁼ optimizations without SIMD // Scan for strictly ascending 𝕩 -// COULD vectorize with find-compare -// Unsigned maximum for integers to avoid a separate negative check -// If (≠÷⌈´)𝕩 is small, find result type with a sparse u8 table -// COULD use a u16 table for i32 𝕩 to detect i16 result +// SHOULD vectorize, maybe with find-compare +// Sorted indices: i8 counter and index+count overflow +// Work in blocks of 128, try galloping if one has start equal to end +// Otherwise use runs-adaptive count (not sums, they're rarely better) +// Long i8 and i16 𝕩: allocate full-range to skip initial range check +// If (≠÷⌈´)𝕩 is small, detect u1 and i8 result with a sparse u8 table +// General-case i8 to i32 𝕩: dedicated SIMD functions +// i8 and i16 𝕩: i16 counter and index overflow (implicit count 1<<15) +// Flush to overflow every 1<<15 writes +// Get range in 2KB blocks to enable count by compare and sum +// Run detection used partly to mitigate write stalls from repeats +// COULD also alternate writes to multiple tables if 𝕩 is long enough #include "../core.h" #include "../utils/mut.h" From 342cadfc008230554bc1556078a82625072b27db Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 16 Nov 2024 21:20:54 -0500 Subject: [PATCH 12/16] =?UTF-8?q?Check=20for=20or=20handle=20argument=20le?= =?UTF-8?q?ngth=20>2=E2=8B=8631=20in=20/=E2=81=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 2a0fe97d..ec515e75 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -951,10 +951,10 @@ B slash_im(B t, B x) { case el_i32: { i32* xp = i32any_ptr(x); TRY_SMALL_OUT(32) + if (xia>I32_MAX) thrM("/⁼: Argument too large"); INIT_RES(32,ria) simd_count_i32_i32(rp, xp, xia); - r = num_squeeze(r); - break; + r = num_squeeze(r); break; } #undef TRY_SMALL_OUT #undef INIT_RES @@ -968,10 +968,12 @@ B slash_im(B t, B x) { TALLOC(usz, t, ria); \ for (usz j=0; jI32_MAX) thrM("/⁼: Argument too large"); i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i Date: Sun, 17 Nov 2024 09:23:51 -0500 Subject: [PATCH 13/16] =?UTF-8?q?Better=20sum=20versus=20runs=20test=20for?= =?UTF-8?q?=20/=E2=81=BC:=20account=20for=20per-element=20cost=20of=20run-?= =?UTF-8?q?based?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/count.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 7f23fa9e..9201a047 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -46,7 +46,7 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { # Fast cases dt := promote{u64, fold_addw{dc}} nc := uT~~(mt - jt) # Number of counts to perform: last is implicit - if (dt < b * (vec/2) and dt*8 < b * promote{u64,nc}) { + if (dt < b * (vec/2) and (b + dt)*4 < b * promote{u64,nc}) { r0 = count_with_runs{x, tab, r} } else if (nc <= 24*vbits/128) { r0 = rv From 33ae8744f587a0577bf461d33bd6480a5b8503ed Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 17 Nov 2024 10:16:48 -0500 Subject: [PATCH 14/16] =?UTF-8?q?Fast=20skip=20ahead=20for=20blocks=20of?= =?UTF-8?q?=20equal=20values=20in=20i8=20and=20i16=20/=E2=81=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/count.singeli | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 9201a047..e06695c6 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -32,8 +32,18 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { # Also record number of differences dc # (double-counts at index vec but it doesn't need to be exact) xv := *V~~x - jv := load{xv}; mv := jv; dc := -(jv != load{*V~~(x+1)}) - @for (xv, xp in *V~~(x-1) over _ from 1 to b) { + jv := load{xv}; mv := jv + ne := jv != load{*V~~(x+1)}; dc := -ne + # Quickly skip ahead if initial values are all equal + a:u64 = 1 + if (not homAny{ne} and b>=4) { + def eq_k{k} = homAll{tree_fold{&, @unroll(x in xv+a over k) x==jv}} + def skip_eq{k} = if (eq_k{k}) { a=2*k; skip_eq{2*k} } + def skip_eq{k==4} = while (a<=b-k and eq_k{k}) a+=k + skip_eq{1} + } + # Now start analysis + @for (xv, xp in *V~~(x-1) over _ from a to b) { jv = min{jv, xv}; mv = max{mv, xv} dc -= xp != xv } From f0223b8bafda4c7d9ecc2e25870e2749e0ec6bce Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 21 Nov 2024 23:02:51 +0200 Subject: [PATCH 15/16] properly add 1<<31 --- src/builtins/slash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index ec515e75..7b8167bf 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -837,7 +837,7 @@ static B finish_small_count(B r, u16* ov) { } if (RARE(on > 0)) { // Overflowed i32! r = taga(cpyF64Arr(r)); f64* rp = tyany_ptr(r); - for (usz i=0; i Date: Thu, 21 Nov 2024 13:20:36 +0200 Subject: [PATCH 16/16] a couple renames & simplifications --- src/builtins/slash.c | 2 +- src/singeli/src/count.singeli | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7b8167bf..c09e00c4 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -932,7 +932,7 @@ B slash_im(B t, B x) { usz sa = 1<<(N-1); \ if (xia < sa || FL_HAS(x,fl_asc)) { \ TRY_SMALL_OUT(N) \ - if (N==8) UD; \ + assert(N != 8); \ sa = ria; \ } \ INIT_RES(16,sa) \ diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index e06695c6..5ed41fca 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -13,7 +13,7 @@ def inc{ptr, ind} = inc{ptr, ind, 1} fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { def vbits = arch_defvw def vec = vbits/width{T} - def uT = ty_u{T} + def TU = ty_u{T} def V = [vec]T def block = (2048*8) / vbits # Target vectors per block def b_max = block + block/4 # Last block max length @@ -55,13 +55,13 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = { # Fast cases dt := promote{u64, fold_addw{dc}} - nc := uT~~(mt - jt) # Number of counts to perform: last is implicit + nc := TU~~(mt - jt) # Number of counts to perform: last is implicit if (dt < b * (vec/2) and (b + dt)*4 < b * promote{u64,nc}) { r0 = count_with_runs{x, tab, r} } else if (nc <= 24*vbits/128) { r0 = rv - count_by_sum{T, V, [vec]uT, xv, b, tab, r0, - promote{u64, uT~~jt}, # Starting count + count_by_sum{T, V, [vec]TU, xv, b, tab, r0, + promote{u64, TU~~jt}, # Starting count promote{u64, nc} # Number of iterations } } @@ -104,7 +104,7 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = { total := trunc{usz, r0} # To compute last count def count_each{js, num} = { j := @collect (k to num) trunc{T, js+k} - c := copy{length{j}, U**0} + c := length{j} ** U**0 e := each{{j}=>V**j, j} @for (xv over b) each{{c,e} => c -= xv == e, c, e} def add_sum{c, j} = { @@ -196,7 +196,7 @@ fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = { d:usz = block d2 := undefined{usz} while ((d2=d+d) <= rem and bxi{d2}) d = d2 - l := (rem &~ (block-1)) - d; if (l > d) l = d + l := min{(rem &~ (block-1)) - d, d} # Target is in [d,d+l); shrink l while (l > block) { h := (l/2) &~ (block-1) @@ -209,7 +209,7 @@ fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = { i += d; xo += d; xi = load{xo} } # Count the next block normally - if (rem > block) rem = block + rem = min{rem, usz~~block} # TODO get rid of the need of the usz~~ here count_adapt_runs{xo, r, rem} rxi := load{r, xi} if (rxi >= block) {