From b655dd47710199b2fdc65f64839f612a9b6b5600 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 1 Mar 2023 22:21:11 -0500 Subject: [PATCH 1/8] =?UTF-8?q?AVX2=20counting=20function=20for=20small-ra?= =?UTF-8?q?nge=201-byte=20/=E2=81=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/src/build.bqn | 1 + makefile | 2 +- src/builtins/slash.c | 10 ++++- src/singeli/src/count.singeli | 71 +++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/singeli/src/count.singeli diff --git a/build/src/build.bqn b/build/src/build.bqn index 9ade08f4..b354f478 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -604,6 +604,7 @@ cachedBin‿linkerCache ← { "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", + "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/makefile b/makefile index 9b992474..3d3927c5 100644 --- a/makefile +++ b/makefile @@ -352,7 +352,7 @@ endif @"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c count.c bits.c transpose.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 9d127312..ed82ed13 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -850,6 +850,14 @@ B slash_c2(B t, B w, B x) { return c2rt(slash, w, x); } +#if SINGELI_AVX2 + #define SINGELI_FILE count + #include "../utils/includeSingeli.h" + #define SINGELI_COUNT_OR(N) \ + if (N==8) avx2_count_i8(t, (u8*)xp, xia); else +#else + #define SINGELI_COUNT_OR(N) +#endif B slash_im(B t, B x) { if (!isArr(x) || RNK(x)!=1) thrM("/⁼: Argument must be an array"); @@ -907,7 +915,7 @@ B slash_im(B t, B x) { } else { \ TALLOC(usz, t, m); \ for (usz j=0; jm/2) thrM("/⁼: Argument cannot contain negative numbers"); \ i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i { r = F{r, extract{x, i}} }, iota{vcount{T}}} + r +} +def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { + c:= x + def EW = elwidth{T} + if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} + if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} + if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} + if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} + extract{c, 0} +} +def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} + +def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} +def inc{ptr, ind} = inc{ptr, ind, 1} + +fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { + def vbits = 256 + def vec = vbits/width{T} + def uT = ty_u{T} + def V = [vec]uT + def iV = [vec]T + def block = (1024*8) / vbits # Desired vectors per block + i:u64 = 0 + while (i < n) { + r:u64 = n - i + b := r / vec + xv := *V~~x + used_eq:u1 = 0 + if (r >= 256) { + b = block; if (r < vec*b) b = r / vec + mv := V**0 + @for (xv over b) mv = max{mv, xv} + mi := iV~~mv + if (homAny{mi < iV**0}) return{1} + if (homAll{mi <= iV**32}) { + used_eq = 1 + r = b * vec + m := fold{max, mv} + total := b*vec + @for (j to promote{u64,m}) { + c := V**0 + e := V**trunc{uT, j} + @for (xv over b) c -= xv == e + s := fold{+, [vec/2]i16~~fold{+, unpackQ{iV~~c, iV**0}}} + total -= promote{u64, s} + inc{tab, j, promote{usz, s}} + } + inc{tab, m, trunc{usz,total}} + } + } + if (not used_eq) @for (x over r) inc{tab, x} + i += r + x += r + } + 0 +} + +export{'avx2_count_i8', count{i8}} From e6a3e846d9f2c75cc8577378998e648c1b905385 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 2 Mar 2023 10:42:06 -0500 Subject: [PATCH 2/8] Sum counts in groups of 4 when possible --- src/singeli/src/count.singeli | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index cfbbf963..a593b3c7 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -23,6 +23,8 @@ def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { } def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} +fn sum_vec{T==[32]i8}(v:T) = fold{+, [16]i16~~fold{+, unpackQ{v, T**0}}} + def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} @@ -39,25 +41,30 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { b := r / vec xv := *V~~x used_eq:u1 = 0 - if (r >= 256) { + if (r >= 128) { b = block; if (r < vec*b) b = r / vec mv := V**0 @for (xv over b) mv = max{mv, xv} mi := iV~~mv if (homAny{mi < iV**0}) return{1} - if (homAll{mi <= iV**32}) { + if (homAll{mi <= iV**48}) { used_eq = 1 r = b * vec - m := fold{max, mv} - total := b*vec - @for (j to promote{u64,m}) { - c := V**0 - e := V**trunc{uT, j} - @for (xv over b) c -= xv == e - s := fold{+, [vec/2]i16~~fold{+, unpackQ{iV~~c, iV**0}}} - total -= promote{u64, s} - inc{tab, j, promote{usz, s}} + m := promote{u64, fold{max, mv}} + total := trunc{usz, b*vec} + def count_each{j} = { + c := copy{tuplen{j}, V**0} + e := each{{j}=>V**trunc{uT, j}, j} + @for (xv over b) each{{c,e} => c -= xv == e, c, e} + def add_sum{c, j} = { + s := promote{usz, sum_vec{iV}(iV~~c)} + total -= s; inc{tab, j, s} + } + each{add_sum, c, j} } + m4 := m / 4 + @for (j4 to m4) count_each{each{{k}=>4*j4 + k, iota{4}}} + @for (j from 4*m4 to m) count_each{tup{j}} inc{tab, m, trunc{usz,total}} } } From 2ddcc14852a55958cd233cad246579effa8e3d80 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 2 Mar 2023 15:39:27 -0500 Subject: [PATCH 3/8] Use minimum in addition to maximum for AVX2 counting --- src/singeli/src/count.singeli | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index a593b3c7..b9b9fa15 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -34,7 +34,8 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { def uT = ty_u{T} def V = [vec]uT def iV = [vec]T - def block = (1024*8) / vbits # Desired vectors per block + def block = (2048*8) / vbits # Target vectors per block + assert{block < 1<= 128) { b = block; if (r < vec*b) b = r / vec - mv := V**0 - @for (xv over b) mv = max{mv, xv} + jv := load{xv}; mv := jv + @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } mi := iV~~mv if (homAny{mi < iV**0}) return{1} - if (homAll{mi <= iV**48}) { + jt := fold{min, jv} + if (homAll{mi <= iV**(48 + i8~~jt)}) { used_eq = 1 r = b * vec - m := promote{u64, fold{max, mv}} + j0 := promote{u64, jt} + m := promote{u64, fold{max, mv}} - j0 total := trunc{usz, b*vec} - def count_each{j} = { + def count_each{js, num} = { + j := (@collect (k to num) js+k) c := copy{tuplen{j}, V**0} e := each{{j}=>V**trunc{uT, j}, j} @for (xv over b) each{{c,e} => c -= xv == e, c, e} @@ -63,9 +67,9 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { each{add_sum, c, j} } m4 := m / 4 - @for (j4 to m4) count_each{each{{k}=>4*j4 + k, iota{4}}} - @for (j from 4*m4 to m) count_each{tup{j}} - inc{tab, m, trunc{usz,total}} + @for (j4 to m4) count_each{j0 + 4*j4, 4} + @for (j from 4*m4 to m) count_each{j0 + j, 1} + inc{tab, j0 + m, trunc{usz,total}} } } if (not used_eq) @for (x over r) inc{tab, x} From 06b4f06e643517c5dbbb68d803bcd1cef09100b9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 17 Mar 2023 12:07:29 -0400 Subject: [PATCH 4/8] Move fold{} from squeeze.singeli to vecfold.singeli --- src/singeli/src/count.singeli | 23 ++++------------------- src/singeli/src/squeeze.singeli | 27 ++------------------------- src/singeli/src/vecfold.singeli | 27 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 44 deletions(-) create mode 100644 src/singeli/src/vecfold.singeli diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index b9b9fa15..bc393e69 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -3,27 +3,12 @@ include './sse' include './avx' include './avx2' include 'util/tup' - -# TODO merge with squeeze -def fold{F, x:T} = { - show{'WARNING: using fallback fold'} - def E = eltype{T} - r:E = 0 - each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} - r -} -def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { - c:= x - def EW = elwidth{T} - if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} - if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} - if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} - if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} - extract{c, 0} -} -def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} +include './vecfold' fn sum_vec{T==[32]i8}(v:T) = fold{+, [16]i16~~fold{+, unpackQ{v, T**0}}} +def minmax{c, a, b} = tern{c{cast_i{i8,a},cast_i{i8,b}}, a, b} +def min{a:i16, b:i16} = minmax{<, a, b} +def max{a:i16, b:i16} = minmax{>, a, b} def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index a282dfd7..9b01d230 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -11,6 +11,7 @@ if (hasarch{'AVX2'}) { include './mask' include './cbqnDefs' include 'util/tup' +include './vecfold' def preserve_negative_zero = 0 @@ -42,30 +43,6 @@ def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = { -def fold{F, x:T} = { - show{'WARNING: using fallback fold for ', F, T} - def E = eltype{T} - r:E = 0 - each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} - r -} -def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = { - c:= x - def EW = elwidth{T} - if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} - if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} - if (hasarch{'SSSE3'} and 0) { - if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} - if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} - extract{c, 0} - } else { - if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} - if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} } - else extract{c, 0} - } -} -def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} - def makeOptBranch{enable, F} = { if (enable) { def skip = makelabel{}; goto{skip} @@ -209,4 +186,4 @@ export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}} export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}} export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}} -export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} \ No newline at end of file +export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli new file mode 100644 index 00000000..b123b596 --- /dev/null +++ b/src/singeli/src/vecfold.singeli @@ -0,0 +1,27 @@ +# Fold associative/commutative operation across a register +# Used by squeeze.singeli, count.singeli +# Has to be included after util/tup because of name conflict + +def fold{F, x:T} = { + show{'WARNING: using fallback fold for ', F, T} + def E = eltype{T} + r:E = 0 + each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} + r +} +def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = { + c:= x + def EW = elwidth{T} + if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} + if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} + if (hasarch{'SSSE3'} and 0) { + if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} + if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} + extract{c, 0} + } else { + if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} + if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{eltype{T}, F{v, v>>8}} } + else extract{c, 0} + } +} +def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} From 4b18466ae296c650aaee856617cd2fa83410fbd2 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 17 Mar 2023 13:22:36 -0400 Subject: [PATCH 5/8] Clean up and simplify count.singeli; allow longer final block --- src/singeli/src/count.singeli | 70 ++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index bc393e69..28fa8688 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -17,47 +17,49 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { def vbits = 256 def vec = vbits/width{T} def uT = ty_u{T} - def V = [vec]uT - def iV = [vec]T + def V = [vec]T def block = (2048*8) / vbits # Target vectors per block - assert{block < 1< vec*b_max) r = vec*block + b := r / vec # Vector case does b full vectors if it runs + r0:u64 = 0 # Elements actually handled by vector case + + # Find range to check for suitability xv := *V~~x - used_eq:u1 = 0 - if (r >= 128) { - b = block; if (r < vec*b) b = r / vec - jv := load{xv}; mv := jv - @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } - mi := iV~~mv - if (homAny{mi < iV**0}) return{1} - jt := fold{min, jv} - if (homAll{mi <= iV**(48 + i8~~jt)}) { - used_eq = 1 - r = b * vec - j0 := promote{u64, jt} - m := promote{u64, fold{max, mv}} - j0 - total := trunc{usz, b*vec} - def count_each{js, num} = { - j := (@collect (k to num) js+k) - c := copy{tuplen{j}, V**0} - e := each{{j}=>V**trunc{uT, j}, j} - @for (xv over b) each{{c,e} => c -= xv == e, c, e} - def add_sum{c, j} = { - s := promote{usz, sum_vec{iV}(iV~~c)} - total -= s; inc{tab, j, s} - } - each{add_sum, c, j} + jv := load{xv}; mv := jv + @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } + jt := fold{min, jv} + mt := fold{max, mv} - jt # Counts needed (last one's implicit) + if (jt < 0) return{1} # Negative number found! + + if (mt <= 48) { + r0 = b * vec + j0 := promote{u64, uT~~jt} # Starting count + m := promote{u64, uT~~mt} # Number of iterations + total := trunc{usz, r0} # To compute last count + def count_each{js, num} = { + j := @collect (k to num) js+k + c := copy{tuplen{j}, [vec]uT ** 0} + e := each{{j}=>V**trunc{T, j}, j} + @for (xv over b) each{{c,e} => c -= xv == e, c, e} + def add_sum{c, j} = { + s := promote{usz, sum_vec{V}(V~~c)} + total -= s; inc{tab, j, s} } - m4 := m / 4 - @for (j4 to m4) count_each{j0 + 4*j4, 4} - @for (j from 4*m4 to m) count_each{j0 + j, 1} - inc{tab, j0 + m, trunc{usz,total}} + each{add_sum, c, j} } + m4 := m / 4 + @for (j4 to m4) count_each{j0 + 4*j4, 4} + @for (j from 4*m4 to m) count_each{j0 + j, 1} + inc{tab, j0 + m, trunc{usz,total}} } - if (not used_eq) @for (x over r) inc{tab, x} + + # Scalar fallback and cleanup + @for (x over _ from r0 to r) inc{tab, x} i += r x += r } From 0e5b98c4912224b26aa6dc7a399fa9f57f4241dc Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 17 Mar 2023 14:00:29 -0400 Subject: [PATCH 6/8] Return max from AVX2 counting function --- src/builtins/slash.c | 31 ++++++++++++++++++++----------- src/singeli/src/count.singeli | 22 ++++++++++++++-------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index ed82ed13..553fc8f4 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -85,6 +85,9 @@ #define SINGELI_FILE constrep #include "../utils/includeSingeli.h" + #define SINGELI_FILE count + #include "../utils/includeSingeli.h" + extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3); extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); @@ -850,15 +853,6 @@ B slash_c2(B t, B w, B x) { return c2rt(slash, w, x); } -#if SINGELI_AVX2 - #define SINGELI_FILE count - #include "../utils/includeSingeli.h" - #define SINGELI_COUNT_OR(N) \ - if (N==8) avx2_count_i8(t, (u8*)xp, xia); else -#else - #define SINGELI_COUNT_OR(N) -#endif - B slash_im(B t, B x) { if (!isArr(x) || RNK(x)!=1) thrM("/⁼: Argument must be an array"); u8 xe = TI(x,elType); @@ -912,10 +906,10 @@ B slash_im(B t, B x) { usz m=1<m/2) thrM("/⁼: Argument cannot contain negative numbers"); \ i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i, a, b} def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} -fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { +# Write counts /⁼x to tab and return ⌈´x +fn count{T}(tab:*usz, x:*T, n:u64) : T = { def vbits = 256 def vec = vbits/width{T} def uT = ty_u{T} @@ -21,25 +22,30 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { def block = (2048*8) / vbits # Target vectors per block def b_max = block + block/4 # Last block max length assert{b_max < 1< vec*b_max) r = vec*block b := r / vec # Vector case does b full vectors if it runs + rv:= b * vec r0:u64 = 0 # Elements actually handled by vector case - # Find range to check for suitability + # Find range to check for suitability; return a negative if found xv := *V~~x jv := load{xv}; mv := jv @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } + @for (x over _ from rv to r) { if (x<0) return{x}; if (x>mx) mx=x } jt := fold{min, jv} - mt := fold{max, mv} - jt # Counts needed (last one's implicit) - if (jt < 0) return{1} # Negative number found! + mt := fold{max, mv} + if (jt < 0) return{jt} + if (mt > mx) mx = mt - if (mt <= 48) { - r0 = b * vec + nc := mt - jt # Number of counts to perform: last is implicit + if (nc <= 48) { + r0 = rv j0 := promote{u64, uT~~jt} # Starting count - m := promote{u64, uT~~mt} # Number of iterations + m := promote{u64, uT~~nc} # Number of iterations total := trunc{usz, r0} # To compute last count def count_each{js, num} = { j := @collect (k to num) js+k @@ -63,7 +69,7 @@ fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { i += r x += r } - 0 + mx } export{'avx2_count_i8', count{i8}} From 9d7d330a03b99b0fba53ce0d2e68ebfcbf5afb8f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 17 Mar 2023 15:44:31 -0400 Subject: [PATCH 7/8] Use AVX2 counting for 1-byte counting sort --- src/builtins/grade.h | 7 ++++++- src/builtins/slash.c | 2 +- src/singeli/src/count.singeli | 18 +++++++++--------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index bf270303..828a8e69 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -81,6 +81,9 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); if (e==n) {break;} k=e; \ } #define WRITE_SPARSE(T) WRITE_SPARSE_##T +extern i8 (*const avx2_count_i8)(usz*, i8*, u64, i8); +#define SINGELI_COUNT_OR(T) \ + if (1==sizeof(T)) avx2_count_i8(c0o, (i8*)xp, n, -128); else #else #define COUNT_THRESHOLD 16 #define WRITE_SPARSE(T) \ @@ -88,13 +91,14 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); usz js = j; \ while (ijmx) mx=x } + @for (x over _ from rv to r) { if (xmx) mx=x } jt := fold{min, jv} mt := fold{max, mv} - if (jt < 0) return{jt} + if (jt < min_allowed) return{jt} if (mt > mx) mx = mt - nc := mt - jt # Number of counts to perform: last is implicit + nc := uT~~(mt - jt) # Number of counts to perform: last is implicit if (nc <= 48) { r0 = rv j0 := promote{u64, uT~~jt} # Starting count - m := promote{u64, uT~~nc} # Number of iterations + m := promote{u64, nc} # Number of iterations total := trunc{usz, r0} # To compute last count def count_each{js, num} = { - j := @collect (k to num) js+k + j := @collect (k to num) trunc{T, js+k} c := copy{tuplen{j}, [vec]uT ** 0} - e := each{{j}=>V**trunc{T, j}, j} + e := each{{j}=>V**j, j} @for (xv over b) each{{c,e} => c -= xv == e, c, e} def add_sum{c, j} = { s := promote{usz, sum_vec{V}(V~~c)} @@ -61,7 +61,7 @@ fn count{T}(tab:*usz, x:*T, n:u64) : T = { m4 := m / 4 @for (j4 to m4) count_each{j0 + 4*j4, 4} @for (j from 4*m4 to m) count_each{j0 + j, 1} - inc{tab, j0 + m, trunc{usz,total}} + inc{tab, trunc{T, j0 + m}, trunc{usz,total}} } # Scalar fallback and cleanup From d13060f72507f4c0a4bbd76d504974eb8fe9d39d Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 6 Apr 2023 19:05:38 +0300 Subject: [PATCH 8/8] move fold to vfold --- src/singeli/src/base.singeli | 4 ++-- src/singeli/src/count.singeli | 9 +++------ src/singeli/src/neon.singeli | 3 +++ src/singeli/src/squeeze.singeli | 2 +- src/singeli/src/vecfold.singeli | 18 ++++++------------ 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 39eb3005..af264639 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -184,8 +184,8 @@ def undefPromote{T, x:X & T==X} = T~~x def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}} def cvt{T, x:X & T==eltype{X}} = x -def min{a, b & knum{a} & knum{b}} = tern{ab, a, b} +def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 89bd16e5..d7ae54c3 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -5,10 +5,7 @@ include './avx2' include 'util/tup' include './vecfold' -fn sum_vec{T==[32]i8}(v:T) = fold{+, [16]i16~~fold{+, unpackQ{v, T**0}}} -def minmax{c, a, b} = tern{c{cast_i{i8,a},cast_i{i8,b}}, a, b} -def min{a:i16, b:i16} = minmax{<, a, b} -def max{a:i16, b:i16} = minmax{>, a, b} +fn sum_vec{T==[32]i8}(v:T) = vfold{+, [16]i16~~fold{+, unpackQ{v, T**0}}} def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} def inc{ptr, ind} = inc{ptr, ind, 1} @@ -36,8 +33,8 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { jv := load{xv}; mv := jv @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } @for (x over _ from rv to r) { if (xmx) mx=x } - jt := fold{min, jv} - mt := fold{max, mv} + jt := vfold{min, jv} + mt := vfold{max, mv} if (jt < min_allowed) return{jt} if (mt > mx) mx = mt diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 810a84d4..53575347 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -71,6 +71,9 @@ def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a} def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a} def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a} +def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x} +def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x} +def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x} # TODO don't rely on regular stores being unaligned local def storeu{ptr:P, e:T} = store{ptr, 0, e} diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 9b01d230..35348be3 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -69,7 +69,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E} # fold with either Max or Bitwise Or, truncating/zero-extending to TE - def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}} + def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}} def foldTotal{TE, x:T & hasarch{'AARCH64'}} = { if (elwidth{T}==64) { if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index b123b596..330122eb 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -2,26 +2,20 @@ # Used by squeeze.singeli, count.singeli # Has to be included after util/tup because of name conflict -def fold{F, x:T} = { +def vfold{F, x:T} = { show{'WARNING: using fallback fold for ', F, T} def E = eltype{T} r:E = 0 each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} r } -def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = { +def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { c:= x def EW = elwidth{T} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} - if (hasarch{'SSSE3'} and 0) { - if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} - if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} - extract{c, 0} - } else { - if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} - if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{eltype{T}, F{v, v>>8}} } - else extract{c, 0} - } + if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} + if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} } + else extract{c, 0} } -def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} +def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}