From 00c454d37a5b0f1f2f68763e625e74d727fdd96b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 9 Aug 2023 19:42:29 -0400 Subject: [PATCH 1/7] Add generic-architecture versions of Singeli scans --- build/src/build.bqn | 2 +- src/builtins/grade.h | 12 ++--- src/builtins/radix.h | 22 ++++---- src/builtins/scan.c | 22 ++++---- src/builtins/slash.c | 21 +++++--- src/singeli/src/scan.singeli | 98 ++++++++++++++++++++---------------- 6 files changed, 98 insertions(+), 79 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index f3db89ba..731e4d81 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -650,7 +650,7 @@ cachedBin‿linkerCache ← { "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", "xag"‿"src/builtins/sort.c"‿"bins" - "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", + "2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan", "2a."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", "xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ diff --git a/src/builtins/grade.h b/src/builtins/grade.h index fe0b5cd3..708edf14 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -78,22 +78,22 @@ } #if SINGELI_AVX2 -extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2); -extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2); -extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2); -extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); +extern void (*const si_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2); +extern void (*const si_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2); +extern void (*const si_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2); +extern void (*const si_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); #define COUNT_THRESHOLD 32 #define WRITE_SPARSE_i8 \ for (usz i=0; ish{4b1110,sh{4b2200,a}}} } -export{'avx2_scan_min_init_i8', avx2_scan_idem{i8 , min}}; export{'avx2_scan_max_init_i8', avx2_scan_idem{i8 , max}} -export{'avx2_scan_min_init_i16', avx2_scan_idem{i16, min}}; export{'avx2_scan_max_init_i16', avx2_scan_idem{i16, max}} -export{'avx2_scan_min_init_i32', avx2_scan_idem{i32, min}}; export{'avx2_scan_max_init_i32', avx2_scan_idem{i32, max}} -export{'avx2_scan_min_init_f64', avx2_scan_idem{f64, min}}; export{'avx2_scan_max_init_f64', avx2_scan_idem{f64, max}} -fn avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = { - def m = 1 << (width{T}-1) - def id = (if (same{op,min}) m-1; else -m) - avx2_scan_idem{T, op}(x, r, len, id) +export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}} +export{'si_scan_min_init_i16', scan_idem{i16, min}}; export{'si_scan_max_init_i16', scan_idem{i16, max}} +export{'si_scan_min_init_i32', scan_idem{i32, min}}; export{'si_scan_max_init_i32', scan_idem{i32, max}} +export{'si_scan_min_init_f64', scan_idem{f64, min}}; export{'si_scan_max_init_f64', scan_idem{f64, max}} + +fn scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = { + scan_idem{T, op}(x, r, len, (if (same{op,min}) maxvalue else minvalue){T}) } -export{'avx2_scan_min_i8', avx2_scan_idem_id{i8 , min}}; export{'avx2_scan_max_i8', avx2_scan_idem_id{i8 , max}} -export{'avx2_scan_min_i16', avx2_scan_idem_id{i16, min}}; export{'avx2_scan_max_i16', avx2_scan_idem_id{i16, max}} -export{'avx2_scan_min_i32', avx2_scan_idem_id{i32, min}}; export{'avx2_scan_max_i32', avx2_scan_idem_id{i32, max}} +export{'si_scan_min_i8', scan_idem_id{i8 , min}}; export{'si_scan_max_i8', scan_idem_id{i8 , max}} +export{'si_scan_min_i16', scan_idem_id{i16, min}}; export{'si_scan_max_i16', scan_idem_id{i16, max}} +export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', scan_idem_id{i32, max}} # Assumes identity is 0 def scan_assoc{op, a:T} = { @@ -92,17 +97,22 @@ def scan_assoc{op, a:T} = { def scan_plus = scan_assoc{+, .} # Associative scan -fn avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = { +def scan_assoc_0 = scan_scal +fn scan_assoc_0{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = { # Prefix op on entire AVX register - scan_post{T, init, x, r, len, op, scan_plus} } -export{'avx2_scan_pluswrap_u8', avx2_scan_assoc_0{u8 , +}} -export{'avx2_scan_pluswrap_u16', avx2_scan_assoc_0{u16, +}} -export{'avx2_scan_pluswrap_u32', avx2_scan_assoc_0{u32, +}} +export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}} +export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}} +export{'si_scan_pluswrap_u32', scan_assoc_0{u32, +}} # Boolean cumulative sum -fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = { +fn bcs{T}(x:*u64, r:*T, l:u64) : void = { + def bitp_get{arr, n} = (load{arr, n>>6} >> (n&63)) & 1 + c:T = 0 + @for (r over i to l) { c+= cast_i{T, bitp_get{x,i}}; r = c } +} +fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { def U = ty_u{T} def w = width{T} def vl= 256 / w @@ -157,9 +167,9 @@ fn avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = { step{load{xv, e}, e, st} } } -export{'avx2_bcs8', avx2_bcs{i8}} -export{'avx2_bcs16', avx2_bcs{i16}} -export{'avx2_bcs32', avx2_bcs{i32}} +export{'si_bcs8', bcs{i8}} +export{'si_bcs16', bcs{i16}} +export{'si_bcs32', bcs{i32}} @@ -190,7 +200,23 @@ def maxabsval{T & issigned{T}} = -minvalue{T} def maxsafeint{T & issigned{T}} = maxvalue{T} def maxsafeint{T==f64} = 1<<53 -def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = { +fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { + i:u64 = 0 + if (hasarch{'AVX2'}) simd_plus_scan_part{X,R}{x, c, r, len, i} + @forUnroll{1,1} (js from i to len) { + def vs = eachx{load, x, js} + each{{j, v} => { + def {b,n} = addChk{c, promote{R, v}} + if (rare{b}) return{j} + store{r, j, n} + c = n + }, js, vs} + } + len +} +# Sum as many vector registers as possible; modifies c and i +def simd_plus_scan_part{X, R}{x:*X, c:(R), r:*R, len:u64, i:u64} = { + def b = max{width{R}/2, width{X}} def bulk = arch_defvw/b def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size @@ -203,7 +229,6 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = { if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} } - i:u64 = 0 cv:= [arch_defvw/width{R}]R ** c if (R==f64 and c != floor{c}) goto{'end'} @@ -237,24 +262,13 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = { setlabel{'end'} c = extract{cv, 0} - - @forUnroll{1,1} (js from i to len) { - def vs = eachx{load, x, js} - each{{j, v} => { - def {b,n} = addChk{c, promote{R, v}} - if (rare{b}) return{j} - store{r, j, n} - c = n - }, js, vs} - } - len } -fn simd_plus_scanG{X, b, R}(x:*X, c:R, r:*R, len:u64) : void = simd_plus_scan{X,b,R}{x, c, r, len} -fn simd_plus_scanC{X, b, R}(x:*X, c:R, r:*R, len:u64) : u64 = simd_plus_scan{X,b,R}{x, c, r, len} +def plus_scanG{X, R} = plus_scan{X, R, void} +def plus_scanC{X, R} = plus_scan{X, R, u64} -export{'simd_scan_plus_i8_i32', simd_plus_scanC{i8, 16, i32}} -export{'simd_scan_plus_i16_i32', simd_plus_scanC{i16, 16, i32}} -export{'simd_scan_plus_i32_i32', simd_plus_scanC{i32, 32, i32}} +export{'si_scan_plus_i8_i32', plus_scanC{i8, i32}} +export{'si_scan_plus_i16_i32', plus_scanC{i16, i32}} +export{'si_scan_plus_i32_i32', plus_scanC{i32, i32}} -export{'simd_scan_plus_i16_f64', simd_plus_scanG{i16, 32, f64}} -export{'simd_scan_plus_i32_f64', simd_plus_scanG{i32, 32, f64}} +export{'si_scan_plus_i16_f64', plus_scanG{i16, f64}} +export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} From f9a4a5b68c2652e2e1983e33bdc0777d4fee272b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 9 Aug 2023 20:36:39 -0400 Subject: [PATCH 2/7] Remove duplicated SSE4.1 min/max bindings --- src/singeli/src/sse.singeli | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index e8c7b895..b04b7a43 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -11,18 +11,14 @@ def extract{x:T, i & w128i{T, 8} & knum{i}} = emit{eltype{T}, '_mm_extract_epi8' def extract{x:T, i & w128i{T,32} & knum{i}} = emit{eltype{T}, '_mm_extract_epi32', x, i} def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64', x, i} def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b} -def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b} -def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b} -def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b} -def __le{a:T,b:T & w128u{T}} = a==min{a,b} -def __ge{a:T,b:T & w128u{T}} = a==max{a,b} # arith def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b} def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b} def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b} def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b} +def __le{a:T,b:T & w128u{T}} = a==min{a,b} +def __ge{a:T,b:T & w128u{T}} = a==max{a,b} def __eq{a:T,b:T & w128i{T,64}} = emit{[2]u64, '_mm_cmpeq_epi64', a, b} From e261e801689965c02493f39cc45ad8d60c0c53e7 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 9 Aug 2023 20:51:46 -0400 Subject: [PATCH 3/7] SSE min/max and pluswrap scans --- src/builtins/radix.h | 4 ++-- src/singeli/src/scan.singeli | 40 +++++++++++++++++++++++------------- src/singeli/src/sse2.singeli | 2 ++ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/builtins/radix.h b/src/builtins/radix.h index aabf848d..538ba9e3 100644 --- a/src/builtins/radix.h +++ b/src/builtins/radix.h @@ -11,7 +11,7 @@ #define RDX_SUM_2(T) GRADE_UD(c1[0]=0;,) T s0=0, s1=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); } #define RDX_SUM_4(T) GRADE_UD(c1[0]=c2[0]=c3[0]=0;,) T s0=0, s1=0, s2=0, s3=0; for(usz j=0;j<256;j++) { RDX_PRE(0); RDX_PRE(1); RDX_PRE(2); RDX_PRE(3); } -#if SINGELI_AVX2 +#if SINGELI_X86_64 extern void (*const si_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); #define RADIX_SUM_1_u8 si_scan_pluswrap_u8 (c0,c0, 256,0); @@ -29,7 +29,7 @@ extern void (*const si_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2, #define RADIX_SUM_4_u32 RDX_SUM_4(u32) #endif -#if SINGELI_AVX2 && !USZ_64 +#if SINGELI_X86_64 && !USZ_64 #define RADIX_SUM_1_usz si_scan_pluswrap_u32(c0,c0, 256,0); #define RADIX_SUM_2_usz si_scan_pluswrap_u32(c0,c0,2*256,0); #define RADIX_SUM_4_usz si_scan_pluswrap_u32(c0,c0,4*256,0); diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index e81964ac..bf53b91f 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -10,8 +10,8 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = { @for (x, r over len) r = m = op{m, x} } -def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}} -def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} +def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}} +def sel8{v:V, t & w256{V} & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} @@ -23,13 +23,18 @@ def spread{a:VT} = { } # Set all elements with the last element of the input -def toLast{n:VT} = { +def toLast{n:VT & hasarch{'X86_64'} & w128{VT}} = { + def l{v, w} = l{zipHi{v,v}, 2*w} + def l{v, w & w>=32} = shuf{[4]i32, v, 4**3} + l{n, elwidth{VT}} +} +def toLast{n:VT & hasarch{'AVX2'} & w256{VT}} = { if (elwidth{VT}<=32) sel{[8]i32, spread{n}, [8]i32**7} - else shuf{[4]u64, n, 4b3333} + else shuf{[4]u64, n, 4**3} } def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = { - def step = 256/width{T} + def step = arch_defvw/width{T} def V = [step]T p:= V**init xv:= *V ~~ x @@ -51,7 +56,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { # Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈ def scan_idem = scan_scal -fn scan_idem{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = { +fn scan_idem{T, op & hasarch{'SSE4.1'}}(x:*T, r:*T, len:u64, init:T) : void = { # Within each lane, scan using shifts by powers of 2. First k elements # when shifting by k don't need to change, so leave them alone. def w = width{T} @@ -62,14 +67,19 @@ fn scan_idem{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = { def pre{a} = { b:= c8{2, c8{1, c32{2, c32{1, a}}}} # After lanewise scan, broadcast end of lane 0 to entire lane 1 - op{b, sel{[8]i32, spread{b}, make{[8]i32, 3*(3sh{4b1110,sh{4b2200,a}}} +fn scan_idem{T==f64, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { + def sc{a} = op{a, zipLo{a,a}} + def sc{a & hasarch{'AVX2'}} = { + def sh{s, a} = op{a, shuf{[4]u64, a, s}} + sh{4b1110,sh{4b2200,a}} + } + scan_post{T, init, x, r, len, op, sc} } export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}} @@ -90,15 +100,17 @@ def scan_assoc{op, a:T} = { def w = elwidth{T} def c32{k, a} = (if (w<=8*k) op{a, shl{[16]u8, a, k}}; else a) b:= c32{8, c32{4, c32{2, c32{1, a}}}} - # After lanewise scan, broadcast end of lane 0 to entire lane 1 - l:= (type{b}~~make{[8]i32,0,0,0,-1,0,0,0,0}) & spread{b} - op{b, sel{[8]i32, l, make{[8]i32,0,0,0,0, 3,3,3,3}}} + if (not hasarch{'AVX2'}) b else { + # After lanewise scan, broadcast end of lane 0 to entire lane 1 + l:= (type{b}~~make{[8]i32,0,0,0,-1,0,0,0,0}) & spread{b} + op{b, sel{[8]i32, l, make{[8]i32,0,0,0,0, 3,3,3,3}}} + } } def scan_plus = scan_assoc{+, .} # Associative scan def scan_assoc_0 = scan_scal -fn scan_assoc_0{T, op & hasarch{'AVX2'}}(x:*T, r:*T, len:u64, init:T) : void = { +fn scan_assoc_0{T, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { # Prefix op on entire AVX register scan_post{T, init, x, r, len, op, scan_plus} } diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index b7193df7..c9a86a03 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -189,6 +189,8 @@ def packQ{a:T,b:T & w128i{T}} = packs{a,b} def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b} def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b} +def zipLo{a:T, b:T & w128f{T}} = emit{T, merge{'_mm_unpacklo_p',if (elwidth{T}==32) 's' else 'd'}, a, b} +def zipHi{a:T, b:T & w128f{T}} = emit{T, merge{'_mm_unpackhi_p',if (elwidth{T}==32) 's' else 'd'}, a, b} def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}} From d16ba6c3b137bcd7d70636f98917731b10a30404 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 10 Aug 2023 14:41:33 -0400 Subject: [PATCH 4/7] Move xor-scan to scan.singeli; remove neq.singeli --- build/src/build.bqn | 2 +- src/builtins/scan.c | 8 ++------ src/singeli/src/neq.singeli | 29 ----------------------------- src/singeli/src/scan.singeli | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 36 deletions(-) delete mode 100644 src/singeli/src/neq.singeli diff --git a/build/src/build.bqn b/build/src/build.bqn index 731e4d81..052abf32 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -651,7 +651,7 @@ cachedBin‿linkerCache ← { "xag"‿"src/builtins/sort.c"‿"bins" "2.."‿"src/builtins/select.c"‿"select", "xag"‿"src/builtins/scan.c"‿"scan", - "2a."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", + "2a."‿"src/builtins/slash.c"‿"constrep", "xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 50312557..d650295d 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -12,18 +12,14 @@ static u64 vg_rand(u64 x) { return x; } #if SINGELI #define SINGELI_FILE scan #include "../utils/includeSingeli.h" - #if __PCLMUL__ - #define SINGELI_FILE neq - #include "../utils/includeSingeli.h" - #endif #endif B scan_ne(B x, u64 p, u64 ia) { // consumes x u64* xp = bitarr_ptr(x); u64* rp; B r=m_bitarrv(&rp,ia); -#if SINGELI_AVX2 && __PCLMUL__ - clmul_scan_ne(p, xp, rp, BIT_N(ia)); +#if SINGELI + si_scan_ne(p, xp, rp, BIT_N(ia)); #if USE_VALGRIND if (ia&63) rp[ia>>6] = vg_def_u64(rp[ia>>6]); #endif diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli deleted file mode 100644 index 11aef7d6..00000000 --- a/src/singeli/src/neq.singeli +++ /dev/null @@ -1,29 +0,0 @@ -include './base' -include './sse' -include './clmul' - -fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { - def V = [2]u64 - m := V**mark - def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total - p := clmul{a, m, i} - t := shr{[16]u8, p, 8} - s := p ^ carry - carry = s ^ t - s - } - xv := *V ~~ x - rv := *V ~~ r - e := words/2 - c := V**init - @for (rv, xv over e) { - rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})} - } - if (words & 1) { - storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} - } -} -fn clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = { - clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1)) -} -export{'clmul_scan_ne', clmul_scan_ne_bit} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index bf53b91f..0d7fc7fd 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -1,5 +1,6 @@ include './base' include './sse' +include './clmul' include './avx' include './avx2' include './mask' @@ -118,6 +119,40 @@ export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}} export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}} export{'si_scan_pluswrap_u32', scan_assoc_0{u32, +}} +# xor scan +fn scan_neq{}(p:u64, x:*u64, r:*u64, nw:u64) : void = { + @for (x, r over nw) { + def sc{v, k} = if (k==64) v else sc{v ^ (v<>63) # repeat sign bit + } +} +fn clmul_scan_ne_any{..._ & hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { + def V = [2]u64 + m := V**mark + def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total + p := clmul{a, m, i} + t := shr{[16]u8, p, 8} + s := p ^ carry + carry = s ^ t + s + } + xv := *V ~~ x + rv := *V ~~ r + e := words/2 + c := V**init + @for (rv, xv over e) { + rv = apply{zipLo, (@collect (j to 2) xor64{xv, j, c})} + } + if (words & 1) { + storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} + } +} +fn scan_neq{..._ & hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { + clmul_scan_ne_any{}(*void~~x, *void~~r, init, nw, -(u64~~1)) +} +export{'si_scan_ne', scan_neq{}} + # Boolean cumulative sum fn bcs{T}(x:*u64, r:*T, l:u64) : void = { def bitp_get{arr, n} = (load{arr, n>>6} >> (n&63)) & 1 From a040a14744c34d2b18724ee614aa3840489953e6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 10 Aug 2023 17:04:55 -0400 Subject: [PATCH 5/7] Unify power-of-two shift pattern for scans --- src/singeli/src/scan.singeli | 43 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 0d7fc7fd..566eabca 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -55,24 +55,26 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { scan_loop{T, init, x, r, len, scan, last} } +# Make prefix scan from op and shifter by applying the operation +# at increasing power-of-two shifts +def prefix_byshift{op, sh} = { + def pre{v:V, k} = if (k < width{V}) pre{op{v, sh{v,k}}, 2*k} else v + {v:T} => pre{v, if (isvec{T}) elwidth{T} else 1} +} + # Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈ def scan_idem = scan_scal fn scan_idem{T, op & hasarch{'SSE4.1'}}(x:*T, r:*T, len:u64, init:T) : void = { # Within each lane, scan using shifts by powers of 2. First k elements # when shifting by k don't need to change, so leave them alone. - def w = width{T} def shift{k,l} = merge{iota{k},iota{l-k}} - def c8 {k, a} = op{a, shuf{[4]u32, a, shift{k,4}}} - def c32{k, a} = (if (w<=8*k) op{a, sel8{a, shift{k,16}}}; else a) - # Prefix op on entire AVX register - def pre{a} = { - b:= c8{2, c8{1, c32{2, c32{1, a}}}} + def shb{v, k} = sel8{v, shift{k/8,16}} + def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}} + def shb{v, k & k==128 & hasarch{'AVX2'}} = { # After lanewise scan, broadcast end of lane 0 to entire lane 1 - if (not hasarch{'AVX2'}) b - else op{b, sel{[8]i32, spread{b}, make{[8]i32, 3*(3>63) # repeat sign bit } } From 0a30fb309d18ac77135886bdbf6dee77689ccb24 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 12 Aug 2023 09:59:42 -0400 Subject: [PATCH 6/7] SSE2 min- and max-scans --- src/singeli/src/scan.singeli | 14 +++++++++++--- src/singeli/src/sse.singeli | 8 ++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 566eabca..78da11ca 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -26,6 +26,7 @@ def spread{a:VT} = { # Set all elements with the last element of the input def toLast{n:VT & hasarch{'X86_64'} & w128{VT}} = { def l{v, w} = l{zipHi{v,v}, 2*w} + def l{v, w & hasarch{'SSSE3'}} = sel8{v, (16-w/8)+iota{16}%(w/8)} def l{v, w & w>=32} = shuf{[4]i32, v, 4**3} l{n, elwidth{VT}} } @@ -62,13 +63,20 @@ def prefix_byshift{op, sh} = { {v:T} => pre{v, if (isvec{T}) elwidth{T} else 1} } +def get_id{op,T} = (match (op) { {_==min}=>maxvalue; {_==max}=>minvalue }){T} + # Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈ def scan_idem = scan_scal -fn scan_idem{T, op & hasarch{'SSE4.1'}}(x:*T, r:*T, len:u64, init:T) : void = { +fn scan_idem{T, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { # Within each lane, scan using shifts by powers of 2. First k elements # when shifting by k don't need to change, so leave them alone. def shift{k,l} = merge{iota{k},iota{l-k}} - def shb{v, k} = sel8{v, shift{k/8,16}} + def shb{v:V, k} = { + def w=width{T}; def c = k/w + def id = make{V, merge{c**get_id{op,T}, (width{V}/w-c)**0}} + shl{[16]u8, v, k/8} | id + } + def shb{v, k & hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}} def shb{v, k & k==128 & hasarch{'AVX2'}} = { # After lanewise scan, broadcast end of lane 0 to entire lane 1 @@ -91,7 +99,7 @@ export{'si_scan_min_init_i32', scan_idem{i32, min}}; export{'si_scan_max_init_i3 export{'si_scan_min_init_f64', scan_idem{f64, min}}; export{'si_scan_max_init_f64', scan_idem{f64, max}} fn scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = { - scan_idem{T, op}(x, r, len, (if (same{op,min}) maxvalue else minvalue){T}) + scan_idem{T, op}(x, r, len, get_id{op, T}) } export{'si_scan_min_i8', scan_idem_id{i8 , min}}; export{'si_scan_max_i8', scan_idem_id{i8 , max}} export{'si_scan_min_i16', scan_idem_id{i16, min}}; export{'si_scan_max_i16', scan_idem_id{i16, max}} diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index b04b7a43..145ae2e5 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -13,10 +13,10 @@ def extract{x:T, i & w128i{T,64} & knum{i}} = emit{eltype{T}, '_mm_extract_epi64 def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} # arith -def min{a:T,b:T & T==[16]i8 } = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 } = emit{T, '_mm_max_epi8', a, b} -def min{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32} = emit{T, '_mm_max_epi32', a, b} -def min{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_max_epu16', a, b} -def min{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32} = emit{T, '_mm_max_epu32', a, b} +def min{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi8', a, b}; def max{a:T,b:T & T==[16]i8 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi8', a, b} +def min{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epi32', a, b}; def max{a:T,b:T & T==[ 4]i32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epi32', a, b} +def min{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu16', a, b}; def max{a:T,b:T & T==[ 8]u16 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu16', a, b} +def min{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_min_epu32', a, b}; def max{a:T,b:T & T==[ 4]u32 & hasarch{'SSE4.1'}} = emit{T, '_mm_max_epu32', a, b} def __le{a:T,b:T & w128u{T}} = a==min{a,b} def __ge{a:T,b:T & w128u{T}} = a==max{a,b} From c85ca66daee864ab3871f19d5896613e36139fab Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 26 Aug 2023 08:20:30 -0400 Subject: [PATCH 7/7] Unify Scan and Bins min-scan code --- src/singeli/src/bins.singeli | 36 +++++---------- src/singeli/src/scan.singeli | 58 +----------------------- src/singeli/src/scan_common.singeli | 69 +++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 82 deletions(-) create mode 100644 src/singeli/src/scan_common.singeli diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 450a044b..b4af2a70 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -4,6 +4,8 @@ if (hasarch{'AVX2'}) { include './sse' include './avx' include './avx2' +} else if (hasarch{'X86_64'}) { + include './sse' } include './mask' include 'util/tup' @@ -27,36 +29,18 @@ def shr16{v:V, n} = V~~(re_el{u16, v} >> n) # Forward or backwards in-place max-scan # Assumes a whole number of vectors and minimum 0 +include './scan_common' fn max_scan{T, up}(x:*T, len:u64) : void = { - def w = width{T} - if (hasarch{'AVX2'} and T!=u64) { + if (hasarch{'X86_64'}) { def op = max - # TODO unify with scan.singeli avx2_scan_idem - def rev{a} = if (up) a else (tuplen{a}-1)-reverse{a} - def maker{T, l} = make{T, rev{l}} - def sel8{v, t} = sel{[16]u8, v, maker{[32]i8, t}} - def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} - def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,rev{n}}} - def spread{a:VT} = { - def w = elwidth{VT} - def b = w/8 - if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}}; else a - } - def shift{k,l} = merge{iota{k},iota{l-k}} - def c8 {k, a} = op{a, shuf{[4]u32, a, shift{k,4}}} - def c32{k, a} = (if (w<=8*k) op{a, sel8{a, shift{k,16}}}; else a) - def pre{a} = { - b:= c8{2, c8{1, c32{2, c32{1, a}}}} - op{b, sel{[8]i32, spread{b}, maker{[8]i32, 3*(3 m) m = x; x = m } } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 78da11ca..773e8675 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -5,36 +5,13 @@ include './avx' include './avx2' include './mask' include './f64' +include './scan_common' # Initialized scan, generic implementation fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = { @for (x, r over len) r = m = op{m, x} } -def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}} -def sel8{v:V, t & w256{V} & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} - -def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} - -# Fill last 4 bytes with last element, in each lane -def spread{a:VT} = { - def w = elwidth{VT} - def b = w/8 - if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}}; else a -} - -# Set all elements with the last element of the input -def toLast{n:VT & hasarch{'X86_64'} & w128{VT}} = { - def l{v, w} = l{zipHi{v,v}, 2*w} - def l{v, w & hasarch{'SSSE3'}} = sel8{v, (16-w/8)+iota{16}%(w/8)} - def l{v, w & w>=32} = shuf{[4]i32, v, 4**3} - l{n, elwidth{VT}} -} -def toLast{n:VT & hasarch{'AVX2'} & w256{VT}} = { - if (elwidth{VT}<=32) sel{[8]i32, spread{n}, [8]i32**7} - else shuf{[4]u64, n, 4**3} -} - def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = { def step = arch_defvw/width{T} def V = [step]T @@ -56,41 +33,10 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = { scan_loop{T, init, x, r, len, scan, last} } -# Make prefix scan from op and shifter by applying the operation -# at increasing power-of-two shifts -def prefix_byshift{op, sh} = { - def pre{v:V, k} = if (k < width{V}) pre{op{v, sh{v,k}}, 2*k} else v - {v:T} => pre{v, if (isvec{T}) elwidth{T} else 1} -} - -def get_id{op,T} = (match (op) { {_==min}=>maxvalue; {_==max}=>minvalue }){T} - # Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈ def scan_idem = scan_scal fn scan_idem{T, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { - # Within each lane, scan using shifts by powers of 2. First k elements - # when shifting by k don't need to change, so leave them alone. - def shift{k,l} = merge{iota{k},iota{l-k}} - def shb{v:V, k} = { - def w=width{T}; def c = k/w - def id = make{V, merge{c**get_id{op,T}, (width{V}/w-c)**0}} - shl{[16]u8, v, k/8} | id - } - def shb{v, k & hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} - def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}} - def shb{v, k & k==128 & hasarch{'AVX2'}} = { - # After lanewise scan, broadcast end of lane 0 to entire lane 1 - sel{[8]i32, spread{v}, make{[8]i32, 3*(3=32} = shuf{[4]i32, v, 4**(up*3)} + l{n, elwidth{VT}} +} +def toLast{n:VT, up & hasarch{'AVX2'} & w256{VT}} = { + if (elwidth{VT}<=32) sel{[8]i32, spread{n,up}, [8]i32**(up*7)} + else shuf{[4]u64, n, 4**(up*3)} +} +def toLast{n:VT} = toLast{n, 1} + +# Make prefix scan from op and shifter by applying the operation +# at increasing power-of-two shifts +def prefix_byshift{op, sh} = { + def pre{v:V, k} = if (k < width{V}) pre{op{v, sh{v,k}}, 2*k} else v + {v:T} => pre{v, if (isvec{T}) elwidth{T} else 1} +} + +def get_id{op,T} = (match (op) { {_==min}=>maxvalue; {_==max}=>minvalue }){T} + +def make_scan_idem{T, op, up} = { + # Within each lane, scan using shifts by powers of 2. First k elements + # when shifting by k don't need to change, so leave them alone. + def shift{k,l} = rev{up, merge{iota{k},iota{l-k}}} + def shb{v:V, k} = { + def w=width{T}; def c = k/w + def merger{a,b} = if (up) merge{a,b} else merge{b,a} + def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}} + (if (up) shl else shr){[16]u8, v, k/8} | id + } + def shb{v, k & hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}} + def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}} + def shb{v, k & k==128 & hasarch{'AVX2'}} = { + # After lanewise scan, broadcast end of lane 0 to entire lane 1 + sel{[8]i32, spread{v,up}, make{[8]i32, rev{up,3*(3