From 0232f915828c9545e7676728ba7fb3d8980fbaa1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 24 Apr 2023 12:37:13 -0400 Subject: [PATCH 01/14] AVX2 bit table Member-of --- src/builtins/search.c | 19 ++++++--- src/singeli/src/search.singeli | 75 +++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 94ff754a..0775eb9c 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -23,6 +23,11 @@ #include "../utils/hash.h" #include "../utils/talloc.h" +#if SINGELI_SIMD + #define SINGELI_FILE search + #include "../utils/includeSingeli.h" +#endif + #define C2i(F, W, X) C2(F, m_i32(W), X) extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); @@ -131,11 +136,6 @@ static B reduceI32Width(B r, usz count) { return count<=I8_MAX? taga(cpyI8Arr(r)) : count<=I16_MAX? taga(cpyI16Arr(r)) : r; } -#if SINGELI_SIMD - #define SINGELI_FILE search - #include "../utils/includeSingeli.h" -#endif - static NOINLINE usz indexOfOne(B l, B e) { void* lp = tyany_ptr(l); usz wia = IA(l); @@ -293,6 +293,15 @@ B memberOf_c2(B t, B w, B x) { if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { B r; + #if SINGELI_AVX2 + if (we==el_i8 && xe==el_i8) { + TALLOC(u8, tab, 256); + u64* rp; r = m_bitarrc(&rp, w); + avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); + TFREE(tab); + return r; + } + #endif TABLE(x, w, i8, 0, 1) return taga(cpyBitArr(r)); } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 03c9c683..48063fdc 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -74,4 +74,77 @@ export{'simd_search_u16', searchOne{u64, u16}} export{'simd_search_u32', searchOne{u64, u32}} export{'simd_search_f64', searchOne{f64, f64}} export{'simd_search_normalizable', searchNormalizable{}} -export{'simd_copy_ordered', copyOrdered{}} \ No newline at end of file +export{'simd_copy_ordered', copyOrdered{}} + + +# In-register bit table +if (hasarch{'AVX2'}) { +def TI = i8 +def VI = [32]TI + +def bittab_selector{loadtab} = { + {t0, t1}:= loadtab{} + low:= VI**7 + hi4:= VI**(-(1<<4)) + b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + def selector{x} = { + top := hi4 + VI~~(([8]u32~~(x&~low))>>3) + byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top} + mask:= sel{[16]i8, b, x & low} + homMask{(mask & byte) == mask} + } + def reload{} = { tup{t0,t1} = loadtab{} } + tup{selector, reload} +} + +fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { + assert{nw > 0} + vtab:= *VI~~tab; btab:= *i8~~tab + z:= VI**0 + @unroll (vtab over 8) vtab = z + def readbytes{} = { + def side{i} = { + def m = @collect (vtab over _ from i to i+4) homMask{vtab} + VI~~make{[8]u32, merge{m,m}} + } + each{side, 4*iota{2}} + } + + # Fill table + u:u8 = 0 # Unseen unique bytes (goes to 255 on first step) + w:= *u8~~w0 + iw:u64 = 32; if (nw<=48) iw=nw + @for (w over iw) { + u-= 1 + u8~~load{btab, w} # Subtract 1 if new + store{btab, w, -1} + } + def {bitsel, reload_tab} = bittab_selector{readbytes} + wv:= *VI~~w0 + while (iw < nw) { + ii:= iw+32 + m:= bitsel{load{wv, iw/32}} + if (ii > nw) { ii=nw; m|=(~u32~~0)<<(nw%32) } + m = ~m + if (m != 0) { # Need to add entries + do { + wj:= load{w, iw+emit{u64, 'CTZ', m}} + u-= 1 + u8~~load{btab, wj} # Subtract 1 if new + store{btab, wj, -1} + m&= m-1 # Clear last bit + } while (m != 0) + + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + return{} + } + reload_tab{} + } + iw = ii + } + + # Read result + @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} +} + +export{'avx2_member_u8', avx2_member_u8} +} # hasarch{'AVX2'} From e45231154c6cca47934ce9816971f37cb2284116 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 24 Apr 2023 16:29:30 -0400 Subject: [PATCH 02/14] Reverse lookup initialization for 1-byte Member-of --- src/singeli/src/search.singeli | 84 ++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 48063fdc..235e11d3 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -97,52 +97,68 @@ def bittab_selector{loadtab} = { tup{selector, reload} } -fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { - assert{nw > 0} - vtab:= *VI~~tab; btab:= *i8~~tab - z:= VI**0 - @unroll (vtab over 8) vtab = z - def readbytes{} = { - def side{i} = { - def m = @collect (vtab over _ from i to i+4) homMask{vtab} - VI~~make{[8]u32, merge{m,m}} - } - each{side, 4*iota{2}} +def readbytes{vtab}{} = { + def side{i} = { + def m = @collect (vtab over _ from i to i+4) homMask{vtab} + VI~~make{[8]u32, merge{m,m}} } + each{side, 4*iota{2}} +} - # Fill table - u:u8 = 0 # Unseen unique bytes (goes to 255 on first step) - w:= *u8~~w0 - iw:u64 = 32; if (nw<=48) iw=nw - @for (w over iw) { - u-= 1 + u8~~load{btab, w} # Subtract 1 if new - store{btab, w, -1} +# Fill table with t (0 or -1) at all bytes in x0 +# Stop early if the sum u reaches 0, indicating all bytes in the table +# are equal: by the time it's checked at least one has been set to t, +# so they're all t +fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { + btab:= *i8~~tab + def settab{x} = { + u+= u8~~(t - load{btab, x}) # u tracks the total of btab + store{btab, x, t} } - def {bitsel, reload_tab} = bittab_selector{readbytes} - wv:= *VI~~w0 - while (iw < nw) { - ii:= iw+32 - m:= bitsel{load{wv, iw/32}} - if (ii > nw) { ii=nw; m|=(~u32~~0)<<(nw%32) } - m = ~m + x:= *u8~~x0 + i:u64 = 32; if (n<=48) i=n + @for (x over i) settab{x} + if (u == 0) return{u} # Won't ever trigger (m != 0)! + def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}} + xv:= *VI~~x0 + while (i < n) { + ii:= i+32 + m:= bitsel{load{xv, i/32}} + if (ii > n) { ii=n; m|=(~u32~~0)<<(n%32) } + m^= u32~~cast_i{i32, t} if (m != 0) { # Need to add entries do { - wj:= load{w, iw+emit{u64, 'CTZ', m}} - u-= 1 + u8~~load{btab, wj} # Subtract 1 if new - store{btab, wj, -1} + settab{load{x, i+emit{u64, 'CTZ', m}}} m&= m-1 # Clear last bit } while (m != 0) - - if (u == 0) { # All found! - @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} - return{} - } + if (u == 0) return{u} # Table is all one value reload_tab{} } - iw = ii + i = ii + } + u +} + +fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { + assert{nw > 0} + vtab:= *VI~~tab + + # Initialize + rev:u1 = nx < nw/4 # Reverse lookup + z:= VI**(-cast_i{i8,rev}) + @unroll (vtab over 8) vtab = z + u:u8 = 0 # Sum of table, either 0 or 256 + if (rev) u = fill_bittab(x0, nx, tab, u, 0) + + # Fill table + u = fill_bittab(w0, nw, tab, u, -1) + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + return{} } # Read result + def {bitsel, _} = bittab_selector{readbytes{vtab}} @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} } From 017e2433fb0974c1b229c79242088715b87931ed Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 24 Apr 2023 20:03:30 -0400 Subject: [PATCH 03/14] Fast paths for all empty-argument search cases --- src/builtins/search.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 0775eb9c..619691da 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -201,7 +201,10 @@ B indexOf_c2(B t, B w, B x) { } else { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); - if (wia == 0) { B r=taga(arr_shCopy(allZeroes(xia), x)); decG(w); decG(x); return r; } + if (wia==0 || xia==0) { + B r=taga(arr_shCopy(allZeroes(xia), x)); + decG(w); decG(x); return r; + } if (elNum(we) && elNum(xe)) { tyEls: if (we==el_bit) { @@ -271,7 +274,10 @@ B memberOf_c2(B t, B w, B x) { many: { u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); - if (xia == 0) { r=taga(arr_shCopy(allZeroes(wia), w)); decG(w); goto dec_x; } + if (wia==0 || xia==0) { + r=taga(arr_shCopy(allZeroes(wia), w)); + decG(w); goto dec_x; + } if (elNum(we) && elNum(xe)) { tyEls: #define WEQ(V) C2(eq, incG(w), V) @@ -330,7 +336,7 @@ B count_c2(B t, B w, B x) { x = t.n; } - if (!isArr(x) || IA(x)<=1) return C2(indexOf, w, x); + if (!isArr(x) || IA(x)<=1 || IA(w)==0) return C2(indexOf, w, x); u8 we = TI(w,elType); usz wia = IA(w); u8 xe = TI(x,elType); usz xia = IA(x); i32* rp; B r = m_i32arrc(&rp, x); From 2fafe11d7c0b8b1d4323a54d38815644d1128e8b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 25 Apr 2023 17:28:31 -0400 Subject: [PATCH 04/14] Fix fill_bittab tail masking for t=0 and simplify --- src/singeli/src/search.singeli | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 235e11d3..01adfe32 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -122,19 +122,18 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}} xv:= *VI~~x0 while (i < n) { - ii:= i+32 - m:= bitsel{load{xv, i/32}} - if (ii > n) { ii=n; m|=(~u32~~0)<<(n%32) } - m^= u32~~cast_i{i32, t} + m:= u32~~cast_i{i32, t} ^ bitsel{load{xv, i/32}} + i0:= i + i+= 32 + if (i > n) m&= (~u32~~0)>>((-n)%32) if (m != 0) { # Need to add entries do { - settab{load{x, i+emit{u64, 'CTZ', m}}} + settab{load{x, i0 + cast_i{u64,ctz{m}}}} m&= m-1 # Clear last bit } while (m != 0) if (u == 0) return{u} # Table is all one value reload_tab{} } - i = ii } u } From 9c7c6aabc526b37bee400cb80971b2d7054bc4ff Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 25 Apr 2023 21:45:27 -0400 Subject: [PATCH 05/14] AVX2 Mark Firsts implementation --- src/builtins/selfsearch.c | 14 +++++++++- src/singeli/src/search.singeli | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 45bac03a..25887f7d 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -213,6 +213,8 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i n) m&= (~u32~~0)>>((-n)%32) + if (m == 0) { # No new values + store{r, iw, m} + } else { # Add values to the table and filter m + rv:= m + do { + new:= settab{load{x, i0 + cast_i{u64,ctz{m}}}} + m1:= m-1; m&= m1 # Clear last bit + rv&= m1 | u32~~cast_i{i32,new} # Clear if not new + } while (m != 0) + store{r, iw, rv} + if (u == 0) { # All bytes seen + @for (r over _ from iw+1 to cdiv{n,32}) r = 0 + return{} + } + reload_tab{} + } + } +} + +export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8} export{'avx2_member_u8', avx2_member_u8} } # hasarch{'AVX2'} From ba43dae90854ec5621bef896dc4f2308db66e74a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 26 Apr 2023 20:16:45 -0400 Subject: [PATCH 06/14] Heuristic filtering when AVX2 Mark Firsts sees multiple new values --- src/singeli/src/search.singeli | 40 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 7bc14226..c38159b7 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -10,6 +10,8 @@ if (hasarch{'AVX2'}) { } include './mask' +def ctzi{x} = promote{u64, ctz{x}} # Count trailing zeros, as index + def findFirst{C, M, F, ...v1} = { def exit = makelabel{} def args = undef{M{...each{{c}=>tupsel{0,c}, v1}}} @@ -30,7 +32,7 @@ def search{E, x, n:u64, OP} = { def VT = [bulk]E def end = makeBranch{ tup{u64, ty_u{VT}}, - {i,c} => return{i*bulk + promote{u64, ctz{homMask{c}}}} + {i,c} => return{i*bulk + ctzi{homMask{c}}} } muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { @@ -122,13 +124,13 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}} xv:= *VI~~x0 while (i < n) { - m:= u32~~cast_i{i32, t} ^ bitsel{load{xv, i/32}} + m:= u32~~promote{i32, t} ^ bitsel{load{xv, i/32}} i0:= i i+= 32 if (i > n) m&= (~u32~~0)>>((-n)%32) if (m != 0) { # Need to add entries do { - settab{load{x, i0 + cast_i{u64,ctz{m}}}} + settab{load{x, i0 + ctzi{m}}} m&= m-1 # Clear last bit } while (m != 0) if (u == 0) return{u} # Table is all one value @@ -144,7 +146,7 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi # Initialize rev:u1 = nx < nw/4 # Reverse lookup - z:= VI**(-cast_i{i8,rev}) + z:= VI**(-promote{i8,rev}) @unroll (vtab over 8) vtab = z u:u8 = 0 # Sum of table, either 0 or 256 if (rev) u = fill_bittab(x0, nx, tab, u, 0) @@ -174,30 +176,44 @@ fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { store{btab, x, 0} v } + def settab{T, x} = T~~promote{ty_s{T}, settab{x}} + def settab1{x} = { store{btab, x, 0}; ++u } # Do first 64 values with a scalar loop x:= *u8~~x0 - i:u64 = 64; if (n<=64) i=n + i:u64 = 32; if (n<=48) i=n rw:u64 = 0 - @for (x over j to i) rw|= u64~~cast_i{i64,settab{x}} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) if (m == 0) { # No new values store{r, iw, m} } else { # Add values to the table and filter m rv:= m - do { - new:= settab{load{x, i0 + cast_i{u64,ctz{m}}}} - m1:= m-1; m&= m1 # Clear last bit - rv&= m1 | u32~~cast_i{i32,new} # Clear if not new - } while (m != 0) + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + rv&= m1 | new # Clear if not new + } + } store{r, iw, rv} if (u == 0) { # All bytes seen @for (r over _ from iw+1 to cdiv{n,32}) r = 0 From 9658104f3ed93ac33b2358e5a64ab1f8248788ad Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 26 Apr 2023 21:33:36 -0400 Subject: [PATCH 07/14] Unify AVX2 bit-table code between Member-of and Mark Firsts --- src/singeli/src/search.singeli | 141 ++++++++++++++++----------------- 1 file changed, 67 insertions(+), 74 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index c38159b7..1077c150 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -111,35 +111,89 @@ def readbytes{vtab}{} = { # Stop early if the sum u reaches 0, indicating all bytes in the table # are equal: by the time it's checked at least one has been set to t, # so they're all t -fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { +# If r0 is a pointer, set it to the unique mask of x0 +def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, r0} = { btab:= *i8~~tab - def settab{x} = { - u+= u8~~(t - load{btab, x}) # u tracks the total of btab + def settab_sub{x, v} = { + u+= u8~~(t - v) # u tracks the total of btab store{btab, x, t} + v } + def settab1{x} = settab_sub{x, -1 - t} # Known new + def settab{x} = settab_sub{x, load{btab, x}} # General case + def settab{T, x} = T~~promote{ty_s{T}, settab{x}} + + def rbit = ~match{r0, 0} + def storebit{i, v:T} = if (rbit) store{*T~~r0, i, v} + + # Do first few values with a scalar loop + # Avoids the cost of ever loading the table into vectors for n<=48 x:= *u8~~x0 i:u64 = 32; if (n<=48) i=n - @for (x over i) settab{x} - if (u == 0) return{u} # Won't ever trigger (m != 0)! + {rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit + if (rbit) rw = 0 + @for (x over j to i) { + new:= settab{u64, x} + if (rbit) rw|= new & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - if (m != 0) { # Need to add entries - do { - settab{load{x, i0 + ctzi{m}}} - m&= m-1 # Clear last bit - } while (m != 0) - if (u == 0) return{u} # Table is all one value + # Any new values? + if (m == 0) { + storebit{iw, m} + } else { + # Add values to the table and filter m + if (rbit) rv = m + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + if (rbit) rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + if (rbit) rv&= m1 | new # Clear if not new + } + } + storebit{iw, rv} + if (u == 0) { # All bytes seen + if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0 + goto{done} + } reload_tab{} } } + setlabel{done} u } +fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { + init:= VI**(-1) + @unroll (t in *VI~~tab over 8) t = init + u:u8 = 0 + do_bittab{x0, n, tab, u, 0, r0} +} + +fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { + do_bittab{x0, n, tab, u, t, 0} +} + fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { assert{nw > 0} vtab:= *VI~~tab @@ -163,67 +217,6 @@ fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} } -fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { - vtab:= *VI~~tab - init:= VI**(-1) - @unroll (vtab over 8) vtab = init - - u:u8 = 0 # Sum of table, either 0 or 256 - btab:= *i8~~tab - def settab{x} = { - v:= load{btab, x} - u-= u8~~v # u tracks the total of btab - store{btab, x, 0} - v - } - def settab{T, x} = T~~promote{ty_s{T}, settab{x}} - def settab1{x} = { store{btab, x, 0}; ++u } - - # Do first 64 values with a scalar loop - x:= *u8~~x0 - i:u64 = 32; if (n<=48) i=n - rw:u64 = 0 - @for (x over j to i) rw|= settab{u64, x} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - if (m == 0) { # No new values - store{r, iw, m} - } else { # Add values to the table and filter m - rv:= m - xi:= load{x, i0 + ctzi{m}} - settab1{xi} - if ((m&(m-1)) != 0) { # More bits than one - # Filter out values equal to the previous, or first new - def pind = (iota{32}&15) - 1 - prev:= make{VI, each{bind{max,0}, pind}} - e:= ~homMask{v == VI**TI~~xi} - e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} - rv&= e | -m # Don't remove first bit - m&= e - while (m != 0) { - new:= settab{u32, load{x, i0 + ctzi{m}}} - m1:= m-1; m&= m1 # Clear low bit - rv&= m1 | new # Clear if not new - } - } - store{r, iw, rv} - if (u == 0) { # All bytes seen - @for (r over _ from iw+1 to cdiv{n,32}) r = 0 - return{} - } - reload_tab{} - } - } -} - export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8} export{'avx2_member_u8', avx2_member_u8} } # hasarch{'AVX2'} From 1700d8474057af4d83b8398b579777baa4a4b9ca Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 26 Apr 2023 21:58:59 -0400 Subject: [PATCH 08/14] AVX2 1-byte Deduplicate implementation --- src/builtins/selfsearch.c | 14 +++++++++++++- src/singeli/src/search.singeli | 32 ++++++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 25887f7d..2d5b3412 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -214,6 +214,7 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i Date: Thu, 27 Apr 2023 21:08:32 -0400 Subject: [PATCH 09/14] Make AVX2 deduplicate branchless --- src/builtins/selfsearch.c | 2 +- src/singeli/src/search.singeli | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 2d5b3412..ba09e73d 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -557,7 +557,7 @@ B find_c1(B t, B x) { } #if SINGELI_AVX2 if (elWidth(xe)==1 && RNK(x)==1 && !FL_HAS(x, fl_asc|fl_dsc)) { - TALLOC(u8, tab, 512); u8* res = tab+256; + TALLOC(u8, tab, 513); u8* res = tab+256; usz ria = avx2_deduplicate_u8(tyany_ptr(x), n, res, tab); B r; i8* rp = m_tyarrv(&r, 1, ria, el2t(xe)); memcpy(rp, res, ria); diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 2fe426c9..d6ce44ff 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -123,7 +123,7 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { btab:= *i8~~tab def settab_sub{x, v} = { - if (rval and v!=0) store{*u8~~r0, u, x} + if (rval) store{*u8~~r0, u, x} u+= u8~~(t - v) # u tracks the total of btab store{btab, x, t} v From 844b537593487829a6eacef73f7e444341060f8d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 27 Apr 2023 21:52:23 -0400 Subject: [PATCH 10/14] Singeli 1-byte table implementations for generic architecture --- build/src/build.bqn | 2 +- src/builtins/search.c | 4 +- src/builtins/selfsearch.c | 12 +-- src/singeli/src/search.singeli | 178 ++++++++++++++++++++------------- 4 files changed, 115 insertions(+), 81 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index 4e5c0d29..40233bcd 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -609,7 +609,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/arithd.c"‿"dyarith", "xa."‿"src/builtins/cmp.c"‿"cmp", "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", - "xa."‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/search.c"‿"search", "xa."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", diff --git a/src/builtins/search.c b/src/builtins/search.c index 619691da..2e903ece 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -299,11 +299,11 @@ B memberOf_c2(B t, B w, B x) { if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { B r; - #if SINGELI_AVX2 + #if SINGELI if (we==el_i8 && xe==el_i8) { TALLOC(u8, tab, 256); u64* rp; r = m_bitarrc(&rp, w); - avx2_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); + simd_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); TFREE(tab); return r; } diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index ba09e73d..6394b67e 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -213,8 +213,8 @@ static NOINLINE void memset64(u64* p, u64 v, usz l) { for (usz i=0; i 0) { + k:= rem; if (k>64) k=64 + rv:u64 = 0 + @for (x over j to k) rv|= u64~~promote{i64,load{t,x}} & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) - # Any new values? - if (m == 0) { - storebit{iw, m} - } else { - # Add values to the table and filter m - if (rbit) rv = m - xi:= load{x, i0 + ctzi{m}} - settab1{xi} - if ((m&(m-1)) != 0) { # More bits than one - # Filter out values equal to the previous, or first new - def pind = (iota{32}&15) - 1 - prev:= make{VI, each{bind{max,0}, pind}} - e:= ~homMask{v == VI**TI~~xi} - e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} - if (rbit) rv&= e | -m # Don't remove first bit - m&= e - while (m != 0) { - new:= settab{u32, load{x, i0 + ctzi{m}}} - m1:= m-1; m&= m1 # Clear low bit - if (rbit) rv&= m1 | new # Clear if not new - } + if (not hasarch{'AVX2'}) { + rem:= n + @for (i to cdiv{n,64}) { + k:= rem; if (k>64) k=64 + rw:u64 = 0 + @for (x over j to k) { + new:= settab{u64, x} + if (rbit) rw|= new & ((u64~~1)< n) m&= (~u32~~0)>>((-n)%32) + # Any new values? + if (m == 0) { + storebit{iw, m} + } else { + # Add values to the table and filter m + if (rbit) rv = m + xi:= load{x, i0 + ctzi{m}} + settab1{xi} + if ((m&(m-1)) != 0) { # More bits than one + # Filter out values equal to the previous, or first new + def pind = (iota{32}&15) - 1 + prev:= make{VI, each{bind{max,0}, pind}} + e:= ~homMask{v == VI**TI~~xi} + e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} + if (rbit) rv&= e | -m # Don't remove first bit + m&= e + while (m != 0) { + new:= settab{u32, load{x, i0 + ctzi{m}}} + m1:= m-1; m&= m1 # Clear low bit + if (rbit) rv&= m1 | new # Clear if not new + } + } + storebit{iw, rv} + if (u == 0) { # All bytes seen + if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0 + goto{done} + } + reload_tab{} + } + } + setlabel{done} } - setlabel{done} u } -fn avx2_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { - init:= VI**(-1) - @unroll (t in *VI~~tab over 8) t = init +fn simd_mark_firsts_u8(x0:*void, n:u64, r0:*void, tab:*void) : void = { + bittab_init{tab, -1} u:u8 = 0 do_bittab{x0, n, tab, u, 0, 'mask', r0} } -fn avx2_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = { +fn simd_deduplicate_u8(x0:*void, n:u64, r0:*void, tab:*void) : u64 = { assert{n != 0} - init:= VI**(-1) - @unroll (t in *VI~~tab over 8) t = init + bittab_init{tab, -1} u:u8 = 0 do_bittab{x0, n, tab, u, 0, 'unique', r0} 1 + promote{u64, u-1} # 0 to 256 @@ -209,30 +247,26 @@ fn fill_bittab(x0:*void, n:u64, tab:*void, u:u8, t:i8) : u8 = { do_bittab{x0, n, tab, u, t, 'none', 0} } -fn avx2_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { +fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : void = { assert{nw > 0} - vtab:= *VI~~tab # Initialize rev:u1 = nx < nw/4 # Reverse lookup - z:= VI**(-promote{i8,rev}) - @unroll (vtab over 8) vtab = z + bittab_init{tab, -promote{i8,rev}} u:u8 = 0 # Sum of table, either 0 or 256 if (rev) u = fill_bittab(x0, nx, tab, u, 0) # Fill table u = fill_bittab(w0, nw, tab, u, -1) - if (u == 0) { # All found! - @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} - return{} - } # Read result - def {bitsel, _} = bittab_selector{readbytes{vtab}} - @for (x in *VI~~x0, r in *u32~~r0 over cdiv{nx,32}) r = bitsel{x} + if (u == 0) { # All found! + @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} + } else { + bittab_lookup{x0, nx, r0, tab} + } } -export{'avx2_mark_firsts_u8', avx2_mark_firsts_u8} -export{'avx2_deduplicate_u8', avx2_deduplicate_u8} -export{'avx2_member_u8', avx2_member_u8} -} # hasarch{'AVX2'} +export{'simd_mark_firsts_u8', simd_mark_firsts_u8} +export{'simd_deduplicate_u8', simd_deduplicate_u8} +export{'simd_member_u8', simd_member_u8} From 464dd27a37b42f00043b7c1e4622d5ca2533801a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 29 Apr 2023 10:50:51 -0400 Subject: [PATCH 11/14] SIMD table intialization for Index-of --- src/builtins/search.c | 12 +++++++++ src/singeli/src/search.singeli | 47 ++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 2e903ece..9708325d 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -226,6 +226,18 @@ B indexOf_c2(B t, B w, B x) { } if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { + #if SINGELI + if (wia>256 && we==el_i8 && xe==el_i8) { + TALLOC(u8, tab, 256*(1+sizeof(usz))); usz* ind = (usz*)(tab+256); + void* fp = tyany_ptr(x); + simd_index_tab_u8(tyany_ptr(w), wia, fp, xia, tab, ind); + decG(w); + i32* rp; B r = m_i32arrc(&rp, x); + for (usz i=0; i64) k=64 rw:u64 = 0 @for (x over j to k) { - new:= settab{u64, x} + new:= settab{u64, x, i*64+j} # Index usually unused if (rbit) rw|= new & ((u64~~1)< 0} - # Initialize rev:u1 = nx < nw/4 # Reverse lookup bittab_init{tab, -promote{i8,rev}} u:u8 = 0 # Sum of table, either 0 or 256 if (rev) u = fill_bittab(x0, nx, tab, u, 0) - # Fill table u = fill_bittab(w0, nw, tab, u, -1) - # Read result if (u == 0) { # All found! @for (r in *u64~~r0 over cdiv{nx,64}) r = maxvalue{u64} } else { @@ -267,6 +269,17 @@ fn simd_member_u8(w0:*void, nw:u64, x0:*void, nx:u64, r0:*void, tab:*void) : voi } } +fn simd_index_tab_u8{I}(w0:*void, nw:u64, x0:*void, nx:u64, tab:*void, i0:*void) : void = { + rev:u1 = nx < nw/4 + bittab_init{tab, -promote{i8,rev}} + ind:= *I~~i0 + @for (ind over 256) ind = cast_i{I, nw} + u:u8 = 0 + if (rev) u = fill_bittab(x0, nx, tab, u, 0) + do_bittab{w0, nw, tab, u, -1, 'index', ind} +} + export{'simd_mark_firsts_u8', simd_mark_firsts_u8} export{'simd_deduplicate_u8', simd_deduplicate_u8} export{'simd_member_u8', simd_member_u8} +export{'simd_index_tab_u8', simd_index_tab_u8{usz}} From 53fb8db06f74a7e47c6f68317d4f908de6d6117c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 30 Apr 2023 21:58:48 -0400 Subject: [PATCH 12/14] SSSE3 support in bit-table code --- src/singeli/src/search.singeli | 45 ++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index dada2532..642d69a1 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -3,6 +3,8 @@ if (hasarch{'AVX2'}) { include './sse' include './avx' include './avx2' +} else if (hasarch{'SSSE3'}) { + include './sse' } else if (hasarch{'X86_64'}) { include './sse2' } else if (hasarch{'AARCH64'}) { @@ -80,24 +82,27 @@ export{'simd_copy_ordered', copyOrdered{}} # In-register bit table +def arch_vec{T} = [arch_defvw/width{T}]T def TI = i8 # Table values -def VI = [32]TI +def VI = arch_vec{TI} +def simd_bittab = hasarch{'SSSE3'} def bittab_init{tab, z} = { @for (t in *TI~~tab over 256) t = z } -def bittab_init{tab, z & hasarch{'AVX2'}} = { +def bittab_init{tab, z & simd_bittab} = { init:= VI**z @unroll (t in *VI~~tab over 256/vcount{VI}) t = init } def bittab_selector{loadtab} = { + def nv = vcount{VI} {t0, t1}:= loadtab{} low:= VI**7 hi4:= VI**(-(1<<4)) - b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + b := VI~~make{[nv]u8, 1 << (iota{nv} & 7)} def selector{x} = { - top := hi4 + VI~~(([8]u32~~(x&~low))>>3) + top := hi4 + VI~~((arch_vec{u32}~~(x&~low))>>3) byte:= sel{[16]i8, t0, hi4^top} | sel{[16]i8, t1, top} mask:= sel{[16]i8, b, x & low} homMask{(mask & byte) == mask} @@ -107,11 +112,13 @@ def bittab_selector{loadtab} = { } def readbytes{vtab}{} = { + def k = vcount{VI}; def l = 128/vcount{VI} def side{i} = { - def m = @collect (vtab over _ from i to i+4) homMask{vtab} - VI~~make{[8]u32, merge{m,m}} + def U = arch_vec{ty_u{k}} + def m = @collect (vtab over _ from i to i+l) homMask{vtab} + VI~~make{U, if (vcount{U}>l) merge{m,m} else m} } - each{side, 4*iota{2}} + each{side, l*iota{2}} } # Look up bits from table @@ -128,9 +135,10 @@ def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = { x+=k; rem-=k; ++r } } -def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & hasarch{'AVX2'}} = { +def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & simd_bittab} = { def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}} - @for (x in *VI~~x0, r in *u32~~r0 over cdiv{n,32}) r = bitsel{x} + def k = vcount{VI} + @for (x in *VI~~x0, r in *ty_u{k}~~r0 over cdiv{n,k}) r = bitsel{x} } # Fill table with t (0 or -1) at all bytes in x0 @@ -162,7 +170,7 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { def settab{T, x, i} = T~~promote{ty_s{T}, settab{x, i}} x:= *u8~~x0 - if (not hasarch{'AVX2'}) { + if (not simd_bittab) { rem:= n @for (i to cdiv{n,64}) { k:= rem; if (k>64) k=64 @@ -178,7 +186,8 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { # Do first few values with a scalar loop # Avoids the cost of ever loading the table into vectors for n<=48 i:u64 = 32; if (n<=48) i=n - {rw,rv} := undef{tup{u64,u32}} # Bit results, used if rbit + def k = vcount{VI}; def uk = ty_u{k}; def ik = ty_s{k} + {rw,rv} := undef{tup{u64,uk}} # Bit results, used if rbit if (rbit) rw = 0 @for (x over j to i) { new:= settab{u64, x, j} @@ -191,12 +200,12 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { def {bitsel, reload_tab} = bittab_selector{readbytes{*VI~~tab}} xv:= *VI~~x0 while (i < n) { - i0:= i; iw:= i/32 + i0:= i; iw:= i/k v:= load{xv, iw} m:= bitsel{v} # Mask of possibly-new values - if (not match{t,0}) m^= u32~~promote{i32, t} - i+= 32 - if (i > n) m&= (~u32~~0)>>((-n)%32) + if (not match{t,0}) m^= uk~~promote{ik, t} + i+= k + if (i > n) m&= (~uk~~0)>>((-n)%k) # Any new values? if (m == 0) { storebit{iw, m} @@ -208,7 +217,7 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { settab1{xi, im} if ((m&(m-1)) != 0) { # More bits than one # Filter out values equal to the previous, or first new - def pind = (iota{32}&15) - 1 + def pind = (iota{k}&15) - 1 prev:= make{VI, each{bind{max,0}, pind}} e:= ~homMask{v == VI**TI~~xi} e&= base{2,pind<0} | ~homMask{v == sel{[16]i8, v, prev}} @@ -216,14 +225,14 @@ def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = { m&= e while (m != 0) { im:= i0 + ctzi{m} - new:= settab{u32, load{x, im}, im} + new:= settab{uk, load{x, im}, im} m1:= m-1; m&= m1 # Clear low bit if (rbit) rv&= m1 | new # Clear if not new } } storebit{iw, rv} if (u == 0) { # All bytes seen - if (rbit) @for (r in *u32~~r0 over _ from iw+1 to cdiv{n,32}) r = 0 + if (rbit) @for (r in *uk~~r0 over _ from iw+1 to cdiv{n,k}) r = 0 goto{done} } reload_tab{} From 25883b410c09386f89c044d66c08fcd037804b4f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 11 May 2023 21:11:43 -0400 Subject: [PATCH 13/14] =?UTF-8?q?Retune=20boundary=20for=20small-?= =?UTF-8?q?=F0=9D=95=A9=20linear=20Member=20Of?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index 9708325d..b1bf967d 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -301,7 +301,7 @@ B memberOf_c2(B t, B w, B x) { decG(w); goto dec_x; } - if (xia<=(xe==el_i16?8:16) && wia>16) { + if (xia<=(xe==el_i8?1:xe==el_i16?4:16) && wia>16) { SGetU(x); r = WEQ(GetU(x,0)); for (usz i=1; i Date: Fri, 12 May 2023 13:02:36 +0300 Subject: [PATCH 14/14] =?UTF-8?q?free=20arguments=20of=20SIMD=20?= =?UTF-8?q?=F0=9D=95=A8=E2=88=8A=F0=9D=95=A9=20&=20=E2=88=8A=F0=9D=95=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/search.c | 5 ++--- src/builtins/selfsearch.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/builtins/search.c b/src/builtins/search.c index b1bf967d..0b2d8345 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -310,14 +310,13 @@ B memberOf_c2(B t, B w, B x) { #undef WEQ if (xia+wia>20 && we<=el_i16 && xe<=el_i16) { - B r; #if SINGELI if (we==el_i8 && xe==el_i8) { TALLOC(u8, tab, 256); u64* rp; r = m_bitarrc(&rp, w); simd_member_u8(tyany_ptr(x), xia, tyany_ptr(w), wia, rp, tab); - TFREE(tab); - return r; + TFREE(tab); decG(w); + goto dec_x; } #endif TABLE(x, w, i8, 0, 1) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 6394b67e..d8b92a56 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -259,7 +259,7 @@ B memberOf_c1(B t, B x) { TALLOC(u8, tab, 256); u64* rp; B r = m_bitarrv(&rp, n); simd_mark_firsts_u8(xv, n, rp, tab); - TFREE(tab); + TFREE(tab); decG(x); return r; #else LOOKUP(8);