diff --git a/src/builtins/fold.c b/src/builtins/fold.c index a02d953d..ac7100f6 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -11,6 +11,34 @@ // •math.Sum: +´ with faster and more precise SIMD code for i32, f64 +// Insert with rank (˝˘ or ˝⎉k), or fold on flat array +// SHOULD optimize dyadic insert with rank +// Length 1, ⊣⊢: implemented as ⊏˘ +// SHOULD reshape identity for fast ˝˘ on empty rows +// Boolean operand, cell size 1: +// +: popcount +// Rows length ≤64: extract rows, popcount each +// TRIED scan-based version, faster for width 3 only +// COULD have bit-twiddling versions of +˝˘ on very short rows +// >64: popcount, boundary corrections (clang auto-vectorizes) +// ∨∧≠=, rows <64: extract from scan or windowed op +// Dedicated auto-vectorizing code for sizes 2, 4 +// Extract is generic with multiplies or BMI2 (SHOULD add SIMD) +// COULD use CLMUL for faster windowed ≠ +// ∨∧: +// SHOULD handle ∨∧ synonyms for boolean fold on rows +// Multiples of 8 <256: comparison function, recurse if needed +// Rows 6424: read word, mask, last bit of popcount +// Rows l<64: word-based scan, correct with SWAR +// ≥64: xor words (auto-vectorizes), correct boundary, popcount +// COULD implement boolean -˝˘ with xor, +˝˘, offset +// Arithmetic on rank 2, short rows: transpose then insert, blocked +// SHOULD extend transpose-insert code to any frame and cell rank +// SHOULD have dedicated +⌊⌈ insert with rank + #include "../core.h" #include "../builtins.h" #include "../utils/mut.h" diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 0ebd813e..f083314c 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -1,3 +1,39 @@ +// Scan (`) +// Empty 𝕩, and length 1 if no 𝕨: return 𝕩 +// Generic operand: +// Constant: copy +// ⊢ identity, ⊣ reshape 𝕨 or first cell +// Boolean operand, rank 1: +// + AVX2 expansion (SHOULD have better generic, add SSE, NEON) +// ∨⌈ ∧×⌊ search+copy, then memset (COULD vectorize search) +// ≠ SWAR shifts, CLMUL, VPCLMUL (SHOULD add SSE, NEON) +// < SWAR +// =≤≥>- in terms of ≠<∨∧+ with adjustments +// Arithmetic operand, rank 1: +// ⌈⌊ Scalar, SSE, AVX in log(vector width) steps +// + Overflow-checked scalar or AVX2 +// Ad-hoc boolean-valued handling for ≠∨ +// SHOULD extend rank 1 special cases to cell bound 1 +// Higher-rank arithmetic, non-tiny cells: apply operand cell-wise +// SHOULD have dedicated high-rank scan optimizations + +// Scan with rank (`˘ or `⎉k) +// SHOULD optimize dyadic scan with rank +// Empty 𝕩, length 1, ⊢: return 𝕩 +// Boolean operand, cell size 1: +// ≠∨∧⊣ and synonyms, rows <64: SWAR, AVX2 (SHOULD add SSE, NEON) +// Power of two row size: autovectorized +// COULD have dedicated SIMD for CPU widths, little improvement +// ⊣ SWAR for <64, select for ≥ +// ∨⌈ ∧×⌊ SWAR with addition for small rows, search for large +// Rows 64≤l<160: SWAR specialized for ≤1 boundary +// Large rows: word-at-a-time search +// ≠ power-of-two shifts for <64, rank-1 scans and boundary corrections if ≥ +// SHOULD have a better intermediate-size (< ~256) SIMD method +// + scan in blocks, correct with mask, ⌊`, subtract +// = as ≠`⌾¬, - as (2×⊣`)-+` +// SHOULD optimize non-boolean scan with rank + #include "../core.h" #include "../utils/mut.h" #include "../builtins.h" @@ -5,6 +41,9 @@ #define F64_MIN -INFINITY #define F64_MAX INFINITY +#define CASE_N_AND case n_and: case n_mul: case n_floor +#define CASE_N_OR case n_or: case n_ceil + #if !USE_VALGRIND static u64 vg_rand(u64 x) { return x; } #endif @@ -215,16 +254,16 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (!(xr==1 && xe<=el_f64)) goto base; if (xe==el_bit) switch (rtid) { default: goto base; - case n_add: return scan_add_bool(x, ia); // + - case n_or: case n_ceil: return scan_or(x, ia); // ∨⌈ - case n_and: case n_mul: case n_floor: return scan_and(x, ia); // ∧×⌊ - case n_ne: return scan_ne(x, 0, ia); // ≠ - case n_eq: return scan_eq(x, ia); // = - case n_lt: return scan_lt(x, 0, ia); // < - case n_le: return bit_negate(scan_lt(bit_negate(x), 0, ia)); // ≤ - case n_gt: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_and(x, ia); // > - case n_ge: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_or (x, ia); // ≥ - case n_sub: return C2(sub, m_f64(2 * (*bitarr_ptr(x) & 1)), scan_add_bool(x, ia)); // - + case n_add: return scan_add_bool(x, ia); // + + CASE_N_OR: return scan_or(x, ia); // ∨⌈ + CASE_N_AND: return scan_and(x, ia); // ∧×⌊ + case n_ne: return scan_ne(x, 0, ia); // ≠ + case n_eq: return scan_eq(x, ia); // = + case n_lt: return scan_lt(x, 0, ia); // < + case n_le: return bit_negate(scan_lt(bit_negate(x), 0, ia)); // ≤ + case n_gt: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_and(x, ia); // > + case n_ge: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_or (x, ia); // ≥ + case n_sub: return C2(sub, m_f64(2 * (*bitarr_ptr(x) & 1)), scan_add_bool(x, ia)); // - } if (rtid==n_add) return scan_plus(0, x, xe, ia); // + if (rtid==n_floor) return scan_min_num(x, xe, ia); // ⌊ @@ -349,13 +388,13 @@ B scan_rows_bit(u8 rtid, B x, usz m) { #if SINGELI switch (rtid) { default: return bi_N; case n_eq: return bit_negate(scan_rows_bit(n_ne, bit_negate(x), m)); - case n_and: case n_or: case n_ne: case n_ltack: { + CASE_N_AND: CASE_N_OR: case n_ne: case n_ltack: { usz ia = IA(x); u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); switch (rtid) { default:UD; - case n_and: si_scan_rows_and (xp, rp, ia, m); break; - case n_or: si_scan_rows_or (xp, rp, ia, m); break; + CASE_N_AND: si_scan_rows_and (xp, rp, ia, m); break; + CASE_N_OR: si_scan_rows_or (xp, rp, ia, m); break; case n_ne: si_scan_rows_ne (xp, rp, ia, m); break; case n_ltack: si_scan_rows_ltack(xp, rp, ia, m); break; } diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 94f88970..8aada08d 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -140,6 +140,7 @@ def fold_rows_bit_lt64{ dm:= cast_i{usz, popc{el}} # minimum output bits per word dm-= promote{usz, l&(l-1) == 0} # for divisors of 64, e0 effectively overflows; subtract 1 to correct def loop{...par} = { + et:= e0 << clz{e0} @for (xo in xp over nw) { x:= xx{xo} def {s, mod_rb} = if (length{select{par,0}} == 3) { @@ -153,7 +154,7 @@ def fold_rows_bit_lt64{ nb:= dm + promote{usz, e>=e0} # = popc{e} # the multiply-mask sequence (last "mask" is the shift by (64 - nb)) def extract = match { - { {q}, {}} => (s & e) * (q << clz{e}) + { {q}, {}} => (et &~ (et - (s & e))) * q {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q } rb:= extract{...par} >> (64 - nb)