diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 5ef00382..a8d4397b 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -169,15 +169,16 @@ B scan_bit_sum(B x, u64* xp, u64 ia, u64 xs) { // consumes x return mut_fv(r0); } B r; + void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); #if SINGELI - i32* rp; r=m_i32arrv(&rp, ia); - avx2_bcs32(xp, rp, ia); + #define SUM(W,T) avx2_bcs##W(xp, rp, ia); #else - void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); - #define CASE(T) case el_##T: { T c=0; for (usz i=0; i> make{[8]u32, 4*tail{1, iota{8}}} s:= sel8{[32]u8~~b, ii32>>3 + bit{2}} p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes d:= sel{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p} d+= sel8{d, bit{2}*(1+bit{3}>>2)-1} - d+= sel8{d, bit{3}-1} - #d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100} - j:= 4*i + d + sel8{d, bit{3}-1} + } + def step{x:u32, i, store1} = { + d:= sumlanes{x} + if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100} + j:= (w/8)*i def out{v, k} = each{out, widen{v}, 2*k+iota{2}} - def out{v0:[8]i32, k} = { - v := [8]u32~~v0 + c - if (tail{1,k}) c = sel{[8]u32, v, make{[8]i32, broadcast{8, 7}}} + def out{v0:[vl]T, k} = { + v := V~~v0 + c + # Update carry at the lane boundary + if (w!=32 or tail{1,k}) { + c = sel{[8]u32, spread{v}, make{[8]i32, broadcast{8, 7}}} + } store1{rv, j+k, v} } out{[32]i8~~d, 0} @@ -120,15 +130,17 @@ avx2_bcs32(x:*u64, r:*i32, l:u64) : void = { if (e*32 != l) { def st{p, j, v} = { - j8 := 8*j - if (j8+8 <= l) { + jv := vl*j + if (jv+vl <= l) { store{p, j, v} } else { - if (j8 < l) maskstoreF{rv, maskOf{[8]i32, l - j8}, j, v} + if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v} return{} } } step{load{xv, e}, e, st} } } -'avx2_bcs32' = avx2_bcs32 +'avx2_bcs8' = avx2_bcs{i8} +'avx2_bcs16' = avx2_bcs{i16} +'avx2_bcs32' = avx2_bcs{i32}