From 92f40ddbe27ba92d7f09f37a38fb1848dda6572d Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 2 Feb 2025 22:38:21 +0200 Subject: [PATCH] mask.singeli def renames more bits of renames --- src/singeli/README.md | 26 +++++----- src/singeli/src/base.singeli | 1 - src/singeli/src/bins.singeli | 2 +- src/singeli/src/bits.singeli | 2 +- src/singeli/src/copy.singeli | 4 +- src/singeli/src/equal.singeli | 10 ++-- src/singeli/src/fold.singeli | 2 +- src/singeli/src/mask.singeli | 85 ++++++++++++++++--------------- src/singeli/src/replicate.singeli | 6 +-- src/singeli/src/scan.singeli | 6 +-- src/singeli/src/select.singeli | 6 +-- src/singeli/src/slash.singeli | 4 +- src/singeli/src/squeeze.singeli | 4 +- src/singeli/src/transpose.singeli | 4 +- 14 files changed, 81 insertions(+), 81 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index 4b77be2b..b38c7f97 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -237,7 +237,7 @@ For float conversions, the used rounding mode is unspecified. - `clz` - count leading zeroes - `cls` - count leading sign bits -- `copyLane` +- `copy_lane` - `mla` - multiply-add - `mls` - multiply-subtract - `ornot` - a|~b @@ -249,22 +249,22 @@ For float conversions, the used rounding mode is unspecified. - `trn1`, `trn2` - 2×2 transposes from pairs of elements across the two arguments ## mask.singeli -- `maskOf{TU, n}` - get a mask of type `TU` whose first `n` elements are all `1`s, and the remaining are `0`s. `0≤n≤vcount{TU}` -- `maskNone` - a mask generator that matches all items -- `maskAfter{n}` - a mask generator that patches the first `n` items -- `loadBatch{p:*E, n, V}` - load the `n`-th batch of `vcount{V}` elements from `*E`; if `E` isn't `eltype{V}`, the result is sign- or zero-extended. -- `storeBatch{p:*E, n, x:V, M}` - store equivalent of the `loadBatch`; if `E` isn't `eltype{V}`, `x` is narrowed via `narrow{}` +- `mask_of_first{TU, n}` - get a mask of type `TU` whose first `n` elements are all `1`s, and the remaining are `0`s. `0≤n≤vcount{TU}` +- `mask_none` - a mask generator that matches all items +- `mask_first{n}` - a mask generator that enables the first `n` items +- `load_widen{p:*E, n, [k]W}` - load the `n`-th batch of `k` elements from `p`; if `W` isn't `E`, the result is sign- or zero-extended. +- `store_narrow{p:*E, n, x:[k]N, M}` - store equivalent of the `load_widen`; if `E` isn't `N`, `x` is narrowed via `narrow{}` -To test whether a mask object `M` is `maskNone` or `maskAfter`, `M{0}` can be used - `maskNone{0}` is `0`, but `maskAfter{n}{0}` is `1`. +To test whether a mask object `M` is `mask_none` or `mask_first`, `M{0}` can be used - `mask_none{0}` is `0`, but `mask_first{n}{0}` is `1`. ### Loops -- `@maskedLoop{bulk}` - loop that generates its body twice, once with a `maskNone` mask, and once with a `maskAfter{n}` one to handle the tail. +- `@maskedLoop{bulk}` - loop that generates its body twice, once with a `mask_none` mask, and once with a `mask_first{n}` one to handle the tail. - `@muLoop{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end. Unrolling is handled by passing a tuple of indices to process as the index variable (the tail generated bodies get a tuple of the one index) @@ -275,19 +275,19 @@ To test whether a mask object `M` is `maskNone` or `maskAfter`, `M{0}` can be us Tuples can be used in the iterated variable list for various things: ``` p:*T - regular pointer -tup{VT,p:*T} - loadBatch/storeBatch vector data +tup{VT,p:*T} - load_widen/store_narrow vector data tup{'b',p:P} - load_bits (bits.singeli) tup{'b',VT,p:P} - load_expand_bits (bits.singeli) -tup{'g',VT,p:*T} - gives a generator, used as g{} to loadBatch and g{newValue} to storeBatch +tup{'g',VT,p:*T} - gives a generator, used as g{} to load_widen and g{newValue} to store_narrow tup{'g',p:*T} - the above, but without load support -'m' - get the mask object - either maskNone or some maskAfter{n} +'m' - get the mask object - either mask_none or some mask_first{n} ``` Stores via those will implicitly do a masked store when required. Loads will load past the end, so `M in 'm'` must be used to mask off elements if they're used for something other than the above stores. -For `muLoop`, `M in 'm'` still gives a single generator, but it's only ever `maskAfter{n}` when there's only one index. +For `muLoop`, `M in 'm'` still gives a single generator, but it's only ever `mask_first{n}` when there's only one index. Example usage for a loop that adds `u16` and bit boolean elements to a `u32` accumulator, early-exiting on overflow: ``` diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 7fda4d40..1395f572 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -251,7 +251,6 @@ def min = __min def max = __max def adds = __adds def subs = __subs -def sqrt = __sqrt diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 8d9e1a3d..ba14b955 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -220,7 +220,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = # Fill with minimum value at the beginning def pre = if (search) load{w} else (if (up) minvalue else maxvalue){T} wg := *V~~(w-gap) - wv0:= blend_hom{load{wg}, V**pre, maskOf{V,gap}} + wv0:= blend_hom{load{wg}, V**pre, mask_of_first{V,gap}} # For multiple lanes, interleave like transpose def maxstep = lb{maxwn} def lstep = lb{svl} diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index d1fef197..1181d1f0 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -72,7 +72,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u next{} } while (dstC < dstE) { - storeBatch{dstC, 0, get{}, maskAfter{dstE - dstC}} + store_narrow{dstC, 0, get{}, mask_first{dstE - dstC}} next{} } }} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 97855549..9d241911 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -25,7 +25,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { def ur = tern{hasarch{'AARCH64'}, 4, 1} if (X==R and R!=u1) { - if (hasarch{'X86_64'} and l<=bulk) storeBatch{rp, 0, loadBatch{xp, 0, RV}, maskAfter{l}} + if (hasarch{'X86_64'} and l<=bulk) store_narrow{rp, 0, load_widen{xp, 0, RV}, mask_first{l}} else emit{void, 'memcpy', r, x, l*(width{X}/8)} } else if (R==u64) { # show{'R==u64', X, R} @@ -49,7 +49,7 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = { def bulk2 = bulk*unr xi:ux = 0 @for_nz (i to cdiv{l,bulk2}) { - store_bits{bulk2, rp, i, hom_to_int{each{{i} => op{loadBatch{xp, xi+i, XV}}, iota{unr}}}} + store_bits{bulk2, rp, i, hom_to_int{each{{i} => op{load_widen{xp, xi+i, XV}}, iota{unr}}}} xi+= unr } } else if (width{X}<=width{R}) { diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 757a1018..17ad86d9 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -20,7 +20,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { if (W==u1) { if (X==u1) { # bitarr ≡ bitarr def BT = [vw/8]u8 - @maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyneBit{w,x,M}) return{0} + @maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyne_bit{w,x,M}) return{0} } else if (X==f64) { # bitarr ≡ f64arr def TF = [vw/64]f64 def TU = [vw/64]u64 @@ -35,7 +35,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { blend_hom{f0, f1, cw} }) cx:= load{*TF ~~ x, i} - if (anynePositive{wu, cx, M}) return{0} + if (anyne_positive{wu, cx, M}) return{0} } } else { # bitarr ≡ i8/i16/i32arr def T = [bulk]X @@ -58,9 +58,9 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { def R = [bulk]X @maskedLoopPositive{bulk}(M in 'm' over i to l) { - cw:= loadBatch{*W~~w, i, R} - cx:= loadBatch{*X~~x, i, R} - if (anynePositive{cw,cx,M}) return{0} + cw:= load_widen{*W~~w, i, R} + cx:= load_widen{*X~~x, i, R} + if (anyne_positive{cw,cx,M}) return{0} } } 1 diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 39bb508c..5075baa4 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -66,7 +66,7 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { def V = [bulk]T xv:= *V ~~ x e:= len / bulk - i:= load{xv, e} & (V~~maskOf{V, len % bulk}) + i:= load{xv, e} & (V~~mask_of_first{V, len % bulk}) r:= reduce_pairwise{op, 2, xv, e, i} if (hasarch{'AARCH64'}) vfold{op, r} else extract{mix{op, r}, 0} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index ca5a269d..23fbddbf 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -1,41 +1,42 @@ -local def maskInit1{w} = { +local def bit_mask_init{w} = { apply{merge, each{{x} => { merge{(w/8-1)**255, (1<>3)^31 + 64*(n&7))} -mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n if width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} +mask256_1:*u8 = bit_mask_init{256}; def mask_of_first_bits{T,n if width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))} +mask128_1:*u8 = bit_mask_init{128}; def mask_of_first_bits{T,n if width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))} mask256:*i64 = merge{4 ** -1, 4 ** 0} -local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} +local def mask_of_impl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))} # get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T} -def maskOf{T,n if w256{T}} = maskOfImpl{T, n, 256} -def maskOf{T,n if w128{T}} = maskOfImpl{T, n, 128} -def maskOf{T,n if w64{T}} = maskOfImpl{T, n, 64} +def mask_of_first{T,n if w256{T}} = mask_of_impl{T, n, 256} +def mask_of_first{T,n if w128{T}} = mask_of_impl{T, n, 128} +def mask_of_first{T,n if w64{T}} = mask_of_impl{T, n, 64} def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~all_hom{x==y} def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = any_hom{M{x!=y}} def anyne{x:T, y:T, M if M{0}==0 and any_int{x}} = x!=y def anyne{x:T, y:T, M if M{0}==1 and any_int{x}} = M{x^y} != 0 -def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} +def anyne_bit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} -def anynePositive{x:T, y:T, M if M{0}==0} = anyne{x, y, M} -def anynePositive{x:T, y:T, M if M{0}==1 and isvec{T}} = { +def anyne_positive{x:V=[_]_, y:V, M if M{0}==0} = anyne{x, y, M} +def anyne_positive{x:V=[_]_, y:V, M if M{0}==1} = { def {n,m} = hom_to_int_ext{x==y} def E = tern{type{m}==u64, u64, u32} (promote{E,~m} << (width{E}-M{'count'}*n)) != 0 } -def maskNone{x} = x -def maskNone{x, 'all bits zeroes'} = and_bit_none{x, x} -def maskAfter{n} = { - def mask{x:X, 'all bits zeroes'} = and_bit_none{x, X~~maskOfBit{X,n}} - def mask{X, 'to sign bits'} = maskOf{X,n} - def mask{X, 'to homogeneous bits'} = maskOf{X,n} +def mask_none{x} = x +def mask_none{x, 'all bits zeroes'} = and_bit_none{x, x} + +def mask_first{n} = { + def mask{x:X, 'all bits zeroes'} = and_bit_none{x, X~~mask_of_first_bits{X,n}} + def mask{X, 'to sign bits'} = mask_of_first{X,n} + def mask{X, 'to homogeneous bits'} = mask_of_first{X,n} def mask{'count'} = n def mask{{x}} = tup{mask{x}} - def mask{x:X if isvec{X}} = x & (X~~maskOf{X,n}) + def mask{x:X if isvec{X}} = x & (X~~mask_of_first{X,n}) def mask{x:X if any_int{x}} = x & ((1<M, '!'} - def vproc{{T,p:*E}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} - def vproc{{'b', p:*E}} = tptr{{i} => load_bits{bulk, hCast{u64,p}, i}, '!'} - def vproc{{'b',T,p:*E}} = tptr{{i} => load_expand_bits{T, hCast{u64,p}, i}, '!'} - def vproc{{'g', p:*E}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} + def vproc{{T,p:*E}} = tptr{{i} => load_widen{p, i, T}, {i,x} => store_narrow{p, i, x, M}} + def vproc{{'b', p:*E}} = tptr{{i} => load_bits{bulk, cast_h{u64,p}, i}, '!'} + def vproc{{'b',T,p:*E}} = tptr{{i} => load_expand_bits{T, cast_h{u64,p}, i}, '!'} + def vproc{{'g', p:*E}} = tptr{{i} => ({x} => store_narrow{p, i, x, M}), '!'} def vproc{{'g',T,p:*E}} = tptr{{i} => { - def dv{} = loadBatch{p, i, T} - def dv{x} = storeBatch{p, i, x, M} + def dv{} = load_widen{p, i, T} + def dv{x} = store_narrow{p, i, x, M} }, '!'} iter{i, each{vproc, vars0}} @@ -92,10 +93,10 @@ def maskedLoop{bulk, i0}{vars,begin==0,end,iter} = { l:u64 = end m:u64 = l / bulk - @for (i from i0 to m) mlExec{i, iter, vars, bulk, maskNone} + @for (i from i0 to m) ml_exec{i, iter, vars, bulk, mask_none} left:= l & (bulk-1) - if (left!=0) mlExec{m, iter, vars, bulk, maskAfter{left}} + if (left!=0) ml_exec{m, iter, vars, bulk, mask_first{left}} } def maskedLoop{bulk} = maskedLoop{bulk,0} @@ -104,10 +105,10 @@ def maskedLoopPositive{bulk}{vars,begin==0,end:L,iter} = { assert{end > 0} i:L = 0 while(i < (end-1)/bulk) { - mlExec{i, iter, vars, bulk, maskNone} + ml_exec{i, iter, vars, bulk, mask_none} ++i } - mlExec{i, iter, vars, bulk, maskAfter{end - i*bulk}} + ml_exec{i, iter, vars, bulk, mask_first{end - i*bulk}} } @@ -125,30 +126,30 @@ def muLoop{bulk, unr, fromunr}{vars,begin==0,end,iter} = { m:u64 = l / bulk if (unr==1) { - @for (i from 0 to m) mlExec{tup{i}, iter, vars, bulk, maskNone} + @for (i from 0 to m) ml_exec{tup{i}, iter, vars, bulk, mask_none} left:= l & (bulk-1) - if (left!=0) mlExec{tup{m}, iter, vars, bulk, maskAfter{left}} + if (left!=0) ml_exec{tup{m}, iter, vars, bulk, mask_first{left}} } else { if (m > 0) { i:u64 = 0 if (unr <= m) { while ((i+unr) <= m) { def is = each{{j}=>i+j, iota{unr}} - mlExec{each{{j}=>i+j, iota{unr}}, iter, vars, bulk, maskNone} + ml_exec{each{{j}=>i+j, iota{unr}}, iter, vars, bulk, mask_none} i+= unr } fromunr{} } if (unr==2) { - if (i!=m) mlExec{tup{i}, iter, vars, bulk, maskNone} + if (i!=m) ml_exec{tup{i}, iter, vars, bulk, mask_none} } else { - @for(j from i to m) mlExec{tup{j}, iter, vars, bulk, maskNone} + @for(j from i to m) ml_exec{tup{j}, iter, vars, bulk, mask_none} } } left:= l & (bulk-1) - if (left!=0) mlExec{tup{m}, iter, vars, bulk, maskAfter{left}} + if (left!=0) ml_exec{tup{m}, iter, vars, bulk, mask_first{left}} } } def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0} diff --git a/src/singeli/src/replicate.singeli b/src/singeli/src/replicate.singeli index b1505217..1ce850eb 100644 --- a/src/singeli/src/replicate.singeli +++ b/src/singeli/src/replicate.singeli @@ -142,7 +142,7 @@ def rep_const_shuffle{wv, onreps, xv:*V=[step]T, rv:*V, n:(u64)} = { # onreps{in }} setlabel{end} q := nr & (step-1) - if (q!=0) store_blended_hom{rv+e, maskOf{V, q}, s} + if (q!=0) store_blended_hom{rv+e, mask_of_first{V, q}, s} } } @@ -217,7 +217,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { setlabel{end} q := (re+step) - r - if (q!=0) store_blended_hom{*V~~r, maskOf{V, q}, s} + if (q!=0) store_blended_hom{*V~~r, mask_of_first{V, q}, s} } @@ -373,7 +373,7 @@ def get_boolvec_writer{V, r, nw} = { def flush{} = { setlabel{l_flush} q := nw & (vwords-1) - if (q != 0) store_blended_hom{rv, V~~maskOf{re_el{u64,V}, q}, last_res} + if (q != 0) store_blended_hom{rv, V~~mask_of_first{re_el{u64,V}, q}, last_res} setlabel{done} } tup{output, check_done, flush} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index b1b7d435..1f595cb3 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -21,7 +21,7 @@ def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = { e:= len/step @for (xv, rv over e) rv = scan{xv,p} q:= len & (step-1) - if (q!=0) store_blended_hom{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}} + if (q!=0) store_blended_hom{rv+e, mask_of_first{V, q}, scan_last{load{xv,e}, p}} } def get_scan_last{op, pre} = { def last{v, p} = op{pre{v}, p} @@ -58,7 +58,7 @@ fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { } @for (xv, rv over _ from ek*k to e) rv = scan{xv,p} q:= len & (step-1) - if (q!=0) store_blended_hom{rv+e, maskOf{V, q}, last{load{xv,e}, p}} + if (q!=0) store_blended_hom{rv+e, mask_of_first{V, q}, last{load{xv,e}, p}} } export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}} @@ -197,7 +197,7 @@ fn bcs{T if hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = { if (jv+vl <= l) { store{p, j, v} } else { - if (jv < l) store_blended_hom{rv+j, maskOf{V, l - jv}, v} + if (jv < l) store_blended_hom{rv+j, mask_of_first{V, l - jv}, v} return{} } } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index a8144922..54769496 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -33,7 +33,7 @@ def masked_multistore{r0, vs, M, end} = { # returns bumped-forwards r r:= r0 def left = if (M{0}) { left:ux = M{'count'} } else 0 def lastMaskedStore = make_opt_branch{M{0}, tup{one_type{vs}}, {c} => { - storeBatch{r, 0, c, maskAfter{left}} + store_narrow{r, 0, c, mask_first{left}} end{} }} @@ -42,7 +42,7 @@ def masked_multistore{r0, vs, M, end} = { # returns bumped-forwards r if (i+1 == length{vs} or left>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack - store_blended_hom{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + store_blended_hom{*R~~r0, mask_of_first{R, end-buf}, load{*R~~buf}} } else { @for (r0, buf over u64~~(end-buf)) r0 = buf } @@ -216,7 +216,7 @@ fn slash{c, T if hasarch{'AVX2'} and T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, s } def tab = if (tw==32) itab else i64tab def step{r, w} = { - s:= loadBatch{*u8~~(tab+w), 0, V} + s:= load_widen{*u8~~(tab+w), 0, V} store{*V~~r, 0, from_ind{s}} } @for_special_buffered{r,8} (w in *u8~~wp over sum) { diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index db71b89f..9a9f6673 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -20,13 +20,13 @@ def any_nonC32{M, x:T=[_]_ if hasarch{'X86_64'}} = { def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} any_top{M{T~~ne}} } -def any_nonC32{(maskNone), x:[k](u64), y:[k](u64)} = { +def any_nonC32{(mask_none), x:[k](u64), y:[k](u64)} = { def T32 = [k*2]u32 def hi = shuf_ind{T32~~x, T32~~y, match(k) { {2} => tup{1,3,5,7} {4} => tup{1,3,9,11,5,7,13,15} # all odd indices, in the order that vshufps can handle }} - anyne{hi, T32**cast_i{u32, cbqn_c32Tag{}<<16}, maskNone} + anyne{hi, T32**cast_i{u32, cbqn_c32Tag{}<<16}, mask_none} } fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 7907ee7e..b646a5e1 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -239,7 +239,7 @@ def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} o xb := xv + 2*nv x0 := load{xb} x1 := V**0; if (n&(l/2) != 0) x1 = load{xb, 1} - mask := maskOf{V, n%l} + mask := mask_of_first{V, n%l} each{store_blended_hom{., mask, .}, tup{rv0+nv,rv1+nv}, uz{x0,x1}} } } @@ -258,7 +258,7 @@ fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = { r0 := get_r{0} rb := rv + 2*nv nr := 2*n; m := nr%l - mask := maskOf{V, m} + mask := mask_of_first{V, m} if (nr&l == 0) { store_blended_hom{rb, mask, r0} } else {