mask.singeli loop renames
This commit is contained in:
parent
92f40ddbe2
commit
49dd6e394a
@ -264,12 +264,12 @@ To test whether a mask object `M` is `mask_none` or `mask_first`, `M{0}` can be
|
||||
|
||||
### Loops
|
||||
|
||||
- `@maskedLoop{bulk}` - loop that generates its body twice, once with a `mask_none` mask, and once with a `mask_first{n}` one to handle the tail.
|
||||
- `@for_masked{bulk}` - loop that generates its body twice, once with a `mask_none` mask, and once with a `mask_first{n}` one to handle the tail.
|
||||
|
||||
- `@muLoop{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end.
|
||||
- `@for_mu{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end.
|
||||
Unrolling is handled by passing a tuple of indices to process as the index variable (the tail generated bodies get a tuple of the one index)
|
||||
|
||||
- `@muLoop{bulk, unr, fromunr}` - `fromunr` is additionally ran after exiting the unrolled loop (such that you can have separate accumulators for the unrolled bit, and convert them to single-vector acccumulators for the tail).
|
||||
- `@for_mu{bulk, unr, fromunr}` - `fromunr` is additionally ran after exiting the unrolled loop (such that you can have separate accumulators for the unrolled bit, and convert them to single-vector acccumulators for the tail).
|
||||
|
||||
|
||||
Tuples can be used in the iterated variable list for various things:
|
||||
@ -287,15 +287,15 @@ Stores via those will implicitly do a masked store when required.
|
||||
|
||||
Loads will load past the end, so `M in 'm'` must be used to mask off elements if they're used for something other than the above stores.
|
||||
|
||||
For `muLoop`, `M in 'm'` still gives a single generator, but it's only ever `mask_first{n}` when there's only one index.
|
||||
For `for_mu`, `M in 'm'` still gives a single generator, but it's only ever `mask_first{n}` when there's only one index.
|
||||
|
||||
Example usage for a loop that adds `u16` and bit boolean elements to a `u32` accumulator, early-exiting on overflow:
|
||||
```
|
||||
fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
|
||||
fn acc_u32_u16_bit_u1(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @for_masked
|
||||
def bulk = 8
|
||||
def VT = [8]u32
|
||||
|
||||
@maskedLoop{bulk}(
|
||||
@for_masked{bulk}(
|
||||
r in tup{VT, r},
|
||||
x in tup{VT, x},
|
||||
b in tup{'b', VT, bits},
|
||||
@ -309,12 +309,13 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
|
||||
}
|
||||
1
|
||||
}
|
||||
export {'acc_u32_u16_bit_u1', acc_u32_u16_bit_u1}
|
||||
|
||||
fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x unrolled
|
||||
fn acc_u32_u16_bit_u2(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @for_mu, 2x unrolled
|
||||
def bulk = 8
|
||||
def VT = [8]u32
|
||||
|
||||
@muLoop{bulk,2}(
|
||||
@for_mu{bulk,2}(
|
||||
r in tup{'g', VT, r},
|
||||
x in tup{VT, x},
|
||||
b in tup{'b', VT, bits},
|
||||
@ -328,6 +329,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un
|
||||
}
|
||||
1
|
||||
}
|
||||
export {'acc_u32_u16_bit_u2', acc_u32_u16_bit_u2}
|
||||
```
|
||||
|
||||
<!--
|
||||
|
||||
@ -11,7 +11,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
|
||||
|
||||
e0v:= VT**e0
|
||||
e1v:= VT**e1
|
||||
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
|
||||
@for_masked{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
|
||||
}
|
||||
|
||||
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
||||
@ -31,7 +31,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
|
||||
def bulk = arch_defvw / width{E}
|
||||
def VT = [bulk]E
|
||||
def one = VT**cast_i{E, one0}
|
||||
@maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
|
||||
@for_masked{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
|
||||
}
|
||||
}
|
||||
|
||||
@ -44,7 +44,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
|
||||
def bulk = arch_defvw / 8
|
||||
def V = [bulk]u8
|
||||
bitalign{tup{2,8,csz}, 8, {s, align} => {
|
||||
@maskedLoop{bulk}(dst in tup{V,*u8~~dst} over cam) {
|
||||
@for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) {
|
||||
dst = align{load{*V~~src}}
|
||||
ptr_add{u8, src, bulk*s/8}
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ include './bitops'
|
||||
def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = {
|
||||
def U = ty_u{V}
|
||||
|
||||
@maskedLoop{bulk}(sr in tup{'g',*T~~rp} over i to l) {
|
||||
@for_masked{bulk}(sr in tup{'g',*T~~rp} over i to l) {
|
||||
x:= loadFn{U, i}
|
||||
sr{V~~(x & U ~~ V**1)}
|
||||
}
|
||||
@ -31,11 +31,11 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
||||
# show{'R==u64', X, R}
|
||||
assert{X==u8 or X==u16 or X==u32}
|
||||
# TODO could maybe read 256 bits and use unpack to write >256
|
||||
@maskedLoop{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)}
|
||||
@for_masked{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)}
|
||||
} else if (X==u1 and R==u1) {
|
||||
# show{'u1u1', X, R}
|
||||
def V64 = [vw/64]u64
|
||||
@maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
|
||||
@for_masked{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
|
||||
} else if (X==u1) {
|
||||
# show{'X==u1', X, R}
|
||||
copyFromBits{[bulk]R, load_expand_bits{., xp, .}, r, l}
|
||||
@ -54,10 +54,10 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
||||
}
|
||||
} else if (width{X}<=width{R}) {
|
||||
# show{'w{X}<=w{R}', X, R}
|
||||
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x}
|
||||
@for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x}
|
||||
} else {
|
||||
# show{'w{X}>w{R}', X, R}
|
||||
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over l) sr{x}
|
||||
@for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over l) sr{x}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -130,13 +130,13 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
|
||||
if (R==u1) {
|
||||
def bulk = vw/64
|
||||
def VT = [bulk]u64
|
||||
@maskedLoop{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over cdiv{len, 64}) r{F{cw,cx}}
|
||||
@for_masked{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over cdiv{len, 64}) r{F{cw,cx}}
|
||||
} else if (same{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6
|
||||
def bulk = vw / width{W}
|
||||
def TU = ty_u{R}
|
||||
def TV = [bulk]TU
|
||||
|
||||
@muLoop{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over len) {
|
||||
@for_mu{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over len) {
|
||||
sr{each{&, cw, cx}}
|
||||
}
|
||||
} else {
|
||||
@ -147,7 +147,7 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
|
||||
def run = runner{same{overflow, 0}, R, F}
|
||||
|
||||
def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty
|
||||
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
|
||||
@for_mu{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
|
||||
sr{arithProcess{F, run, overflow, M, is, cw, cx, TY}}
|
||||
}
|
||||
}
|
||||
@ -182,7 +182,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
|
||||
cw:= ty_sc{W, TY}**getW{w}
|
||||
|
||||
def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl
|
||||
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
|
||||
@for_mu{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
|
||||
def cws = length{is}**cw
|
||||
sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}}
|
||||
}
|
||||
@ -199,7 +199,7 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||
def T8 = [bulk]u8
|
||||
def T64 = [bulk/8]u64
|
||||
maskFull:= T8~~T64**maskU64
|
||||
@maskedLoop{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
|
||||
@for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
|
||||
}
|
||||
|
||||
export{'simd_andBytes', andBytes{arch_defvw}}
|
||||
|
||||
@ -20,13 +20,13 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
if (W==u1) {
|
||||
if (X==u1) { # bitarr ≡ bitarr
|
||||
def BT = [vw/8]u8
|
||||
@maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyne_bit{w,x,M}) return{0}
|
||||
@for_masked{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyne_bit{w,x,M}) return{0}
|
||||
} else if (X==f64) { # bitarr ≡ f64arr
|
||||
def TF = [vw/64]f64
|
||||
def TU = [vw/64]u64
|
||||
f0:= TF**0.0
|
||||
f1:= TF**1.0
|
||||
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
|
||||
@for_masked_pos{bulk}(M in 'm' over i to l) {
|
||||
wu:= (if (hasarch{'AVX2'}) {
|
||||
cw:= load_bits_lo{bulk, *u64~~w, i}
|
||||
blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
|
||||
@ -46,7 +46,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
|
||||
# TODO compare with doing the comparison in vector registers
|
||||
badBits:= T ** ~(X~~1)
|
||||
@maskedLoop{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
|
||||
@for_masked{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
|
||||
if (~and_bit_none{M{x}, badBits}) return{0}
|
||||
if (anyne{promote{u64,mask{sh{x}}}, promote{u64,cw}, M}) return{0}
|
||||
}
|
||||
@ -57,7 +57,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
|
||||
|
||||
def R = [bulk]X
|
||||
|
||||
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
|
||||
@for_masked_pos{bulk}(M in 'm' over i to l) {
|
||||
cw:= load_widen{*W~~w, i, R}
|
||||
cx:= load_widen{*X~~x, i, R}
|
||||
if (anyne_positive{cw,cx,M}) return{0}
|
||||
|
||||
@ -89,7 +89,7 @@ local def ml_exec{i, iter, vars0, bulk, M} = {
|
||||
}
|
||||
|
||||
# i0 - initial batch index; not used as begin because it's in a different scale compared to end
|
||||
def maskedLoop{bulk, i0}{vars,begin==0,end,iter} = {
|
||||
def for_masked{bulk, i0}{vars,begin==0,end,iter} = {
|
||||
l:u64 = end
|
||||
|
||||
m:u64 = l / bulk
|
||||
@ -98,10 +98,10 @@ def maskedLoop{bulk, i0}{vars,begin==0,end,iter} = {
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) ml_exec{m, iter, vars, bulk, mask_first{left}}
|
||||
}
|
||||
def maskedLoop{bulk} = maskedLoop{bulk,0}
|
||||
def for_masked{bulk} = for_masked{bulk,0}
|
||||
|
||||
|
||||
def maskedLoopPositive{bulk}{vars,begin==0,end:L,iter} = {
|
||||
def for_masked_pos{bulk}{vars,begin==0,end:L,iter} = {
|
||||
assert{end > 0}
|
||||
i:L = 0
|
||||
while(i < (end-1)/bulk) {
|
||||
@ -121,7 +121,7 @@ def maskedLoopPositive{bulk}{vars,begin==0,end:L,iter} = {
|
||||
# begin must be 0
|
||||
# end is scalar element count
|
||||
# index given is a tuple of batch indexes to process
|
||||
def muLoop{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
|
||||
def for_mu{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
|
||||
l:u64 = end
|
||||
|
||||
m:u64 = l / bulk
|
||||
@ -152,4 +152,4 @@ def muLoop{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
|
||||
if (left!=0) ml_exec{tup{m}, iter, vars, bulk, mask_first{left}}
|
||||
}
|
||||
}
|
||||
def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0}
|
||||
def for_mu{bulk, unr} = for_mu{bulk, unr, {}=>0}
|
||||
|
||||
@ -8,7 +8,7 @@ include './mask'
|
||||
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||
def bulk = arch_defvw/width{T}
|
||||
def VT = [bulk]T
|
||||
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
|
||||
@for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
|
||||
if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
|
||||
sr{each{abs, cx}}
|
||||
}
|
||||
|
||||
@ -329,7 +329,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
||||
d4:usz = width{V} % l
|
||||
m:= make{V, scan{{a,_} => a>>d | a<<(l-d), tup{ms, ...iota{3}}}}
|
||||
c:= V**0
|
||||
@maskedLoop{4} (x in tup{V, xp},
|
||||
@for_masked{4} (x in tup{V, xp},
|
||||
r in tup{V, rp} over promote{u64,nw}) {
|
||||
s := scan_words{x, m}
|
||||
pc:= c; c = shuf{-(s>>63), 3,0,1,2}
|
||||
|
||||
@ -25,7 +25,7 @@ def search{E, x, n:(u64), OP} = {
|
||||
tup{u64, ty_u{VT}},
|
||||
{i,c} => return{i*bulk + promote{u64, ctz_ext{hom_to_int_ext{c}}}}
|
||||
}
|
||||
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
||||
@for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
||||
eq:= each{OP, x}
|
||||
if (any_hom{M{tree_fold{|, eq}}}) {
|
||||
findFirst{
|
||||
@ -53,7 +53,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
|
||||
def E = f64
|
||||
def bulk = arch_defvw/width{E}
|
||||
def VT = [bulk]E
|
||||
@maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
|
||||
@for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
|
||||
if (any_hom{M{x!=x}}) return{1}
|
||||
sr{x + VT**0}
|
||||
}
|
||||
@ -324,7 +324,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
||||
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
|
||||
def minA = acc{2, VT**min1}
|
||||
def maxA = acc{2, VT**min1}
|
||||
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
|
||||
@for_mu{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
|
||||
if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
|
||||
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
|
||||
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend
|
||||
|
||||
@ -57,7 +57,7 @@ fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void =
|
||||
if (has_simd) {
|
||||
def bulk = arch_defvw / width{TI}
|
||||
def VT = [bulk]TI
|
||||
@maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
|
||||
@for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
|
||||
dst = blend_hom{src, src + VT**cyc, src < VT**0}
|
||||
}
|
||||
} else {
|
||||
@ -198,7 +198,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
# show{TD, nt, ni, G}
|
||||
# lprintf{'LUT of ', VI, ' ⊏ ', [nt]TD, ' with ', wl, ' ≡ ≠𝕨, ', xl, ' ≡ ≠𝕩'}
|
||||
def lut = G{x}
|
||||
@maskedLoop{ni}(w0 in tup{VI,w}, M in 'm' over wl) {
|
||||
@for_masked{ni}(w0 in tup{VI,w}, M in 'm' over wl) {
|
||||
def w = wrapChk{w0, xlf, M}
|
||||
def rs = lut{ty_u{w}}
|
||||
r = masked_multistore{r, rs, M, {} => return{1}}
|
||||
@ -220,7 +220,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
def VD = [bulk]TDE
|
||||
def xlf = VI**cast_i{TIE, xl}
|
||||
|
||||
@maskedLoop{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over wl) {
|
||||
@for_masked{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over wl) {
|
||||
cw:= wrapChk{cw0, xlf, M}
|
||||
got:= gather{VD**0, x, cw, M}
|
||||
if (TDE!=TD) got&= VD**((1<<wd)-1)
|
||||
@ -276,7 +276,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
||||
|
||||
if (hasarch{'AARCH64'}) {
|
||||
def xrev = rbit{load{*VU ~~ x0}}
|
||||
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
|
||||
@for_masked{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
|
||||
def cw = ty_u{wrapChk{cw0, xlf, M}}
|
||||
def byte = shuf{[16]u8, xrev, cw>>3}
|
||||
r = hom_to_int{ty_s{byte << (cw & VU**7)} < VI**0}
|
||||
@ -284,7 +284,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
||||
} else {
|
||||
if (wl>32 and xl<=16) {
|
||||
xb:= shuf{[4]u64, expand_bits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
|
||||
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
@for_masked{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
cw:= wrapChk{cw0, xlf, M}
|
||||
sr = hom_to_int{shuf{[16]i8, xb, cw}}
|
||||
}
|
||||
@ -292,7 +292,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
||||
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
|
||||
low:= VI**7
|
||||
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
@for_masked{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
|
||||
cw:= wrapChk{cw0, xlf, M}
|
||||
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
|
||||
mask:= shuf{[16]i8, b, cw & low}
|
||||
@ -308,7 +308,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
||||
def bulk = arch_defvw / 8
|
||||
def V = [bulk]u8
|
||||
def lut = shuf_u8bits{inds, indn}
|
||||
@maskedLoop{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
|
||||
@for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
|
||||
dst = lut{src}
|
||||
}
|
||||
}
|
||||
|
||||
@ -81,7 +81,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
|
||||
if (CHR) { # c16/c32/B → char
|
||||
mt:= XV**0 # unused for c16
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
def orx = M{tree_fold{|, xs}}
|
||||
if (B) {
|
||||
if (any_nonC32{M, ...xs}) return{3}
|
||||
@ -100,21 +100,21 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
def EH = w_h{X}
|
||||
def acc = int_acc{XV}
|
||||
if (acc{'minmax'}) {
|
||||
@muLoop{bulk, 4}(xs in xb, M in 'm' over len) {
|
||||
@for_mu{bulk, 4}(xs in xb, M in 'm' over len) {
|
||||
minc:= tree_fold{min, xs}
|
||||
maxc:= tree_fold{max, xs}
|
||||
if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff}
|
||||
acc{M, minc, maxc}
|
||||
}
|
||||
} else {
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
def mixed = acc{M, xs}
|
||||
if (M{0}==0 and any_hom{mixed > XV**maxvalue{EH}}) return{0xffff_ffff}
|
||||
}
|
||||
}
|
||||
acc{}
|
||||
} else if (X==i8) {
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
|
||||
if (~and_bit_none{M{tree_fold{|, xs}}, XV ** -2}) return{2}
|
||||
}
|
||||
0
|
||||
@ -122,14 +122,14 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
||||
def case_B = make_branch{tup{ux}, {bulkCont} => {
|
||||
def i0 = bulkCont*bulk
|
||||
x:= i0 + *u64~~x0
|
||||
if (B) @muLoop{bulk, 2}(xs in tup{[bulk]u64, x}, M in 'm' over len-i0) {
|
||||
if (B) @for_mu{bulk, 2}(xs in tup{[bulk]u64, x}, M in 'm' over len-i0) {
|
||||
if (any_sNaN{M, ...xs}) return{0xffff_fffe} # not even a number
|
||||
}
|
||||
return{0xffff_ffff} # float
|
||||
}}
|
||||
|
||||
def acc = int_acc{re_el{i32, XV}}
|
||||
@muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) {
|
||||
@for_mu{bulk, 2}(xs in xb, M in 'm' over is to len) {
|
||||
if (hasarch{'X86_64'}) {
|
||||
def ns = each{narrow{i32,.}, xs}
|
||||
if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user