diff --git a/src/builtins/select.c b/src/builtins/select.c index ccd3759f..6eaa962c 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -557,10 +557,12 @@ B select_cells_base(B inds, B x0, ux csz, ux cam); #define INDS_BUF_MAX 64 // only need 32 bytes for AVX2 & 16 for NEON, but have more for past-the-end pointers and writes B select_rows_typed(B x, ux csz, ux cam, void* inds, ux indn, u8 ie, bool shouldBoundsCheck) { // ⥊ (indn↑inds As ie)⊸⊏˘ cam‿csz⥊z; xe cannot be el_bit or el_B, unless csz==1; ie must be ≤el_i8 if csz≤128 - u8 inds_buf[INDS_BUF_MAX]; (void)inds_buf; assert(csz!=0 && cam!=0); assert(csz*cam == IA(x)); assert(ie<=el_i32); + + u8 inds_buf[INDS_BUF_MAX]; (void)inds_buf; + bool generic_allowed = true; // whether required interpretation of x hasn't changed from its real one if (csz==1) { // TODO maybe move to select_rows_B and require csz>=2 here? i64 bounds[2]; if (!getRange_fns[ie](inds, bounds, indn) || bounds[0]<-1 || bounds[1]>0) goto generic; // could put under shouldBoundsCheck but ideally things setting that to false should handle size-1 cells themselves @@ -575,13 +577,14 @@ B select_rows_typed(B x, ux csz, ux cam, void* inds, ux indn, u8 ie, bool should u8 lb = arrTypeWidthLog(TY(x)); ux xbump = csz<32 || indn>32 || indn>INDS_BUF_MAX) { // TODO properly tune - u8* rp = m_tyarrv_same(&r, indn * cam, x); + u8* rp = m_tyarrv_same(&r, ria, x); for (ux i = 0; i < cam; i++) { bitselFns[lb](rp, inds, loadu_u64(xp), loadu_u64(xp + (1< { # } }}} +def raw_widen_inds{[k]D, x:[k0]S if k0>=k} = { # : [k*sc]S + def sc = width{D} / width{S} + def add = make{[k*sc]S, range{k*sc} % sc} + if (hasarch{'AVX2'} and [k]D == [4]u64 and S==u32) { + (sel{[8]u32, undefPromote{[8]u32, x}, make{[8]u32, range{8}>>1}}< tup{nt1/sc, ni1/sc, loader{{TG} => { def prev = G{TG} def ni = ni1/sc def WV = [ni]primtype{'u', 8*sc} {is:([ni]u8)} => { - def isw = widen{WV, is} * WV**base{256, sc**sc} + WV**base{256, range{sc}} - each{re_el{E,.}, prev{re_el{u8, isw}}} + each{re_el{E,.}, prev{raw_widen_inds{WV, is}}} } }}} {x} => x @@ -168,10 +178,13 @@ def lut_gen{mode, E==u8, nt, ni if hasarch{'AARCH64'} and nt<=16*4 and ni<=16} = {is:([16]u8)} => tup{sel{lut, is}} }}}} -def lut_gen{mode, E, nt, ni if (E==u16 or E==u64) and mode=='i'} = zip_halves{mode, E, nt, ni} -def lut_gen{mode, E, nt, ni if (E==u16 or E==u64) and mode=='c'} = widen_inds{mode, E, nt, max{ni,16}, 2} -def lut_gen{mode, E, nt, ni if E==u32 and hasarch{'AARCH64'}} = zip_halves{mode, E, nt, ni} -# def lut_gen{mode, E, nt, ni if E==u32 and hasarch{'AARCH64'}} = widen_inds{mode, E, nt, ni, 2} -def lut_gen{mode, E, nt, ni if E==u64 and hasarch{'AARCH64'}} = widen_inds{mode, E, nt, ni, 2} +def lut_gen{mode, E, nt, ni if mode=='i' and hasarch{'AVX2'} and (E==u16 or E==u64)} = zip_halves{mode, E, nt, ni} +# def lut_gen{mode, E, nt, ni if mode=='c' and hasarch{'AVX2'} and E==u16} = widen_inds{mode, E, nt, max{ni,16}, 2} +# def lut_gen{mode, E, nt, ni if mode=='c' and hasarch{'AVX2'} and E==u64} = widen_inds{mode, E, nt, ni, 2} +# def lut_gen{mode, E, nt, ni if mode=='c' and E==u64} = zip_halves{mode, E, nt, ni} # widen_inds{mode, E, nt, max{ni,16}, 2} def lut_gen{mode, E==u64, nt, ni if nt>16 and hasarch{'AVX2'}} = 0 + + +def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and (E==u16 or E==u32)} = zip_halves{mode, E, nt, ni} +def lut_gen{mode, E, nt, ni if hasarch{'AARCH64'} and E==u64} = widen_inds{mode, E, nt, ni, 2} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 2f9afaba..8145489f 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -49,12 +49,14 @@ def masked_multistore{r0, vs, M, end} = { # returns bumped-forwards r +def vptr{VT=[_]E, ptr} = tup{VT, *E~~ptr} + fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = { def cyc = cast_i{TI,cyc0} if (has_simd) { def bulk = arch_defvw / width{TI} def VT = [bulk]TI - @maskedLoop{bulk}(src in tup{VT, *TI~~src}, dst in tup{VT, *TI~~dst} over n) { + @maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) { dst = homBlend{src, src + VT**cyc, src < VT**0} } } else { @@ -65,6 +67,8 @@ exportT{'si_wrap_inds', each{wrap_inds, tup{i8}}} +def inds_buf_max = 64 +export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max} (if (has_sel) { fn select_rows_fn{TD, nt, ni, G}(inds:*u8, x0:*void, xbump:u64, r0:*void, rbump:u64, r1:*void) : ux = G{inds, *TD~~x0, xbump, *TD~~r0, rbump, r1} # TG,nt,ni args just for prettier names for debugging @@ -150,6 +154,27 @@ exportT{'si_wrap_inds', each{wrap_inds, tup{i8}}} def exportP{T, n, vs} = { a:*T = vs; export{n, a} } exportP{u8, 'select_rows_max_indn', each{{row} => if (length{row}==0) 0 else oneVal{ each{select{.,1}, row}}, select_rows_parts}} exportP{u8, 'select_rows_min_logcsz', each{{row} => if (length{row}==0) 0 else lb{fold{min, each{select{.,0}, row}}}, select_rows_parts}} + + def select_rows_better = scan{{p,{v,i}} => if (length{v}==0) p else i, 0, each{tup, select_rows_parts, range{4}}} + exportP{u8, 'select_rows_better', select_rows_better+1} + + fn select_rows_widen{sh}(src:*void, dst:*void, n:ux) : ux = { + if (sh != 0) { + def bulk = (arch_defvw/8) >> sh + def WV = [bulk< inds_buf_max) return{0} + @for_backwards (i to inds_buf_max/(width{WV}/8)) { + def s = load{*[bulk]u8~~src, i} + def v = raw_widen_inds{bulk, sh, s} + store{*WV~~dst, i, v} + } + } + sh + } + exportT{'select_rows_widen', each{{t0, t1} => { + def {S, D} = each{select{tup{u8,u16,u32,u64},.}, tup{t0, t1}} + select_rows_widen{t1-t0} + }, select_rows_better, range{4}}} })