170 lines
5.3 KiB
Plaintext
170 lines
5.3 KiB
Plaintext
include './base'
|
|
include './cbqnDefs'
|
|
include './sse3'
|
|
include './avx'
|
|
include './avx2'
|
|
include './mask'
|
|
include './bitops'
|
|
include 'util/tup'
|
|
|
|
oper &~ andnot infix none 35
|
|
|
|
# def:T - masked original content
|
|
# b:B - pointer to data to index; if width{B}<elwidth{T}, padding bytes are garbage read after wanted position
|
|
# idx - actual (unscaled) index list
|
|
def gather{def:T, b:B, idx:[8]i32, M & w256{T,32}} = {
|
|
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
|
|
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, elwidth{B}/8}
|
|
}
|
|
def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
|
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', def, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
|
|
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, elwidth{B}/8}
|
|
}
|
|
|
|
def wrapChk{cw0, VI,xlf, M} = {
|
|
cw:= cw0 + (xlf & VI~~(cw0<VI**0))
|
|
if (homAny{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
|
cw
|
|
}
|
|
|
|
def storeExp{dst, ind, val, M, ext, rd, wl} = {
|
|
def s{M} = storeBatch{dst, ind, val, M}
|
|
if (ext==1 or not M{0}) s{M}
|
|
else if (ind*rd+rd <= wl) s{maskNone}
|
|
else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} }
|
|
}
|
|
|
|
def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
|
|
def VI = [ri]TI
|
|
def ext = ri/rd
|
|
xlf:= VI**cast_i{TI, xl}
|
|
maskedLoop{ri, wl, {i, M} => {
|
|
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
|
is:= (if (ext>1) i<<lb{ext}; else i)
|
|
def se{e, c, o} = {
|
|
c2:= shuf{[4]u64, c+c, 4b3120}
|
|
each{
|
|
{c,o} => se{e*2, VI~~c, o},
|
|
unpackQ{c2, c2 + VI**1},
|
|
2*o + iota{2}
|
|
}
|
|
}
|
|
def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl}
|
|
se{1, cw, 0}
|
|
}}
|
|
}
|
|
|
|
def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
|
|
def VI = [ri]TI
|
|
def ext = ri/rd
|
|
xlf:= VI**cast_i{TI, xl}
|
|
maskedLoop{ri, wl, {i, M} => {
|
|
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
|
is:= (if (ext>1) i<<lb{ext}; else i)
|
|
def part{o} = cvt{i8, [8]i32, shuf{[4]u64, cw, 4b3210+o}}
|
|
def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl}
|
|
each{se, iota{ext}}
|
|
}}
|
|
}
|
|
|
|
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
|
|
def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m}
|
|
def bblendn{m}{tf} = bblend{m}{reverse{tf}}
|
|
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})
|
|
|
|
def bs{b, c, x} = cshuf{x, c}
|
|
def bs{b, c, x & tuplen{b}>0} = {
|
|
tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}}
|
|
}
|
|
|
|
def i = iota{logv}
|
|
def vs = each{bind{broadcast,VI}, nsel<<reverse{i}}
|
|
{c} => VD~~bs{each{bb{c},i==0,vs}, c, xd}
|
|
}
|
|
def makeshuf{VI, VD, x0, logv} = {
|
|
x:= *VD~~x0
|
|
def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}}
|
|
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)}
|
|
def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010}
|
|
def readx{l==1,o} = halves{load{x, o}}
|
|
xd:= readx{logv, 0}
|
|
makeselx{VI,VD,16,xd,logv, bind{sel,[16]i8}}
|
|
}
|
|
def makeperm{VI, VD, x0, logv} = {
|
|
x:= *VD~~x0
|
|
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-1)}
|
|
def readx{l==0,o} = load{x, o}
|
|
makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}}
|
|
}
|
|
|
|
select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
|
w:= *TI ~~ w0
|
|
x:= *TD ~~ x0
|
|
r:= *TD ~~ r0
|
|
|
|
def wd = width{TD}; def rd = rw/wd
|
|
def wi = width{TI}; def ri = rw/wi
|
|
def reg_select{sel,make}{l} = {
|
|
sel{ri, rd, TI, w, r, wl, xl, make{[ri]TI,[rd]TD, x,l}}
|
|
}
|
|
def shuf_select = reg_select{shuf_select, makeshuf}
|
|
def perm_select = reg_select{perm_select, makeperm}
|
|
if (wi==8 and wd==32 and xl*wd<=256 ) perm_select{0}
|
|
else if (wi==8 and wd==32 and xl*wd<=256<<1) perm_select{1}
|
|
else if (wi==8 and wd<=16 and xl*wd<=128 ) shuf_select{0}
|
|
else if (wi==8 and wd<=16 and xl*wd<=128<<1) shuf_select{1}
|
|
else if (wi==8 and wd<=16 and xl*wd<=128<<2) shuf_select{2}
|
|
else if (wi==8 and wd<= 8 and xl*wd<=128<<3) shuf_select{3}
|
|
else {
|
|
def TIE = i32
|
|
def TDE = tern{wd<32, u32, TD}
|
|
def bulk = rw / width{TDE}
|
|
def VI = [bulk]TIE
|
|
def VD = [bulk]TDE
|
|
def xlf = VI**cast_i{TIE, xl}
|
|
|
|
maskedLoop{bulk, wl, {i, M} => {
|
|
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
|
got:= gather{VD**0, x, cw, M}
|
|
if (TDE!=TD) got&= VD**((1<<wd)-1)
|
|
storeBatch{r, i, got, M}
|
|
}}
|
|
}
|
|
1
|
|
}
|
|
def select{TI, TD} = select{256, TI, TD}
|
|
|
|
def selects = join{table{select, tup{i8, i16, i32}, # indices
|
|
tup{u8, u16, u32, u64}}} # values
|
|
avx2_select_tab:*type{tupsel{0,selects}} = selects
|
|
'avx2_select_tab' = avx2_select_tab
|
|
|
|
|
|
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
|
def TI = i8
|
|
def VI = [32]TI
|
|
w:= *VI ~~ w0
|
|
r:= *u32 ~~ r0
|
|
xlf:= VI**cast_i{TI, xl}
|
|
|
|
if (wl>32 and xl<=16) {
|
|
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
|
|
maskedLoop{32, wl, {i, M} => {
|
|
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
|
store{r, i, getmask{sel{[16]i8, xb, cw}}}
|
|
}}
|
|
} else {
|
|
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
|
|
low:= VI**7
|
|
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
|
maskedLoop{32, wl, {i, M} => {
|
|
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
|
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
|
|
mask:= sel{[16]i8, b, cw & low}
|
|
store{r, i, getmask{(mask & byte) == mask}}
|
|
}}
|
|
}
|
|
1
|
|
}
|
|
'avx2_select_bool128' = avx2_select_bool128
|