make arch_defvw a generator
This commit is contained in:
parent
624c74058c
commit
4e12204c52
@ -161,7 +161,7 @@ def el_s{V=[_]T} = re_el{w_h{T}, V}
|
|||||||
|
|
||||||
|
|
||||||
# vector definitions
|
# vector definitions
|
||||||
def arch_defvw = if (hasarch{'AVX2'}) 256 else 128
|
def arch_defvw{} = if (hasarch{'AVX2'}) 256 else 128
|
||||||
def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'}
|
def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'}
|
||||||
def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'}
|
def has_sel = hasarch{'AVX2'} or hasarch{'AARCH64'}
|
||||||
def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP
|
def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP
|
||||||
|
|||||||
@ -22,7 +22,7 @@ fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
|
|||||||
|
|
||||||
def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}}
|
def table{w} = each{bitsel_i{w, .}, tup{u8, u16, u32, u64}}
|
||||||
|
|
||||||
export_tab{'simd_bitsel', table{arch_defvw}}
|
export_tab{'simd_bitsel', table{arch_defvw{}}}
|
||||||
|
|
||||||
fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : void = {
|
fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : void = {
|
||||||
if (same{E,'!'}) {
|
if (same{E,'!'}) {
|
||||||
@ -30,7 +30,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
|
|||||||
} else if (E==u1) {
|
} else if (E==u1) {
|
||||||
emit{void, 'blendArrScalarBits', rp, zero, cast_i{E, one0}, mask, len}
|
emit{void, 'blendArrScalarBits', rp, zero, cast_i{E, one0}, mask, len}
|
||||||
} else {
|
} else {
|
||||||
def bulk = arch_defvw / width{E}
|
def bulk = arch_defvw{} / width{E}
|
||||||
def VT = [bulk]E
|
def VT = [bulk]E
|
||||||
def one = VT**cast_i{E, one0}
|
def one = VT**cast_i{E, one0}
|
||||||
@for_mu{bulk,mask_unr{bulk}}(r in tup{'g',*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r{eachx{blend_hom, zero, one, mask}}
|
@for_mu{bulk,mask_unr{bulk}}(r in tup{'g',*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r{eachx{blend_hom, zero, one, mask}}
|
||||||
@ -43,7 +43,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
|
|||||||
fn bitwiden_n_8(src:*void, dst:*void, csz:ux, cam:ux) : void = {
|
fn bitwiden_n_8(src:*void, dst:*void, csz:ux, cam:ux) : void = {
|
||||||
assert{cam>0}
|
assert{cam>0}
|
||||||
assert{(csz>1) & (csz<8)}
|
assert{(csz>1) & (csz<8)}
|
||||||
def bulk = arch_defvw / 8
|
def bulk = arch_defvw{} / 8
|
||||||
def V = [bulk]u8
|
def V = [bulk]u8
|
||||||
bitalign{tup{2,8,csz}, 8, {s, align} => {
|
bitalign{tup{2,8,csz}, 8, {s, align} => {
|
||||||
@for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) {
|
@for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) {
|
||||||
@ -59,7 +59,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
|
|||||||
fn bitnarrow_8_n(src:*void, dst:*void, csz:ux, cam:ux) : void = {
|
fn bitnarrow_8_n(src:*void, dst:*void, csz:ux, cam:ux) : void = {
|
||||||
assert{cam>0}
|
assert{cam>0}
|
||||||
assert{(csz>1) & (csz<8)}
|
assert{(csz>1) & (csz<8)}
|
||||||
def bulk = arch_defvw / 8
|
def bulk = arch_defvw{} / 8
|
||||||
def V = [bulk]u8
|
def V = [bulk]u8
|
||||||
dstC:= *u8~~dst
|
dstC:= *u8~~dst
|
||||||
dstE:= *u8~~dst + cdiv{csz*cam, 8}
|
dstE:= *u8~~dst + cdiv{csz*cam, 8}
|
||||||
|
|||||||
@ -130,9 +130,9 @@ def tableAA{w, F, F_bit} = table{1, F, w, {} => bitAA{F_bit}, {V, u} => aa2bit{V
|
|||||||
def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}}
|
def tableAS{w, F } = table{0, F, w, {} => bitAS{F}, {V, u} => as2bit{V, u, F}}
|
||||||
|
|
||||||
|
|
||||||
export_tab{'simd_eqAS', tableAS{arch_defvw, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw, __eq, {a,b}=>a ^ ~b}}
|
export_tab{'simd_eqAS', tableAS{arch_defvw{}, __eq}}; export_tab{'simd_eqAA', tableAA{arch_defvw{}, __eq, {a,b}=>a ^ ~b}}
|
||||||
export_tab{'simd_neAS', tableAS{arch_defvw, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw, __ne, {a,b}=>a ^ b}}
|
export_tab{'simd_neAS', tableAS{arch_defvw{}, __ne}}; export_tab{'simd_neAA', tableAA{arch_defvw{}, __ne, {a,b}=>a ^ b}}
|
||||||
export_tab{'simd_gtAS', tableAS{arch_defvw, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw, __gt, {a,b}=>a & ~b}}
|
export_tab{'simd_gtAS', tableAS{arch_defvw{}, __gt}}; export_tab{'simd_gtAA', tableAA{arch_defvw{}, __gt, {a,b}=>a & ~b}}
|
||||||
export_tab{'simd_geAS', tableAS{arch_defvw, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw, __ge, {a,b}=>a | ~b}}
|
export_tab{'simd_geAS', tableAS{arch_defvw{}, __ge}}; export_tab{'simd_geAA', tableAA{arch_defvw{}, __ge, {a,b}=>a | ~b}}
|
||||||
export_tab{'simd_ltAS', tableAS{arch_defvw, __lt}}
|
export_tab{'simd_ltAS', tableAS{arch_defvw{}, __lt}}
|
||||||
export_tab{'simd_leAS', tableAS{arch_defvw, __le}}
|
export_tab{'simd_leAS', tableAS{arch_defvw{}, __le}}
|
||||||
|
|||||||
@ -14,7 +14,7 @@ def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = {
|
|||||||
|
|
||||||
|
|
||||||
fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
||||||
def vw = arch_defvw
|
def vw = arch_defvw{}
|
||||||
assert{l!=0}
|
assert{l!=0}
|
||||||
|
|
||||||
def bulk = vw/__max{width{X}, width{R}}
|
def bulk = vw/__max{width{X}, width{R}}
|
||||||
@ -68,7 +68,7 @@ fn copy_ubit{R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
|
|||||||
# if ((xs&7)==0) {
|
# if ((xs&7)==0) {
|
||||||
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
|
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
|
||||||
# } else {
|
# } else {
|
||||||
def vw = arch_defvw
|
def vw = arch_defvw{}
|
||||||
def bulk = vw/width{R}
|
def bulk = vw/width{R}
|
||||||
def RV = [bulk]R
|
def RV = [bulk]R
|
||||||
rp:= *R~~r
|
rp:= *R~~r
|
||||||
|
|||||||
@ -6,7 +6,7 @@ def inc{ptr, ind} = inc{ptr, ind, 1}
|
|||||||
|
|
||||||
# Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x
|
# Write counts (2⋆15)|/⁼x to tab, overflows to ov, and return ⌈´x
|
||||||
fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
|
fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
|
||||||
def vbits = arch_defvw
|
def vbits = arch_defvw{}
|
||||||
def vec = vbits/width{T}
|
def vec = vbits/width{T}
|
||||||
def TU = ty_u{T}
|
def TU = ty_u{T}
|
||||||
def V = [vec]T
|
def V = [vec]T
|
||||||
@ -77,7 +77,7 @@ fn count{T if T<=i16}(tab:*u16, ov:*u16, xp:*void, n:u64, min_allowed:T) : T = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
|
fn flush_counts(tab:*u16, ov:*u16, n:usz) : usz = {
|
||||||
def vl = arch_defvw/16
|
def vl = arch_defvw{}/16
|
||||||
def V = [vl]u16
|
def V = [vl]u16
|
||||||
def bot = tail{15}
|
def bot = tail{15}
|
||||||
on:usz = 0
|
on:usz = 0
|
||||||
@ -147,7 +147,7 @@ def count_adapt_runs{x0, tab, n} = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
def mark_run_ends{x:*T, m:(ux)} = {
|
def mark_run_ends{x:*T, m:(ux)} = {
|
||||||
def vec = arch_defvw/width{T}
|
def vec = arch_defvw{}/width{T}
|
||||||
def V = [vec]T
|
def V = [vec]T
|
||||||
@unroll (j to width{ux} / vec) {
|
@unroll (j to width{ux} / vec) {
|
||||||
def jv = j*vec
|
def jv = j*vec
|
||||||
@ -175,7 +175,7 @@ fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = count_adapt_runs{x, tab, n}
|
|||||||
# ov is sorted but may not be unique, and oc contains multiples of 128
|
# ov is sorted but may not be unique, and oc contains multiples of 128
|
||||||
# Return the shared length of ov and oc
|
# Return the shared length of ov and oc
|
||||||
fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = {
|
fn count_sorted{T}(r:*u8, ov:*usz, oc:*usz, x:*T, n:usz) : usz = {
|
||||||
def V = [arch_defvw/width{T}]T
|
def V = [arch_defvw{}/width{T}]T
|
||||||
def block = 128
|
def block = 128
|
||||||
i:usz = 0
|
i:usz = 0
|
||||||
on:usz = 0
|
on:usz = 0
|
||||||
|
|||||||
@ -164,7 +164,7 @@ fn arithAAu{vw, mode, F, W, X, R}(r:*void, w:*void, x:*void, len:u64) : void = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def arithAA{mode, F, W, X, R} = {
|
def arithAA{mode, F, W, X, R} = {
|
||||||
def vw = arch_defvw
|
def vw = arch_defvw{}
|
||||||
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
|
if (mode==1 or mode==2) arithAAc{vw, mode, F, W, X, R}
|
||||||
else arithAAu{vw, mode, F, W, X, R}
|
else arithAAu{vw, mode, F, W, X, R}
|
||||||
}
|
}
|
||||||
@ -192,7 +192,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
|
|||||||
else len
|
else len
|
||||||
}
|
}
|
||||||
|
|
||||||
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw, mode, F, swap, W, X, R}
|
def arithSA{mode, F, swap, W, X, R} = arithSAf{arch_defvw{}, mode, F, swap, W, X, R}
|
||||||
|
|
||||||
fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
||||||
assert{vw>=64}
|
assert{vw>=64}
|
||||||
@ -203,6 +203,6 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
|
|||||||
@for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
|
@for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
|
||||||
}
|
}
|
||||||
|
|
||||||
export{'simd_andBytes', andBytes{arch_defvw}}
|
export{'simd_andBytes', andBytes{arch_defvw{}}}
|
||||||
export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}}
|
export{'orSAc_f64_f64_f64', arithSA{2,bqn_or,0,f64,f64,f64}}
|
||||||
include 'gen/arDefs'
|
include 'gen/arDefs'
|
||||||
|
|||||||
@ -12,7 +12,7 @@ def swap{w,x} = {
|
|||||||
|
|
||||||
# width{W} ≤ width{X}
|
# width{W} ≤ width{X}
|
||||||
fn equal{W, X}(w:*void, x:*void, l:ux, d:u64) : u1 = {
|
fn equal{W, X}(w:*void, x:*void, l:ux, d:u64) : u1 = {
|
||||||
def vw = arch_defvw
|
def vw = arch_defvw{}
|
||||||
def bulk = vw / width{X}
|
def bulk = vw / width{X}
|
||||||
if (W!=X) if (d!=0) swap{w,x}
|
if (W!=X) if (d!=0) swap{w,x}
|
||||||
assert{l>0}
|
assert{l>0}
|
||||||
@ -83,7 +83,7 @@ def any_ne_reflexive_qnan{M, a:V=[k](f64), b:V} = { # (a==b) | (isQNaN{a} & isQN
|
|||||||
fn equal_reflexive{}(w:*void, x:*void, l:ux, d:u64) : u1 = {
|
fn equal_reflexive{}(w:*void, x:*void, l:ux, d:u64) : u1 = {
|
||||||
def w = *f64~~w
|
def w = *f64~~w
|
||||||
def x = *f64~~x
|
def x = *f64~~x
|
||||||
def bulk = arch_defvw / width{f64}
|
def bulk = arch_defvw{} / width{f64}
|
||||||
def unr = if (hasarch{'AARCH64'}) 2 else 1
|
def unr = if (hasarch{'AARCH64'}) 2 else 1
|
||||||
def V = [bulk]f64
|
def V = [bulk]f64
|
||||||
@for_mu{bulk, unr}(w in tup{V,w}, x in tup{V,x}, M in 'm' over i to l) {
|
@for_mu{bulk, unr}(w in tup{V,w}, x in tup{V,x}, M in 'm' over i to l) {
|
||||||
|
|||||||
@ -29,7 +29,7 @@ fn fold_idem{T, op}(x:*T, len:u64) : T = {
|
|||||||
a
|
a
|
||||||
}
|
}
|
||||||
fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
||||||
def bulk = arch_defvw/width{T}
|
def bulk = arch_defvw{}/width{T}
|
||||||
def V = [bulk]T
|
def V = [bulk]T
|
||||||
xv:= *V ~~ x
|
xv:= *V ~~ x
|
||||||
r:V = [bulk]f64**0
|
r:V = [bulk]f64**0
|
||||||
@ -62,7 +62,7 @@ fn fold_assoc_0{T, op}(x:*T, len:u64) : T = {
|
|||||||
a
|
a
|
||||||
}
|
}
|
||||||
fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
||||||
def bulk = arch_defvw/width{T}
|
def bulk = arch_defvw{}/width{T}
|
||||||
def V = [bulk]T
|
def V = [bulk]T
|
||||||
xv:= *V ~~ x
|
xv:= *V ~~ x
|
||||||
e:= len / bulk
|
e:= len / bulk
|
||||||
@ -75,7 +75,7 @@ export{'si_sum_f64', fold_assoc_0{f64,+}}
|
|||||||
|
|
||||||
|
|
||||||
def extract_column_pow2{T, x0, r0, nv, k} = {
|
def extract_column_pow2{T, x0, r0, nv, k} = {
|
||||||
def V = [arch_defvw / width{T}]T
|
def V = [arch_defvw{} / width{T}]T
|
||||||
xv := *V~~x0
|
xv := *V~~x0
|
||||||
@for (r in *V~~r0 over i to nv) {
|
@for (r in *V~~r0 over i to nv) {
|
||||||
xs := each{load{xv, .}, iota{k}}
|
xs := each{load{xv, .}, iota{k}}
|
||||||
@ -234,7 +234,7 @@ def extract_column_modperm{x0, r0, nv, l, el, vl} = {
|
|||||||
# Maximum of n result values, return actual number written
|
# Maximum of n result values, return actual number written
|
||||||
fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = {
|
fn extract_column(x0:*void, r0:*void, n:usz, l:usz, el:u8) : usz = {
|
||||||
n <<= el
|
n <<= el
|
||||||
def vl = arch_defvw / 8
|
def vl = arch_defvw{} / 8
|
||||||
def thr = __min{vl+2, 20}
|
def thr = __min{vl+2, 20}
|
||||||
if ((not has_simd) or n < vl or l > usz~~thr>>el or l<<el >= thr) return{0}
|
if ((not has_simd) or n < vl or l > usz~~thr>>el or l<<el >= thr) return{0}
|
||||||
nv := n / vl
|
nv := n / vl
|
||||||
@ -529,7 +529,7 @@ fn sum_small{T if has_simd}(xv:*void, ia:usz) : i64 = {
|
|||||||
def A = if (width{T}<=16) i32 else i64
|
def A = if (width{T}<=16) i32 else i64
|
||||||
xv:= *T~~xv
|
xv:= *T~~xv
|
||||||
def unr = 4
|
def unr = 4
|
||||||
def bulk = arch_defvw / width{T}
|
def bulk = arch_defvw{} / width{T}
|
||||||
def V = [bulk]T
|
def V = [bulk]T
|
||||||
|
|
||||||
def acc = sum_accumulator{A, unr, V}
|
def acc = sum_accumulator{A, unr, V}
|
||||||
|
|||||||
@ -21,7 +21,7 @@ def group_statistics{T} = {
|
|||||||
fn group_statistics{T}(w:*void, xn:usz, outs:each{__pnt,types}) : void = {
|
fn group_statistics{T}(w:*void, xn:usz, outs:each{__pnt,types}) : void = {
|
||||||
def w = *T~~w
|
def w = *T~~w
|
||||||
def accs = if (has_simd) {
|
def accs = if (has_simd) {
|
||||||
def bulk = arch_defvw/width{T}
|
def bulk = arch_defvw{}/width{T}
|
||||||
def V = [bulk]T
|
def V = [bulk]T
|
||||||
def VU = ty_u{V}
|
def VU = ty_u{V}
|
||||||
|
|
||||||
|
|||||||
@ -36,7 +36,7 @@ local def loader{G} = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def unzip_load{E, n, TG} = each{merge, unzip_load{E, n/2, TG}, unzip_load{E, n/2, TG{'offset', E, n}}}
|
def unzip_load{E, n, TG} = each{merge, unzip_load{E, n/2, TG}, unzip_load{E, n/2, TG{'offset', E, n}}}
|
||||||
def unzip_load{E, n, TG if width{E}*n <= arch_defvw} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}}
|
def unzip_load{E, n, TG if width{E}*n <= arch_defvw{}} = each{tup, unzip{TG{[n]E, 0}, TG{[n]E, 1}}}
|
||||||
|
|
||||||
def widen_tup{(u32), is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is
|
def widen_tup{(u32), is:([16]u8)} = tup{ # compiler will deduplicate all the repeated calls of this on the same is
|
||||||
widen{[8]u32, is},
|
widen{[8]u32, is},
|
||||||
|
|||||||
@ -6,7 +6,7 @@ include './mask'
|
|||||||
|
|
||||||
|
|
||||||
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
|
||||||
def bulk = arch_defvw/width{T}
|
def bulk = arch_defvw{}/width{T}
|
||||||
def VT = [bulk]T
|
def VT = [bulk]T
|
||||||
@for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
|
@for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
|
||||||
if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk}
|
if (T!=f64 and any_hom{M, ...eachx{==, cx, VT**minvalue{T}}}) return{select{is,0}*bulk}
|
||||||
|
|||||||
@ -82,7 +82,7 @@ def fact_inds = slice{iota{fact_size+1},8}
|
|||||||
def fact_tab = makefact{basic_rep, fact_inds}
|
def fact_tab = makefact{basic_rep, fact_inds}
|
||||||
factors:*u8 = fact_tab
|
factors:*u8 = fact_tab
|
||||||
|
|
||||||
def sdtype = [arch_defvw/8]i8 # shuf data type
|
def sdtype = [arch_defvw{}/8]i8 # shuf data type
|
||||||
def get_shufs{step, wv, len} = {
|
def get_shufs{step, wv, len} = {
|
||||||
def i = iota{len*step}
|
def i = iota{len*step}
|
||||||
split{step, (i - i%wv)/wv}
|
split{step, (i - i%wv)/wv}
|
||||||
@ -240,7 +240,7 @@ fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
|
|||||||
|
|
||||||
def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = {
|
def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = {
|
||||||
assert{kv > 0}
|
assert{kv > 0}
|
||||||
def V = [arch_defvw/width{T}]T
|
def V = [arch_defvw{}/width{T}]T
|
||||||
@for (x over n) {
|
@for (x over n) {
|
||||||
v := V**x
|
v := V**x
|
||||||
@loop (j to kv) store{*V~~r, j, v}
|
@loop (j to kv) store{*V~~r, j, v}
|
||||||
@ -264,7 +264,7 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
def wT = width{T}
|
def wT = width{T}
|
||||||
def vn = arch_defvw/wT
|
def vn = arch_defvw{}/wT
|
||||||
def V = [vn]T
|
def V = [vn]T
|
||||||
def max_shuffle = 2*vn
|
def max_shuffle = 2*vn
|
||||||
if (wv <= max_shuffle) {
|
if (wv <= max_shuffle) {
|
||||||
|
|||||||
@ -17,7 +17,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = {
|
def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = {
|
||||||
def step = arch_defvw/width{T}
|
def step = arch_defvw{}/width{T}
|
||||||
def V = [step]T
|
def V = [step]T
|
||||||
p:= V**init
|
p:= V**init
|
||||||
xv:= *V~~x
|
xv:= *V~~x
|
||||||
@ -42,7 +42,7 @@ def scan_idem = scan_scal
|
|||||||
fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = {
|
fn scan_idem{T, op if has_simd}(x:*T, r:*T, len:u64, init:T) : void = {
|
||||||
def {scan, last} = get_scan_last{op, make_scan_idem{T, op}}
|
def {scan, last} = get_scan_last{op, make_scan_idem{T, op}}
|
||||||
def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) }
|
def cmp = match (op) { {(__min)} => (>); {(__max)} => (<) }
|
||||||
def step = arch_defvw/width{T}
|
def step = arch_defvw{}/width{T}
|
||||||
def V = [step]T
|
def V = [step]T
|
||||||
p:= V**init
|
p:= V**init
|
||||||
xv:= *V ~~ x
|
xv:= *V ~~ x
|
||||||
@ -183,7 +183,7 @@ fn scan_stride_assoc{op, T, Ret, check_over}(xv:*void, rv:*void, ia:usz, l:usz)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (same{op, +} and T<=i32 and has_simd and (has_shuf or l>=vl)) {
|
} else if (same{op, +} and T<=i32 and has_simd and (has_shuf or l>=vl)) {
|
||||||
def vl = arch_defvw/wT; def V = [vl]T
|
def vl = arch_defvw{}/wT; def V = [vl]T
|
||||||
@for_masked{vl} (x in tup{V, x+l}, r in tup{V, r+l}, p in tup{V, r}, M in 'm' over ia-l) {
|
@for_masked{vl} (x in tup{V, x+l}, r in tup{V, r+l}, p in tup{V, r}, M in 'm' over ia-l) {
|
||||||
r = op_chk{M, p, x}
|
r = op_chk{M, p, x}
|
||||||
}
|
}
|
||||||
@ -225,7 +225,7 @@ fn scan_neq{}(c:u64, x:*u64, r:*u64, nw:usz) : void = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:usz) : void = {
|
fn scan_neq{if has_simd}(c0:u64, x:*u64, r:*u64, nw:usz) : void = {
|
||||||
def vl = arch_defvw / 64
|
def vl = arch_defvw{} / 64
|
||||||
def V = [vl]u64
|
def V = [vl]u64
|
||||||
c := V**c0
|
c := V**c0
|
||||||
@for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) {
|
@for_masked{vl} (x in tup{V, x}, r in tup{V, r} over nw) {
|
||||||
@ -362,9 +362,9 @@ def addChk{a:T, b:T==f64} = tup{0, a+b}
|
|||||||
def widenFull{E, xs} = {
|
def widenFull{E, xs} = {
|
||||||
merge{...each{{x:X=[k]T} => {
|
merge{...each{{x:X=[k]T} => {
|
||||||
def tb = width{E} * k
|
def tb = width{E} * k
|
||||||
if (tb<=arch_defvw) tup{widen{[k]E, x}}
|
if (tb<=arch_defvw{}) tup{widen{[k]E, x}}
|
||||||
else if (1) {
|
else if (1) {
|
||||||
assert{tb == 2*arch_defvw}
|
assert{tb == 2*arch_defvw{}}
|
||||||
tup{
|
tup{
|
||||||
widen{[k/2]E, half{x,0}},
|
widen{[k/2]E, half{x,0}},
|
||||||
widen{[k/2]E, half{x,1}}
|
widen{[k/2]E, half{x,1}}
|
||||||
@ -395,7 +395,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
|
|||||||
# Sum as many vector registers as possible; modifies c and i
|
# Sum as many vector registers as possible; modifies c and i
|
||||||
def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
|
def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
|
||||||
def b = __max{width{R}/2, width{X}}
|
def b = __max{width{R}/2, width{X}}
|
||||||
def bulk = arch_defvw/b
|
def bulk = arch_defvw{}/b
|
||||||
|
|
||||||
def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size
|
def wd = (X!=R) & (width{X}<32) # whether to widen the working copy one size
|
||||||
def WE = tern{wd, w_d{X}, X} # working copy element type
|
def WE = tern{wd, w_d{X}, X} # working copy element type
|
||||||
@ -407,7 +407,7 @@ def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = {
|
|||||||
|
|
||||||
if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} }
|
if (R!=f64) { def m = maxFastA + maxFastE*bulk; assert{m<=maxvalue{R}}; assert{-m>=minvalue{R}} }
|
||||||
|
|
||||||
cv:= [arch_defvw/width{R}]R ** c
|
cv:= [arch_defvw{}/width{R}]R ** c
|
||||||
|
|
||||||
if (R==f64 and c != __floor{c}) goto{'end'}
|
if (R==f64 and c != __floor{c}) goto{'end'}
|
||||||
while (1) {
|
while (1) {
|
||||||
@ -494,7 +494,7 @@ def loop_with_unaligned_mask{x, r, nw, l, step} = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
||||||
def vl = arch_defvw / 64
|
def vl = arch_defvw{} / 64
|
||||||
def V = [vl]u64
|
def V = [vl]u64
|
||||||
{ms, d} := unaligned_spaced_mask_mod{l}
|
{ms, d} := unaligned_spaced_mask_mod{l}
|
||||||
d4:usz = width{V} % l
|
d4:usz = width{V} % l
|
||||||
@ -513,7 +513,7 @@ def vec_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
|||||||
}
|
}
|
||||||
def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry, apply_carry} = {
|
def vec_loop_with_loose_mask{xp, rp, nw, l, id, scan_words, propagate, fix_carry, apply_carry} = {
|
||||||
assert{l >= 64}
|
assert{l >= 64}
|
||||||
def vl = arch_defvw / 64
|
def vl = arch_defvw{} / 64
|
||||||
def V = [vl]u64
|
def V = [vl]u64
|
||||||
def get_m = loose_mask_gen{V, l}
|
def get_m = loose_mask_gen{V, l}
|
||||||
c := V**id # carry, 0 or 1
|
c := V**id # carry, 0 or 1
|
||||||
|
|||||||
@ -20,13 +20,13 @@ def find_first{C, M, F, ...v1} = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def search{E, x, n:(u64), OP} = {
|
def search{E, x, n:(u64), OP} = {
|
||||||
def bulk = arch_defvw/width{E}
|
def bulk = arch_defvw{}/width{E}
|
||||||
def VT = [bulk]E
|
def VT = [bulk]E
|
||||||
def end = make_branch{
|
def end = make_branch{
|
||||||
tup{u64, ty_u{VT}},
|
tup{u64, ty_u{VT}},
|
||||||
{i,c} => return{i*bulk + promote{u64, first_hom{c}}}
|
{i,c} => return{i*bulk + promote{u64, first_hom{c}}}
|
||||||
}
|
}
|
||||||
@for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
@for_mu{bulk, tern{arch_defvw{}>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
|
||||||
eq:= each{OP, x}
|
eq:= each{OP, x}
|
||||||
if (any_hom{M, ...eq}) {
|
if (any_hom{M, ...eq}) {
|
||||||
find_first{
|
find_first{
|
||||||
@ -52,7 +52,7 @@ fn searchNormalizable{}(x:*f64, len:u64) : u64 = {
|
|||||||
|
|
||||||
fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
|
fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
|
||||||
def E = f64
|
def E = f64
|
||||||
def bulk = arch_defvw/width{E}
|
def bulk = arch_defvw{}/width{E}
|
||||||
def VT = [bulk]E
|
def VT = [bulk]E
|
||||||
@for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
|
@for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
|
||||||
if (any_hom{M, x!=x}) return{1}
|
if (any_hom{M, x!=x}) return{1}
|
||||||
@ -72,7 +72,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
|
|||||||
|
|
||||||
|
|
||||||
# In-register bit table
|
# In-register bit table
|
||||||
def arch_vec{T} = [arch_defvw/width{T}]T
|
def arch_vec{T} = [arch_defvw{}/width{T}]T
|
||||||
def TI = i8 # Table values
|
def TI = i8 # Table values
|
||||||
def VI = arch_vec{TI}
|
def VI = arch_vec{TI}
|
||||||
def simd_bittab = hasarch{'SSSE3'} or hasarch{'AARCH64'}
|
def simd_bittab = hasarch{'SSSE3'} or hasarch{'AARCH64'}
|
||||||
@ -312,7 +312,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
|
|||||||
min1:= undefined{E}
|
min1:= undefined{E}
|
||||||
max1:= undefined{E}
|
max1:= undefined{E}
|
||||||
if (has_simd) {
|
if (has_simd) {
|
||||||
def bulk = arch_defvw/width{E}
|
def bulk = arch_defvw{}/width{E}
|
||||||
def VT = [bulk]E
|
def VT = [bulk]E
|
||||||
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
|
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
|
||||||
def min_a = assoc_accumulator{__min, unr, VT}
|
def min_a = assoc_accumulator{__min, unr, VT}
|
||||||
|
|||||||
@ -55,7 +55,7 @@ def vptr{VT=[_]E, ptr} = tup{VT, *E~~ptr}
|
|||||||
fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = {
|
fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void = {
|
||||||
def cyc = cast_i{TI,cyc0}
|
def cyc = cast_i{TI,cyc0}
|
||||||
if (has_simd) {
|
if (has_simd) {
|
||||||
def bulk = arch_defvw / width{TI}
|
def bulk = arch_defvw{} / width{TI}
|
||||||
def VT = [bulk]TI
|
def VT = [bulk]TI
|
||||||
@for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
|
@for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
|
||||||
dst = blend_hom{src, src + VT**cyc, src < VT**0}
|
dst = blend_hom{src, src + VT**cyc, src < VT**0}
|
||||||
@ -161,7 +161,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
|
|||||||
|
|
||||||
fn select_rows_widen{sh}(src:*void, dst:*void, max:ux, n:ux) : ux = {
|
fn select_rows_widen{sh}(src:*void, dst:*void, max:ux, n:ux) : ux = {
|
||||||
if (sh != 0) {
|
if (sh != 0) {
|
||||||
def bulk = (arch_defvw/8) >> sh
|
def bulk = (arch_defvw{}/8) >> sh
|
||||||
def WV = [bulk<<sh]u8
|
def WV = [bulk<<sh]u8
|
||||||
def V = [bulk]u8
|
def V = [bulk]u8
|
||||||
if ((n<<sh) > inds_buf_max) return{0}
|
if ((n<<sh) > inds_buf_max) return{0}
|
||||||
@ -257,7 +257,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
|||||||
}
|
}
|
||||||
1
|
1
|
||||||
}
|
}
|
||||||
def select_fn{TI, TD} = select_fn{arch_defvw, TI, TD}
|
def select_fn{TI, TD} = select_fn{arch_defvw{}, TI, TD}
|
||||||
|
|
||||||
export_tab{'si_select_tab', join{table{select_fn,
|
export_tab{'si_select_tab', join{table{select_fn,
|
||||||
tup{i8, i16, i32}, # indices
|
tup{i8, i16, i32}, # indices
|
||||||
@ -269,7 +269,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
|||||||
(if(has_sel) {
|
(if(has_sel) {
|
||||||
fn simd_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
fn simd_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||||
def TI = i8
|
def TI = i8
|
||||||
def VI = [arch_defvw/8]TI
|
def VI = [arch_defvw{}/8]TI
|
||||||
def VU = ty_u{VI}
|
def VU = ty_u{VI}
|
||||||
w:= *VI ~~ w0
|
w:= *VI ~~ w0
|
||||||
xlf:= VI**cast_i{TI, xl}
|
xlf:= VI**cast_i{TI, xl}
|
||||||
@ -305,7 +305,7 @@ export_tab{'si_select_tab', join{table{select_fn,
|
|||||||
export{'simd_select_bool128', simd_select_bool128}
|
export{'simd_select_bool128', simd_select_bool128}
|
||||||
|
|
||||||
fn si_select_rows_8bit(inds:*u8, indn:ux, src:*void, dst:*void, rows:ux) : void = { # leaves zeroes in result cells above indn
|
fn si_select_rows_8bit(inds:*u8, indn:ux, src:*void, dst:*void, rows:ux) : void = { # leaves zeroes in result cells above indn
|
||||||
def bulk = arch_defvw / 8
|
def bulk = arch_defvw{} / 8
|
||||||
def V = [bulk]u8
|
def V = [bulk]u8
|
||||||
def lut = shuf_u8bits{inds, indn}
|
def lut = shuf_u8bits{inds, indn}
|
||||||
@for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
|
@for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
|
||||||
|
|||||||
@ -59,7 +59,7 @@ def for_special_buffered{r:*T, write_len}{vars,begin,sum,iter} = {
|
|||||||
} else {
|
} else {
|
||||||
if (has_simd) {
|
if (has_simd) {
|
||||||
def bufw = bufn * tw
|
def bufw = bufn * tw
|
||||||
def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw;
|
def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw{}} / tw;
|
||||||
def R = [vc]T
|
def R = [vc]T
|
||||||
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc }
|
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{r0, load{R, buf}}; r0+=vc; buf+=vc }
|
||||||
assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack
|
assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack
|
||||||
|
|||||||
@ -147,12 +147,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
|
|||||||
} else assert{0}
|
} else assert{0}
|
||||||
}
|
}
|
||||||
|
|
||||||
export{'si_squeeze_i8', squeeze{arch_defvw, i8, 0, 0}}
|
export{'si_squeeze_i8', squeeze{arch_defvw{}, i8, 0, 0}}
|
||||||
export{'si_squeeze_i16', squeeze{arch_defvw, i16, 0, 0}}
|
export{'si_squeeze_i16', squeeze{arch_defvw{}, i16, 0, 0}}
|
||||||
export{'si_squeeze_i32', squeeze{arch_defvw, i32, 0, 0}}
|
export{'si_squeeze_i32', squeeze{arch_defvw{}, i32, 0, 0}}
|
||||||
export{'si_squeeze_f64', squeeze{arch_defvw, f64, 0, 0}}
|
export{'si_squeeze_f64', squeeze{arch_defvw{}, f64, 0, 0}}
|
||||||
export{'si_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}}
|
export{'si_squeeze_numB', squeeze{arch_defvw{}, f64, 0, 1}}
|
||||||
|
|
||||||
export{'si_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}}
|
export{'si_squeeze_c16', squeeze{arch_defvw{}, u16, 1, 0}}
|
||||||
export{'si_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}}
|
export{'si_squeeze_c32', squeeze{arch_defvw{}, u32, 1, 0}}
|
||||||
export{'si_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}
|
export{'si_squeeze_chrB', squeeze{arch_defvw{}, u64, 1, 1}}
|
||||||
|
|||||||
@ -53,12 +53,12 @@ def store1of2{a:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) {
|
|||||||
}
|
}
|
||||||
def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
|
def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
|
||||||
def store_k{VT, dst, x, l, h} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
|
def store_k{VT, dst, x, l, h} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
|
||||||
def load_k {VT, src, l, w if width{VT} < arch_defvw} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
|
def load_k {VT, src, l, w if width{VT} < arch_defvw{}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
|
||||||
def store_k{VT, dst, x, l, h if width{VT} < arch_defvw} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
|
def store_k{VT, dst, x, l, h if width{VT} < arch_defvw{}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
|
||||||
|
|
||||||
# Transpose kernel of size kw,kh in size w,h array
|
# Transpose kernel of size kw,kh in size w,h array
|
||||||
def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = {
|
def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = {
|
||||||
def n = (kw*kh*width{T}) / arch_defvw # Number of vectors
|
def n = (kw*kh*width{T}) / arch_defvw{} # Number of vectors
|
||||||
def xvs = load_k{[kw]T, src, n, w}
|
def xvs = load_k{[kw]T, src, n, w}
|
||||||
def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n
|
def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n
|
||||||
def rvs = if (n==kw) xt else halved_pass{n,xt} # To kh by kh for packed square
|
def rvs = if (n==kw) xt else halved_pass{n,xt} # To kh by kh for packed square
|
||||||
@ -89,7 +89,7 @@ def kernel_part{part_w}{src:*T, dst:*T, kw, kh, w, h} = {
|
|||||||
def kernel = kernel_part{0}
|
def kernel = kernel_part{0}
|
||||||
|
|
||||||
def kernel_part_h{part_h}{src:*T==i8, dst:*T, kw==16, kh==16, w, h} = {
|
def kernel_part_h{part_h}{src:*T==i8, dst:*T, kw==16, kh==16, w, h} = {
|
||||||
def n = (kw*kh*width{T}) / arch_defvw
|
def n = (kw*kh*width{T}) / arch_defvw{}
|
||||||
def VT = [kw]T
|
def VT = [kw]T
|
||||||
off := part_h - kh/2
|
off := part_h - kh/2
|
||||||
def xvs = @unroll (i to n) { s := src + i*w; load2{*VT~~s, *VT~~(s+off*w)} }
|
def xvs = @unroll (i to n) { s := src + i*w; load2{*VT~~s, *VT~~(s+off*w)} }
|
||||||
@ -221,13 +221,13 @@ fn interleave{T}(r0:*void, x0:*void, x1:*void, n:u64) : void = {
|
|||||||
}
|
}
|
||||||
# SIMD implementations
|
# SIMD implementations
|
||||||
def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} or width{T}>=32 or hasarch{'SSSE3'})} = {
|
def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} or width{T}>=32 or hasarch{'SSSE3'})} = {
|
||||||
def l = arch_defvw / width{T}
|
def l = arch_defvw{} / width{T}
|
||||||
def V = [l]T
|
def V = [l]T
|
||||||
rv0 := *V~~r0; rv1 := *V~~r1; xv := *V~~xp
|
rv0 := *V~~r0; rv1 := *V~~r1; xv := *V~~xp
|
||||||
nv := n / l
|
nv := n / l
|
||||||
def uz = if (not hasarch{'X86_64'}) unzip else ({...xs} => {
|
def uz = if (not hasarch{'X86_64'}) unzip else ({...xs} => {
|
||||||
def reinterpret{V, xs if ktup{xs}} = each{~~{V,.}, xs}
|
def reinterpret{V, xs if ktup{xs}} = each{~~{V,.}, xs}
|
||||||
def q = tr_quads{arch_defvw/128}
|
def q = tr_quads{arch_defvw{}/128}
|
||||||
def k = flat_table{+, iota{2}, 2 * iota{64 / width{T}}}
|
def k = flat_table{+, iota{2}, 2 * iota{64 / width{T}}}
|
||||||
def px = each{shuf{., k}, xs}
|
def px = each{shuf{., k}, xs}
|
||||||
V~~each{q, zip128{...re_el{u64,V}~~px}}
|
V~~each{q, zip128{...re_el{u64,V}~~px}}
|
||||||
@ -244,11 +244,11 @@ def uninterleave{r0:*T, r1:*T, xp:*T, n if has_simd and (not hasarch{'X86_64'} o
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = {
|
fn interleave{T if has_simd}(r:*void, x0:*void, x1:*void, n:u64) : void = {
|
||||||
def l = arch_defvw / width{T}
|
def l = arch_defvw{} / width{T}
|
||||||
def V = [l]T
|
def V = [l]T
|
||||||
xv0 := *V~~x0; xv1 := *V~~x1; rv := *V~~r
|
xv0 := *V~~x0; xv1 := *V~~x1; rv := *V~~r
|
||||||
nv := n / l
|
nv := n / l
|
||||||
def q = tr_quads{arch_defvw/128}
|
def q = tr_quads{arch_defvw{}/128}
|
||||||
@for (x0 in xv0, x1 in xv1 over i to nv) {
|
@for (x0 in xv0, x1 in xv1 over i to nv) {
|
||||||
each{store{rv+2*i, ., .}, iota{2}, zip128{q{x0},q{x1}}}
|
each{store{rv+2*i, ., .}, iota{2}, zip128{q{x0},q{x1}}}
|
||||||
}
|
}
|
||||||
@ -294,7 +294,7 @@ def loop_fixed_width{xp, rp, h, k, st, kern} = {
|
|||||||
|
|
||||||
# Transpose a contiguous kernel of width w*p from x to r with stride rst
|
# Transpose a contiguous kernel of width w*p from x to r with stride rst
|
||||||
def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = {
|
def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = {
|
||||||
def h = arch_defvw / 8
|
def h = arch_defvw{} / 8
|
||||||
def ih = iota{h}; def iw = iota{w}
|
def ih = iota{h}; def iw = iota{w}
|
||||||
def I = [h]u8; def V = I
|
def I = [h]u8; def V = I
|
||||||
def e = width{T} / 8
|
def e = width{T} / 8
|
||||||
@ -324,7 +324,7 @@ def modular_kernel{w,p if w%2==1 and 2%p==0}{xp:*T, rp0:*T, rst:(u64)} = {
|
|||||||
each{perm_store, xs}
|
each{perm_store, xs}
|
||||||
}
|
}
|
||||||
def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = {
|
def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = {
|
||||||
def h = arch_defvw / 8
|
def h = arch_defvw{} / 8
|
||||||
def V = [h]u8; def U = n_h{V}
|
def V = [h]u8; def U = n_h{V}
|
||||||
def ih = iota{h}%16; def e = width{T} / 8
|
def ih = iota{h}%16; def e = width{T} / 8
|
||||||
# Permutation to unzip by 4 within each lane
|
# Permutation to unzip by 4 within each lane
|
||||||
@ -343,7 +343,7 @@ def modular_kernel{2,2}{xp:*T, rp0:*T, rst:(u64)} = {
|
|||||||
}
|
}
|
||||||
def transpose_fixed_width{rp:*T, xp:*T, wk, h, hs} = {
|
def transpose_fixed_width{rp:*T, xp:*T, wk, h, hs} = {
|
||||||
def p = if (wk%2) 1 else 2; def w = wk/p
|
def p = if (wk%2) 1 else 2; def w = wk/p
|
||||||
def vl = arch_defvw / (p*width{T})
|
def vl = arch_defvw{} / (p*width{T})
|
||||||
loop_fixed_width{xp, rp, h, vl, wk, modular_kernel{w,p}{., ., hs}}
|
loop_fixed_width{xp, rp, h, vl, wk, modular_kernel{w,p}{., ., hs}}
|
||||||
}
|
}
|
||||||
def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = {
|
def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = {
|
||||||
@ -353,7 +353,7 @@ def transpose_fixed_width{rp:*T, xp:*T, 2, h, hs} = {
|
|||||||
# Transpose a kernel of height w*p from x with stride xst to contiguous r
|
# Transpose a kernel of height w*p from x with stride xst to contiguous r
|
||||||
# w and h are named for the result, not argument, to match modular_kernel
|
# w and h are named for the result, not argument, to match modular_kernel
|
||||||
def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = {
|
def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = {
|
||||||
def h = arch_defvw / 8
|
def h = arch_defvw{} / 8
|
||||||
def ih = iota{h}; def iw = iota{w}
|
def ih = iota{h}; def iw = iota{w}
|
||||||
def I = [h]u8; def V = I
|
def I = [h]u8; def V = I
|
||||||
def e = width{T} / 8
|
def e = width{T} / 8
|
||||||
@ -387,18 +387,18 @@ def modular_kernel_rev{w,p if w%2==1 and 2%p==0}{xp0:*T, rp:*T, xst:(u64)} = {
|
|||||||
each{store{*V~~rp, ., .}, iw, select{xs, h/e*iw % w}}
|
each{store{*V~~rp, ., .}, iw, select{xs, h/e*iw % w}}
|
||||||
}
|
}
|
||||||
def modular_kernel_rev{2,2}{xp0:*T, rp:*T, xst:(u64)} = {
|
def modular_kernel_rev{2,2}{xp0:*T, rp:*T, xst:(u64)} = {
|
||||||
def V = [arch_defvw / width{T}]T; def U = n_h{V}
|
def V = [arch_defvw{} / width{T}]T; def U = n_h{V}
|
||||||
xl := @unroll (i to 2) {
|
xl := @unroll (i to 2) {
|
||||||
xp := xp0 + i*xst
|
xp := xp0 + i*xst
|
||||||
x := load2{*U~~xp, *U~~(xp + 2*xst)}
|
x := load2{*U~~xp, *U~~(xp + 2*xst)}
|
||||||
if (arch_defvw==128) x else shuf{[8]u32, x, tr_iota{1,2,0}}
|
if (arch_defvw{}==128) x else shuf{[8]u32, x, tr_iota{1,2,0}}
|
||||||
}
|
}
|
||||||
xs := unpack_typed{...unpack_typed{...xl}}
|
xs := unpack_typed{...unpack_typed{...xl}}
|
||||||
each{store{*V~~rp, ., .}, iota{2}, each{~~{V,.},xs}}
|
each{store{*V~~rp, ., .}, iota{2}, each{~~{V,.},xs}}
|
||||||
}
|
}
|
||||||
def transpose_fixed_height{rp:*T, xp:*T, w, ws, hk} = {
|
def transpose_fixed_height{rp:*T, xp:*T, w, ws, hk} = {
|
||||||
def p = if (hk%2) 1 else 2; def h = hk/p
|
def p = if (hk%2) 1 else 2; def h = hk/p
|
||||||
def vl = arch_defvw / (p*width{T})
|
def vl = arch_defvw{} / (p*width{T})
|
||||||
loop_fixed_height{xp, rp, w, vl, hk, modular_kernel_rev{h,p}{., ., ws}}
|
loop_fixed_height{xp, rp, w, vl, hk, modular_kernel_rev{h,p}{., ., ws}}
|
||||||
}
|
}
|
||||||
def transpose_fixed_height{rp:*T, xp:*T, w, ws, 2} = {
|
def transpose_fixed_height{rp:*T, xp:*T, w, ws, 2} = {
|
||||||
@ -409,7 +409,7 @@ fn transpose{T, {k, kh}}(r0:*void, x0:*void, w:u64, h:u64, ws:u64, hs:u64) : voi
|
|||||||
rp:*T = *T~~r0
|
rp:*T = *T~~r0
|
||||||
xp:*T = *T~~x0
|
xp:*T = *T~~x0
|
||||||
def wT = width{T}
|
def wT = width{T}
|
||||||
def vl = arch_defvw / wT
|
def vl = arch_defvw{} / wT
|
||||||
# Transposes with code dedicated to a particular width or height
|
# Transposes with code dedicated to a particular width or height
|
||||||
def try_fixed_dim{tr, l, lst, nl, l_max} = {
|
def try_fixed_dim{tr, l, lst, nl, l_max} = {
|
||||||
def incl{l} = if (k>4) 1 else l!=4
|
def incl{l} = if (k>4) 1 else l!=4
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user