From 90863851b7756ea8fff278819c83ba7c51987f2d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 25 May 2024 17:47:28 -0400 Subject: [PATCH] Use new Singeli type destructuring (no change to generated C) --- build/singeliSubmodule | 2 +- src/singeli/src/avx.singeli | 34 ++++----- src/singeli/src/avx2.singeli | 58 +++++++-------- src/singeli/src/avx512.singeli | 25 +++---- src/singeli/src/base.singeli | 89 +++++++++++------------ src/singeli/src/bins.singeli | 12 ++-- src/singeli/src/bitops.singeli | 106 +++++++++++++++------------- src/singeli/src/bmi2.singeli | 8 +-- src/singeli/src/cbqnDefs.singeli | 14 ++-- src/singeli/src/cmp.singeli | 27 ++++--- src/singeli/src/copy.singeli | 13 ++-- src/singeli/src/count.singeli | 2 +- src/singeli/src/dyarith.singeli | 24 +++---- src/singeli/src/hashtab.singeli | 4 +- src/singeli/src/mask.singeli | 39 +++++----- src/singeli/src/neon.singeli | 72 +++++++++---------- src/singeli/src/replicate.singeli | 32 ++++----- src/singeli/src/scan.singeli | 19 +++-- src/singeli/src/scan_common.singeli | 10 +-- src/singeli/src/search.singeli | 20 +++--- src/singeli/src/slash.singeli | 10 +-- src/singeli/src/squeeze.singeli | 55 +++++++-------- src/singeli/src/sse.singeli | 20 +++--- src/singeli/src/sse2.singeli | 42 +++++------ src/singeli/src/transpose.singeli | 11 ++- src/singeli/src/vecfold.singeli | 6 +- 26 files changed, 368 insertions(+), 386 deletions(-) diff --git a/build/singeliSubmodule b/build/singeliSubmodule index ce6ef5d0..e3eb3d56 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit ce6ef5d06d35777f0016bbfe0c6c1cf6a9c1b48e +Subproject commit e3eb3d56b52604b43b07493f69dedfc43c83966d diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index bb0d9337..18c136d4 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -1,37 +1,37 @@ # compact casting for the annoying intrinsic type system -def v2i{x:T if w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x +def v2i{x:T=[_]E if w256{T}} = if(isint{E}) x else [32]u8 ~~ x def v2f{x:T if w256{T}} = [8]f32 ~~ x def v2d{x:T if w256{T}} = [4]f64 ~~ x -def undefPromote{T, x:X if w128{X} and w256{T} and eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} +def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store -def loadLow{ptr:P, w if w256{eltype{P}} and w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} -def loadLow{ptr:P, w if w256{eltype{P}} and w==256} = load{ptr} +def loadLow{ptr:*T, w if w256{T} and w<=128} = undefPromote{T, loadLow{*n_h{T} ~~ ptr, w}} +def loadLow{ptr:*T, w if w256{T} and w==256} = load{ptr} -def storeLow{ptr:P, w, x:T if w256{T} and w<=128} = storeLow{ptr, w, half{x, 0}} -def storeLow{ptr:P, w, x:T if w256{T} and w==256} = store{*T~~ptr, 0, x} +def storeLow{ptr:*T, w, x:X if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}} +def storeLow{ptr:*T, w, x:X if w256{X} and w==256} = store{*X~~ptr, 0, x} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n} -def unord{a:T,b:T if T==[8]f32} = f32cmpAVX{a,b,3} -def unord{a:T,b:T if T==[4]f64} = f64cmpAVX{a,b,3} +def unord{a:T, b:T==[8]f32} = f32cmpAVX{a,b,3} +def unord{a:T, b:T==[4]f64} = f64cmpAVX{a,b,3} # f32 arith -def rsqrtE{a:([8]f32)} = emit{[8]f32, '_mm256_rsqrt_ps', a} -def rcpE{a:([8]f32)} = emit{[8]f32, '_mm256_rcp_ps', a} +def rsqrtE{a:T==[8]f32} = emit{T, '_mm256_rsqrt_ps', a} +def rcpE{a:T==[8]f32} = emit{T, '_mm256_rcp_ps', a} # conversion def half{x:T, i if w256{T} and knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} -def half{x:T, (0) if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} +def half{x:T, 0 if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} def pair{a:T,b:T if w128{T}} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} -def widen{T==[4]f64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x} -def widen{T==[4]f64, x:X if X==[4]f32} = emit{T, '_mm256_cvtps_pd', x} -def widen{T==[4]f64, x:X if w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def widen{T, x:X if w256{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}} +def widen{T==[4]f64, x:([4]i32)} = emit{T, '_mm256_cvtepi32_pd', x} +def widen{T==[4]f64, x:([4]f32)} = emit{T, '_mm256_cvtps_pd', x} +def widen{T==[4]f64, x:X=[_]U if w128i{X} and width{U}<32} = widen{T, widen{[4]i32, x}} +def widen{T=[k]_, x:X=[l]_ if w256{X} and l>k} = widen{T, half{x,0}} # structural operations @@ -50,5 +50,5 @@ def homMask{x:T if w256{T}} = topMask{x} def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff -def topAny{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} != 0 -def topAll{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} == (1<=32} = topMask{x} != 0 +def topAll{x:T=[k]E if w256i{T} and width{E}>=32} = topMask{x} == (1<=32} = topMaskStore{p,m,v} -def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} -def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}} +def topMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:*P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v} +def homMaskStoreF{p:*T, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}}} = store{p, 0, homBlend{load{p}, v, m}} # mask stuff def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x} @@ -57,8 +57,8 @@ def topMask{x:T if w256{T, 16}} = { def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}} def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff -def topAny{x:T if w256i{T}} = topMask{x} != 0 -def topAll{x:T if w256i{T}} = topMask{x} == (1< emit{M, merge{'_cvtu',w,'_mask',l}, promote{W, a}}} } -def reinterpret{T, a:M if ismask{M} and width{T}==width{M}} = { +def reinterpret{T, a:M=[l==width{T}](u1)} = { re_mask{M, {l,w,W} => cast_i{T, emit{W, merge{'_cvtmask',l,'_u',w}, a}}} } -def maskStore{p:*V, m:M, v:V if ismask{M} and isvec{V} and vcount{M}==vcount{V}} = { +def maskStore{p:*V, m:[l](u1), v:V=[l]_} = { emit{void, merge{pref{V}, 'mask_storeu_', suf{V}}, p, m, v} } -def topMaskReg{x:V} = emit{[vcount{V}]u1, merge{pref{V},'mov',suf{V},'_mask'}, x} -def topMask{x:T if isvec{T} and 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x} -def homMask{x:T if isvec{T} and 512==width{T}} = topMask{x} +def topMaskReg{x:V=[k]_} = emit{[k]u1, merge{pref{V},'mov',suf{V},'_mask'}, x} +def topMask{x:V=[k]_ if 512==width{V}} = ty_u{k}~~topMaskReg{x} +def homMask{x:V=[_]_ if 512==width{V}} = topMask{x} -def maskToHom{T, x:M if ismask{M} and isvec{T} and vcount{M}==vcount{T}} = { - emit{T, merge{pref{T},'movm_',suf{T}}, x} -} +def maskToHom{V=[l]_, x:[l](u1)} = emit{V, merge{pref{V},'movm_',suf{V}}, x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 62c92cd4..45aa9fe4 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -28,16 +28,16 @@ def andnot{a, b if anyNum{a} and anyNum{b}} = a & ~b -def load{p:P, n if isvec{eltype{P}}} = assert{0} -def store{p:P, n, v if isvec{eltype{P}}} = assert{0} -def load{p:P if isptr{P}} = load{p, 0} -# def store{p:P, v if isptr{P}} = store{p, 0, v} -def loadu{p:T if isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} -def storeu{p:T, v:(eltype{T}) if isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} -def loadu{p:T if issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p} -def storeu{p:T, v:(eltype{T}) if issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}} -def loadu{p:T if elwidth{T}==8} = load{p} -def storeu{p:T, v:(eltype{T}) if elwidth{T}==8} = store{p, v} +def load {p:*[_]_, n } = assert{0} +def store{p:*[_]_, n, v} = assert{0} +def load{p:*_} = load{p, 0} +# def store{p:*_, v} = store{p, 0, v} +def loadu {p:*T if isunsigned{T}} = emit{T, merge{'loadu_u', fmtnat{width{T}}}, p} +def storeu{p:*T, v:T if isunsigned{T}} = emit{void, merge{'storeu_u',fmtnat{width{T}}}, p, v} +def loadu {p:*T if issigned{T}} = loadu {*ty_u{T} ~~ p} +def storeu{p:*T, v:T if issigned{T}} = storeu{*ty_u{T} ~~ p, ty_u{v}} +def loadu {p:*T if width{T}==8} = load{p} +def storeu{p:*T, v:T if width{T}==8} = store{p, v} def reinterpret{T, x:T} = x @@ -49,8 +49,8 @@ def exportT{name, fs} = { v:*oneType{fs} = fs; export{name, v} } def rare{x if knum{x}} = x def rare{x:(u1)} = emit{u1, '__builtin_expect', x, 0} def assert{c, ...msg} = { if (not same{c,1}) { show{...msg}; 0{} } } -def assert{(0)} = assert{0, 'failed assertion'} -def assert{(1)} = 1 +def assert{0} = assert{0, 'failed assertion'} +def assert{1} = 1 def unreachable{} = emit{void, 'si_unreachable'} def assert{x:(u1)} = { if (not x) emit{void, 'si_unreachable'} } @@ -62,11 +62,8 @@ def oneVal{{h, ...t}} = { def oneVal{{}} = {} def oneType{x} = oneVal{each{type, x}} -def anyNum{x} = knum{x} -def anyNum{x:T} = isprim{T} - -def anyInt{x} = knum{x} and (x>>0) == x -def anyInt{x:T} = isint{T} +def anyNum = match { {x:T}=>isprim{T}; {x} => knum{x} } +def anyInt = match { {x:T}=>isint{T} ; {x} => knum{x} and (x>>0) == x } # vector width/type checks def w64 {T} = isvec{T} and width{T}==64; def w64 {T,w} = w64{T} and elwidth{T}==w @@ -74,10 +71,9 @@ def w128{T} = isvec{T} and width{T}==128; def w128{T,w} = w128{T} and elwidth{T} def w256{T} = isvec{T} and width{T}==256; def w256{T,w} = w256{T} and elwidth{T}==w # width+type checks -def genchk{B, F} = { - def r{T } = B{T} and F{eltype{T}} - def r{T,w} = B{T} and F{eltype{T}} and elwidth{T}==w - r +def genchk{B, F} = match { + {V=[_]T } => B{V} and F{T} + {V=[_]T, w} => B{V} and F{T} and width{T}==w } def w256i = genchk{w256, isint}; def w128i = genchk{w128, isint}; def w64i = genchk{w64, isint} def w256s = genchk{w256, issigned}; def w128s = genchk{w128, issigned}; def w64s = genchk{w64, issigned} @@ -104,7 +100,7 @@ def re_el{E, x:V} = re_el{E,V} ~~ x local def qualChange{q} = { def f{w if knum{w}} = primtype{q, w} def f{T if isprim{T}} = primtype{q, width{T}} - def f{T if isvec{T}} = re_el{f{eltype{T}}, T} + def f{V=[_]T} = re_el{f{T}, V} def f{x:T} = f{T}~~x } def ty_u = qualChange{'u'} @@ -115,14 +111,14 @@ def w_n{T, w if isprim{T}} = primtype{quality{T}, w} def w_d{T if isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width def w_h{T if isprim{T}} = to_w{T, width{T}/2} -def n_d{T if isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count -def n_h{T if isvec{T}} = [vcount{T}/2](eltype{T}) +def n_d{[k]T} = [k*2]T # double/halve vector count +def n_h{[k]T} = [k/2]T -def el_d{T if isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count -def el_h{T if isvec{T}} = [vcount{T}](w_h{eltype{T}}) +def el_d{[k]T} = [k](w_d{T}) # double/halve element width, preserving count +def el_h{[k]T} = [k](w_h{T}) -def el_m{T if isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width -def el_s{T if isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x +def el_m{V=[_]T} = re_el{w_d{T}, V}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width +def el_s{V=[_]T} = re_el{w_h{T}, V}; def el_s{x:T} = re_el{T}~~x # type stats def minvalue{T if isunsigned{T}} = 0 @@ -153,7 +149,7 @@ def has_simd = hasarch{'X86_64'} or hasarch{'AARCH64'} def fast_BMI2{} = hasarch{'BMI2'} and not SLOW_PDEP # test if vector has a specific width & element type -def lvec{T, n, w} = isvec{T} and vcount{T}==n and elwidth{T}==w +def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 } # base cases def { @@ -169,11 +165,12 @@ def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeate def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX def homMask{...vs if length{vs}>1} = { def n = length{vs} - def T = oneType{vs} - def RT = ty_u{max{8,vcount{T}*n}} - def a = promote{RT, homMask{...slice{vs,0,n/2}}} - def b = promote{RT, homMask{...slice{vs,n/2}}} - (b<<(n/2 * vcount{T})) | a + def [k]_ = oneType{vs} + def RT = ty_u{max{8,k*n}} + def sl{...a} = promote{RT, homMask{...slice{vs,...a}}} + def h = n/2 + def a = sl{0,h} + (sl{h} << (h * k)) | a } if_inline (hasarch{'X86_64'}) { @@ -195,15 +192,15 @@ def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} def packQ{{a, b}} = packQ{a, b} def pair{{a, b}} = pair{a, b} -def widen{T, x:X if T==X} = x -def narrow{T, x:X if T==eltype{X}} = x -def undefPromote{T, x:X if T==X} = x -def cvt{T, x:X if T==eltype{X}} = x +def widen{T, x:T} = x +def narrow{T, x:[_]T} = x +def undefPromote{T, x:T} = x +def cvt{T, x:[_]T} = x -def broadcast{T, v if isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} -def make{T, ...xs if isvec{T}} = vec_make{T, ...xs} -def iota{T if isvec{T}} = make{T, ...iota{vcount{T}}} -def absu{a:T if isvec{T}} = ty_u{abs{a}} +def broadcast{V=[_]T, v} = vec_broadcast{V, promote{T,v}} +def make{V=[_]_, ...xs} = vec_make{V, ...xs} +def iota{V=[k]_} = make{V, ...iota{k}} +def absu{a:[_]_} = ty_u{abs{a}} def floor = __floor def ceil = __ceil @@ -246,13 +243,13 @@ def tail{n,x} = x & ((1< if(ktup{x}) length{x} else 0, args}} + def l = tree_fold{max, each{{x} => if(istup{x}) length{x} else 0, args}} each{F, ...each{{x} => if (istup{x}) x else l**x, args}} } diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 8273f5d9..bfe88de0 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -41,21 +41,21 @@ fn max_scan{T, up}(x:*T, len:u64) : void = { def getsel{...x} = assert{'shuffling not supported', show{...x}} if_inline (hasarch{'AVX2'}) { - def getsel{h:H if lvec{H, 16, 8}} = { + def getsel{h:H=[16]T if width{T}==8} = { sel{H, pair{h,h}, .} } - def getsel{v:V if lvec{V, 32, 8}} = { + def getsel{v:V=[vl==32]T if width{T}==8} = { def H = n_h{V} - vtop := V**(vcount{V}/2) + vtop := V**(vl/2) hs := each{shuf{[4]u64, v, .}, tup{4b3232, 4b1010}} {i} => homBlend{...each{sel{H,.,i}, hs}, V~~i>6}>>(n&63)) & 1) != 0 } -def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) -def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) -def b_getBatchLo{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatchLo{sz, x:(*u64), n:(ux)} = match (sz) { + {2} => (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) + {4} => (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) + {sz if sz>=8} => load{*ty_u{sz}~~x, n} +} -def b_getBatch{sz, x:(*u64), n:(ux) if sz==2} = b_getBatchLo{sz, x, n} & 3 -def b_getBatch{sz, x:(*u64), n:(ux) if sz==4} = b_getBatchLo{sz, x, n} & 15 -def b_getBatch{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n} +def b_getBatch{sz, x:(*u64), n:(ux)} = match (sz) { + {2} => b_getBatchLo{sz, x, n} & 3 + {4} => b_getBatchLo{sz, x, n} & 15 + {sz if sz>=8} => load{*ty_u{sz}~~x, n} +} def b_set{x:(*u64), n:(ux), v:(u1)} = { @@ -20,39 +24,40 @@ def b_set{x:(*u64), n:(ux), v:(u1)} = { else store{x,n>>6,p & ~m} } -def b_setBatch{sz, x:(*u64), n:(ux), v} = { - vc:u64 = promote{u64,v} - am:u64 = 64/sz - w:u64 = load{x,n/am} - sh:u64 = (n&(am-1)) * sz - w&= ~(ones{u64,sz}< { + x8:= *u8 ~~ x + + #w:u64 = cast_i{u64, load{x8,n/2}} + #sh:u64 = (n&1) * 4 + #w&= ~(15< store{*u8 ~~ x, n, cast_i{u8, v}} + {16} => store{*u16 ~~ x, n, cast_i{u16,v}} + {32} => store{*u32 ~~ x, n, cast_i{u32,v}} + {64} => store{ x, n, cast_i{u64,v}} + {sz} => { + vc:u64 = promote{u64,v} + am:u64 = 64/sz + w:u64 = load{x,n/am} + sh:u64 = (n&(am-1)) * sz + w&= ~(ones{u64,sz}<=8}} - andnz{b, make{[16]u8, 1<<(iota{16}&7)}} + b:= sel{T, T~~[8]u16**a, make{[16]i8, iota{16}>=8}} + andnz{b, make{T, 1<<(iota{16}&7)}} } def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = { - b:= [16]u8~~[8]u16**a - exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} - (exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0 + b:= T~~[8]u16**a + exp:= T~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100} + (exp & make{T, 1<<(iota{16}&7)}) != T**0 } -def spreadBits{T, a if vcount{T} <= elwidth{T} and quality{eltype{T}}=='u'} = { - b:= make{T, 1<>3))} >> (i&7) } def loaduBit{x:(*u64), i, n} = { - assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))} + assert{(n<58) | ((n==58 or n==60) & (i%n == 0))} loaduBitRaw{x, i} } def loaduBitTrunc{x:(*u64), i, n if knum{n}} = truncBits{n, loaduBit{x, i, n}} -def loadBatchBit{T, x:(*u64), is if ktup{is}} = { +def loadBatchBit{T, x:(*u64), {...is}} = { # def len = length{is} - # def count = vcount{T} + # def [count]_ = T # assert{count*len <= 64} # bits:= b_getBatchLo{count*len, x, select{is,0}} # @collect(i to len) spreadBits{T, truncBits{count, bits>>(i*count)}} diff --git a/src/singeli/src/bmi2.singeli b/src/singeli/src/bmi2.singeli index e26237ed..5609738a 100644 --- a/src/singeli/src/bmi2.singeli +++ b/src/singeli/src/bmi2.singeli @@ -1,4 +1,4 @@ -def pdep{x:T, m:T if T==u64} = emit{T, '_pdep_u64', x, m} -def pdep{x:T, m:T if T==u32} = emit{T, '_pdep_u32', x, m} -def pext{x:T, m:T if T==u64} = emit{T, '_pext_u64', x, m} -def pext{x:T, m:T if T==u32} = emit{T, '_pext_u32', x, m} +def pdep{x:T, m:T==u64} = emit{T, '_pdep_u64', x, m} +def pdep{x:T, m:T==u32} = emit{T, '_pdep_u32', x, m} +def pext{x:T, m:T==u64} = emit{T, '_pext_u64', x, m} +def pext{x:T, m:T==u32} = emit{T, '_pext_u32', x, m} diff --git a/src/singeli/src/cbqnDefs.singeli b/src/singeli/src/cbqnDefs.singeli index 962f1da4..8cd73af8 100644 --- a/src/singeli/src/cbqnDefs.singeli +++ b/src/singeli/src/cbqnDefs.singeli @@ -21,18 +21,14 @@ def cbqn_nspTag{} = emit{u64, '', 'NSP_TAG'} def cbqn_objTag{} = emit{u64, '', 'OBJ_TAG'} def cbqn_arrTag{} = emit{u64, '', 'ARR_TAG'} -def cbqn_elType{(u1 )} = 0 -def cbqn_elType{(i8 )} = 1 -def cbqn_elType{(i16)} = 2 -def cbqn_elType{(i32)} = 3 -def cbqn_elType{(f64)} = 4 -def cbqn_elType{(u8 )} = 5 -def cbqn_elType{(u16)} = 6 -def cbqn_elType{(u32)} = 7 +def cbqn_elType = match { + {(u1)} => 0; {(i8)} => 1; {(i16)} => 2; {(i32)} => 3; {(f64)} => 4 + {(u8)} => 5; {(u16)} => 6; {(u32)} => 7 +} def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'} def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len} def tfree{ptr} = emit{void, 'TFREE', ptr} def fmt_type{T} = merge{quality{T}, fmtnat{width{T}}} -def fmt_type{T if isptr{T}} = merge{'*',fmt_type{eltype{T}}} +def fmt_type{*T} = merge{'*',fmt_type{T}} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 0d58ae1c..2626c6c4 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -32,7 +32,7 @@ def pathAS{dst, len, T, op, x if issigned{T}} = { if (isNaN{xf}) { call{cmpIX, dst, len, x, op{0,1}}; return{}; } } } - if (~hasarch{'X86_64'}) nanchk{} + if (not hasarch{'X86_64'}) nanchk{} if (rare{promote{i64, xT}!=xi64}) { if (hasarch{'X86_64'}) nanchk{} fillbits{dst, len, op{0,xf}} @@ -40,9 +40,9 @@ def pathAS{dst, len, T, op, x if issigned{T}} = { xT } -def pathAS{dst, len, T, op, x if T==f64} = { +def pathAS{dst, len, T==f64, op, x} = { if (rare{~q_f64{x}}) { - if (~eqne{op}) if (~q_chr{x}) cmp_err{x} + if (not eqne{op} and ~q_chr{x}) cmp_err{x} fillbits{dst, len, op{0,1}, x} } from_B{T,x} @@ -50,7 +50,7 @@ def pathAS{dst, len, T, op, x if T==f64} = { def pathAS{dst, len, T, op, x if isunsigned{T}} = { if (rare{~q_chr{x}}) { - if (~eqne{op}) if (~q_f64{x}) cmp_err{x} + if (not eqne{op} and ~q_f64{x}) cmp_err{x} fillbits{dst, len, op{1,0}, x} } xc32:u32 = from_B{u32,x} @@ -60,11 +60,10 @@ def pathAS{dst, len, T, op, x if isunsigned{T}} = { -def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = { - def bulk = vcount{VT}*unr +def any2bit{VT=[k]T, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = { + def bulk = k*unr xi:ux = 0 - def T = eltype{VT} def op = match (op0) { {_ if not hasarch{'X86_64'} or hasarch{'AVX512F'}} => op0 {(__le) if issigned{T}} => __gt @@ -81,15 +80,15 @@ def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = { xi+= unr } } -fn aa2bit{VT, unr, op}(dst:*u64, wr:*void, xr:*void, len:ux) : void = { - wv:= *VT~~wr; ws:= *eltype{VT}~~wr - xv:= *VT~~xr; xs:= *eltype{VT}~~xr +fn aa2bit{VT=[_]T, unr, op}(dst:*u64, wr:*void, xr:*void, len:ux) : void = { + wv:= *VT~~wr; ws:= *T~~wr + xv:= *VT~~xr; xs:= *T~~xr any2bit{VT, unr, op, load{ws,.}, load{wv,.}, load{xs,.}, load{xv,.}, dst, len} } -fn as2bit{VT, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = { - wv:= *VT~~wr; ws:= *eltype{VT}~~wr - xv:= VT**pathAS{dst, len, eltype{VT}, op, x} +fn as2bit{VT=[_]T, unr, op}(dst:*u64, wr:*void, x:u64, len:ux) : void = { + wv:= *VT~~wr; ws:= *T~~wr + xv:= VT**pathAS{dst, len, T, op, x} any2bit{VT, unr, op, load{ws,.}, load{wv,.}, {i}=>x, {i}=>xv, dst, len} } @@ -120,7 +119,7 @@ def table{aa, F, w, G_bit, G_vec} = { tup{ G_bit{}, ...each{{E}=>{ - def E2 = (if (aa and isunsigned{E} and ((same{F,__eq} | same{F,__ne}) | (E==u32))) {ty_s{E}} else E) + def E2 = (if (aa and isunsigned{E} and (eqne{F} or E==u32)) {ty_s{E}} else E) def bulk = w/width{E2} def unr = if (hasarch{'X86_64'} and width{E2}==16) 2 else max{8/bulk,1} G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, unr}} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index a4bb3e9a..2ea55932 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -3,13 +3,12 @@ include './mask' include './cbqnDefs' include './bitops' -def copyFromBits{T, loadFn, rp, l:(u64)} = { - def bulk = vcount{T} - def TU = ty_u{T} +def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = { + def U = ty_u{V} - @maskedLoop{bulk}(sr in tup{'g',*eltype{T}~~rp} over i to l) { - x:= loadFn{TU, i} - sr{T~~(x & TU ~~ T**1)} + @maskedLoop{bulk}(sr in tup{'g',*T~~rp} over i to l) { + x:= loadFn{U, i} + sr{V~~(x & U ~~ V**1)} } } @@ -74,7 +73,7 @@ fn copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { def RV = [bulk]R rp:= *R~~r - copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l} + copyFromBits{RV, {T=[k]_, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*k, k}}, r, l} # } } diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 624de946..f333ef3a 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -4,7 +4,7 @@ include './vecfold' if_inline (hasarch{'SSE2'}) { fn sum_vec{T}(v:T) = vfold{+, fold{+, unpackQ{v, T**0}}} - def fold_addw{v:T if eltype{T}==i8} = sum_vec{T}(v) + def fold_addw{v:T=[_](i8)} = sum_vec{T}(v) } def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 7926f00f..6057e615 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -7,7 +7,7 @@ include 'util/tup' def rootty{T if isprim{T}} = T -def rootty{T if isvec{T}} = eltype{T} +def rootty{[_]T} = T def is_s{X} = issigned{rootty{X}} def is_u{X} = isunsigned{rootty{X}} @@ -20,10 +20,10 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - -def arithChk1{F==__add, M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} -def arithChk1{F==__sub, M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} -def arithChk1{F==__add, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r} -def arithChk1{F==__sub, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r} +def arithChk1{(__add), M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} +def arithChk1{(__sub), M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} +def arithChk1{(__add), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', adds{w,x}, r} +def arithChk1{(__sub), M, w:T=[_]E, x:T, r:T if not hasarch{'X86_64'} or width{E}<=16} = tup{'anyne', subs{w,x}, r} @@ -33,7 +33,7 @@ def arithChk2{F, M, w:T, x:T if is_s{T} and (same{F,__add} or same{F,__sub})} = } # × -def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and hasarch{'X86_64'}} = { +def arithChk2{F=(__mul), M, w:T=[_](i8), x:T if hasarch{'X86_64'}} = { def wp = unpackQ{w, T ~~ (T**0 > w)} def xp = unpackQ{x, T ~~ (T**0 > x)} def rp = each{__mul, wp, xp} @@ -46,12 +46,12 @@ def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and tup{packQ{rp}, tup{'~andAllZero', RU~~tree_fold{|, bad}, RU**0xff80}} } } -def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i16==eltype{T} and hasarch{'X86_64'}} = { +def arithChk2{F=(__mul), M, w:T=[_](i16), x:T if hasarch{'X86_64'}} = { rl:= __mul{w,x} rh:= mulh{w,x} tup{rl, tup{'anyne', rh, rl>>15}} } -def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} and hasarch{'X86_64'}} = { +def arithChk2{F=(__mul), M, w:T=[_](i32), x:T if hasarch{'X86_64'}} = { max:= re_el{f32, (ty_u{T})**0x4efffffe} def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} f32mul:= cf32{w} * cf32{x} @@ -68,11 +68,11 @@ def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} an # tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M } -def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and hasarch{'AARCH64'}} = { +def arithChk2{F=(__mul), M, w:T=[_]E, x:T if hasarch{'AARCH64'}} = { def r12 = mulw{w, x} rl:= packLo{r12} rh:= packHi{r12} - tup{rl, tup{'homAny', M{rh != (rl >> (elwidth{T}-1))}}} + tup{rl, tup{'homAny', M{rh != (rl >> (width{E}-1))}}} } @@ -88,7 +88,7 @@ def runner{u, R, F} = { def run{F, M, w, x if u} = tup{F{w, x}, tup{'none'}} # trivial base implementation - def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1)) + def toggleTop{x:X=[_]E} = x ^ X**(1<<(width{E}-1)) def run{F==__sub, M, w:VU, x:VU if c and is_u{VU}} = { # 'b'-'a' def VS = ty_s{VU} run{F, M, VS~~toggleTop{w}, VS~~toggleTop{x}} @@ -99,7 +99,7 @@ def runner{u, R, F} = { } def run{F==__add, M, w:VS, x:VU if c and is_s{VS} and is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3 - def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1 + def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} or same{F,__sub})} = { # 'a'+1, 'a'-1 r:= F{ty_u{w}, ty_u{x}} tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} } diff --git a/src/singeli/src/hashtab.singeli b/src/singeli/src/hashtab.singeli index 502fc1a2..4572e6fc 100644 --- a/src/singeli/src/hashtab.singeli +++ b/src/singeli/src/hashtab.singeli @@ -9,8 +9,8 @@ def map{{f,...fs}, {t,...ts}, v} = if (v==f) t else map{fs, ts, v} def to_prim = map{names, prims, .} # Defined in C -def memset{p:pT, v, l} = { - emit{void, merge{'memset',fmtnat{elwidth{pT}}}, p, v, l} +def memset{p:*T, v, l} = { + emit{void, merge{'memset',fmtnat{width{T}}}, p, v, l} } # These hashes are stored in tables and must be invertible! diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index adc575e2..83b5794e 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -42,29 +42,26 @@ def maskAfter{n} = { -def loadLowBatch{T, ptr:P, w, n if eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w} +def loadLowBatch{V=[_]T, ptr:*T, w, n} = loadLow{*V ~~ (ptr + n*(w/width{T})), w} -# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M -def storeBatch{ptr:P, n, x:T, M} = { - def rpos = ptr + n*vcount{T} - def E0 = eltype{P} - def TF = re_el{E0, T} +# store k items into the n'th batch of ptr elements, compressing the items if needed; masked by M +def storeBatch{ptr:*E0, n, x:V=[k]T, M} = { + def rpos = ptr + n*k + def TF = re_el{E0, V} xu:= narrow{E0, x} if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}} - else storeLow{rpos, vcount{T}*width{E0}, xu} + else storeLow{rpos, k*width{E0}, xu} } -# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T -def loadBatch{ptr:P, n, T} = { - def rpos = ptr + n*vcount{T} - def E0 = eltype{P} - - widen{T, loadLow{*re_el{E0, T} ~~ rpos, vcount{T}*width{E0}}} +# (sign/zero)-extend n'th batch of k elements of ptr into elements of V +def loadBatch{ptr:*E0, n, V=[k]T} = { + def rpos = ptr + n*k + widen{V, loadLow{*re_el{E0, V} ~~ rpos, k*width{E0}}} } -def loadBatch {ptr:P, ns, T if istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns} -def storeBatch{ptr:P, ns, xs, M if istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} +def loadBatch {ptr:P, {...ns}, T } = each{{n } => loadBatch {ptr, n, T }, ns} +def storeBatch{ptr:P, {...ns}, xs, M} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} @@ -74,14 +71,14 @@ def hCast{T,p:*T} = p def hCast{T,p:(*void)} = *T~~p def mlExec{i, iter, vars0, bulk, M} = { - def vproc{p:P if isptr{P}} = p - def vproc{('m')} = tptr{{_}=>M, '!'} + def vproc{p:*T} = p + def vproc{'m'} = tptr{{_}=>M, '!'} def vproc{{T,p:P}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} - def vproc{{('b'), p:P}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} - def vproc{{('b'),T,p:P}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'} - def vproc{{('g'), p:P}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} - def vproc{{('g'),T,p:P}} = tptr{{i} => { + def vproc{{'b', p:P}} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} + def vproc{{'b',T,p:P}} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'} + def vproc{{'g', p:P}} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} + def vproc{{'g',T,p:P}} = tptr{{i} => { def dv{} = loadBatch{p, i, T} def dv{x} = storeBatch{p, i, x, M} }, '!'} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 2f964841..3b5b515d 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -10,7 +10,7 @@ def nty{T} = { def q = quality{T} merge{if (q=='i') 's' else q, fmtnat{width{T}}} } -def nty{T if isvec{T}} = nty{eltype{T}} +def nty{[_]T} = nty{T} def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}} def ntyp0{S, T} = merge{S, '_', nty{T}} @@ -28,12 +28,12 @@ def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # ( def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f} def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m} -def addpw { x:T if nveci{T} and elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening -def addpwa{a:D, x:T if nveci{T} and elwidth{T}<=32 and D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate +def addpw { x:T if nveci{T} and elwidth{T}<=32} = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening +def addpwa{a:D==el_m{T}, x:T if nveci{T} and elwidth{T}<=32} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x} -def rev{w, x:T if w==elwidth{T}} = x +def rev{(width{T}), x:[_]T} = x def rev{w==16, x:T if elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window def rev{w==32, x:T if elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x} def rev{w==64, x:T if elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x} @@ -41,33 +41,32 @@ def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x} def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x} def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x} -def fold_add {a:T if nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a} -def fold_addw{a:T if nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a} -def fold_min {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a} -def fold_max {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a} +def fold_add {a:T=[_]E if nvec{T}} = emit{E, ntyp{'vaddv', T}, a} +def fold_addw{a:T=[_]E if nveci{T}} = emit{w_d{E}, ntyp{'vaddlv', T}, a} +def fold_min {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vminv', T}, a} +def fold_max {a:T=[_]E if nvec{T} and ~nveci{T,64}} = emit{E, ntyp{'vmaxv', T}, a} def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x} def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x} def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x} -def storeLow{ptr:P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } -def storeLow{ptr:P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} +def storeLow{ptr:*P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } +def storeLow{ptr:*P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} -def loadLow{ptr:P, w if w<=64} = { # a broadcast load - def T=eltype{P} +def loadLow{ptr:*T, w if w<=64} = { # a broadcast load def L=re_el{ty_u{w}, T} T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} } -def loadLow{ptr:P, w if w==elwidth{P}} = load{ptr} +def loadLow{ptr:*T, (width{T})} = load{ptr} -def undefPromote{T, x:X if w64{X} and w128{T} and eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯ +def undefPromote{T=[_]E, x:X=[_]E if w64{X} and w128{T}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯ def half{x:T, (0) if w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x} def half{x:T, (1) if w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x} def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b} -def copyLane{dst:D, di, src:S, si if w64{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si} -def copyLane{dst:D, di, src:S, si if w128{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} +def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w64{D} and nvec{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si} +def copyLane{dst:D=[_]T, di, src:S=[_]T, si if w128{D} and nvec{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si} def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i} def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n} @@ -89,22 +88,22 @@ def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers -def cvt{T==f64, x:X if nveci{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_f64', X}, x} -def cvt{T==i64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_s64', X}, x} -def cvt{T==u64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_u64', X}, x} +def cvt{T==f64, x:X=[k]_ if nveci{X,64}} = emit{[k]T, ntyp{'vcvt', '_f64', X}, x} +def cvt{T==i64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_s64', X}, x} +def cvt{T==u64, x:X=[k]_ if nvecf{X,64}} = emit{[k]T, ntyp{'vcvt', '_u64', X}, x} def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x} def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}} -def widen{T, x:X if w64{X} and isfloat{eltype{T}}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}} -def widen{T, x:X if w128{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}} +def widen{T=[k]E, x:X if w64{X} and isfloat{E}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{E, widen{[k](to_w{eltype{X},elwidth{T}}), x}} +def widen{T=[k]E, x:X if w128{X} and vcount{X}>k} = widen{T, half{x,0}} def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}} def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x} def narrow{T, x:X if w128{X} and isfloat{T}!=isfloat{eltype{X}} and width{T}=vcount{T}} = { - truncBits{vcount{T}, fold_add{x & make{T, 1<=k} = { + truncBits{k, fold_add{x & make{T, 1<1 and elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} -def homMask{a:T,b:T if vcount{T}*2<=elwidth{T}} = { - def n = vcount{T} - truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}} +def homMask{a:T,b:T=[n]E if n*2<=width{E}} = { + truncBits{n*2, fold_add{shrm{a,width{E}-n,b} & make{T, (1<1} } else { @@ -59,7 +59,7 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}} # Constant replicate -if_inline (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) { +if_inline (not (hasarch{'AVX2'} or hasarch{'AARCH64'})) { fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}} @@ -101,8 +101,7 @@ def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} rcsh4_dat:*i8 = join{join{each{get_shuf_data{., 4}, rcsh4_dom}}} rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} -def read_shuf_vecs{l, ellw:(u64), shp:P} = { # tuple of byte selectors in 1< ...} - def step = vcount{V} +def rep_const_shuffle{wv, onreps, xv:*V=[step]T, rv:*V, n:(u64)} = { # onreps{inputVector, {nextOutputVector} => ...} nv := n / step j:u64 = 0 def write{v} = { store{rv, j, v}; ++j } @@ -168,7 +166,7 @@ if_inline (hasarch{'AVX2'}) { {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } - def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} + def rep_const_shuffle{wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{wv, get_rep_iter{V, wv}, xv, rv, n} } else if_inline (hasarch{'AARCH64'}) { @@ -176,8 +174,8 @@ if_inline (hasarch{'AVX2'}) { each{{s} => gen{sel{[16]u8, x, s}}, sh} } - def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:(u64)} = { - def E = ty_u{eltype{V}} + def rep_const_shuffle{wv==2, xv0:*V=[_]T, rv0:*V, n:(u64)} = { + def E = ty_u{T} rv:= *E~~rv0 @for (x in *E~~xv0 over i to n) { # autovectorized well enough, probably store{rv, i*2, x} @@ -190,7 +188,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { def h = 4 def V = sdtype def sh = read_shuf_vecs{h, ellw, *V~~rcsh4_dat + h*load{rcsh4_lkup,wv}} - def step = vcount{V} # Bytes written + def [step]_ = V # Bytes written def wvb = wv << ellw def hs = (h*step) / wvb # Actual step size in argument elements def shufbase{i if hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010} @@ -226,7 +224,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { fn rcsh_sub{wv, V}(ellw:u64, x:*i8, r:*i8, n:u64, sh:*V) : void = { def st = read_shuf_vecs{wv, ellw, sh} - rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} + rep_const_shuffle{wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} } fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = { @@ -271,7 +269,7 @@ fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { def max_shuffle = 2*vn if (wv <= max_shuffle) { def specialize{k} = { - if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} + if (wv==k) return{rep_const_shuffle{k, *V~~x, *V~~r, n}} } if (special_2) specialize{2} rep_const_shuffle_any(wv, lb{wT/8}, *i8~~x, *i8~~r, n) diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index c3f6f628..60877ce1 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -12,7 +12,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = { @for (x, r over len) r = m = op{m, x} } -def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = { +def scan_loop{init, x:*T, r:*T, len:(u64), scan, scan_last} = { def step = arch_defvw/width{T} def V = [step]T p:= V**init @@ -23,20 +23,20 @@ def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = { q:= len & (step-1) if (q!=0) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}} } -def scan_post{T, init, x:*T, r:*T, len:(u64), op, pre} = { +def scan_post{init, x:*T, r:*T, len:(u64), op, pre} = { def last{v, p} = op{pre{v}, p} def scan{v, p} = { n:= last{v, p} p = toLast{n} n } - scan_loop{T, init, x, r, len, scan, last} + scan_loop{init, x, r, len, scan, last} } # Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈ def scan_idem = scan_scal fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { - scan_post{T, init, x, r, len, op, make_scan_idem{T, op}} + scan_post{init, x, r, len, op, make_scan_idem{T, op}} } export{'si_scan_min_init_i8', scan_idem{i8 , min}}; export{'si_scan_max_init_i8', scan_idem{i8 , max}} @@ -67,7 +67,7 @@ def scan_plus = scan_assoc{+} def scan_assoc_0 = scan_scal fn scan_assoc_0{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = { # Prefix op on entire AVX register - scan_post{T, init, x, r, len, op, scan_plus} + scan_post{init, x, r, len, op, scan_plus} } export{'si_scan_pluswrap_u8', scan_assoc_0{u8 , +}} export{'si_scan_pluswrap_u16', scan_assoc_0{u16, +}} @@ -202,11 +202,10 @@ def addChk{a:T, b:T} = { def bad = emit{u1, '__builtin_add_overflow', a, b, mem} tup{bad, load{mem}} } -def addChk{a:T, b:T if T==f64} = tup{0, a+b} +def addChk{a:T, b:T==f64} = tup{0, a+b} def widenFull{E, xs} = { - merge{...each{{x:X} => { - def n = vcount{X} + merge{...each{{x:X=[n]T} => { def tb = width{E} * n if (tb<=arch_defvw) tup{widen{[n]E, x}} else if (1) { @@ -226,7 +225,7 @@ def maxsafeint{T==f64} = 1<<53 fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { i:u64 = 0 - if (hasarch{'AVX2'}) simd_plus_scan_part{X,R}{x, c, r, len, i} + if (hasarch{'AVX2'}) simd_plus_scan_part{x, c, r, len, i} @forUnroll{1,1} (js from i to len) { def vs = eachx{load, x, js} each{{j, v} => { @@ -239,7 +238,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = { len } # Sum as many vector registers as possible; modifies c and i -def simd_plus_scan_part{X, R}{x:(*X), c:(R), r:(*R), len:(u64), i:(u64)} = { +def simd_plus_scan_part{x:*X, c:R, r:*R, len:(u64), i:(u64)} = { def b = max{width{R}/2, width{X}} def bulk = arch_defvw/b diff --git a/src/singeli/src/scan_common.singeli b/src/singeli/src/scan_common.singeli index b6c444a5..2bfb5651 100644 --- a/src/singeli/src/scan_common.singeli +++ b/src/singeli/src/scan_common.singeli @@ -12,8 +12,8 @@ def sel8{v, t, up} = sel8{v, rev{up,t}} def zip{up, x} = (if (up) zipHi else zipLo){x,x} # Fill last 4 bytes with last element, in each lane -def spread{a:VT, ...up} = { - def w = elwidth{VT} +def spread{a:[_]T, ...up} = { + def w = width{T} def b = w/8 if (w<=16) sel8{a,merge{iota{12},(16-b)+iota{4}%b}, ...up} else a } @@ -39,7 +39,7 @@ def prefix_byshift{op, sh} = { {v:T} => pre{v, if (isvec{T}) elwidth{T} else 1} } -def get_id{op,T} = (match (op) { {_==min}=>maxvalue; {_==max}=>minvalue }){T} +def get_id{op,T} = (match (op) { {(min)}=>maxvalue; {(max)}=>minvalue }){T} def make_scan_idem{T, op, up} = { # Within each lane, scan using shifts by powers of 2. First k elements @@ -59,8 +59,8 @@ def make_scan_idem{T, op, up} = { } prefix_byshift{op, shb} } -def make_scan_idem{T==f64, op, up} = { - def sc{a:T if vcount{T}==2} = op{a, zip{~up,a}} +def make_scan_idem{(f64), op, up} = { + def sc{a:[2]T} = op{a, zip{~up,a}} def sc{a:T if hasarch{'AVX2'} and w256{T}} = { def sh{s, a} = op{a, shuf{[4]u64, a, rev{up,s}}} sh{tup{0,1,1,1},sh{tup{0,0,2,2},a}} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 6770f5da..8bc5c67c 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -85,7 +85,7 @@ def bittab_init{tab, z if simd_bittab} = { } def bittab_selector{loadtab} = { - def nv = vcount{VI} + def [nv]_ = VI {t0, t1}:= loadtab{} low:= VI**7 hi4:= VI**(-(1<<4)) @@ -101,7 +101,7 @@ def bittab_selector{loadtab} = { } def readbytes{vtab}{} = { - def k = vcount{VI}; def l = 128/vcount{VI} + def [k]_ = VI; def l = 128/k def side{i} = { def U = arch_vec{ty_u{k}} def m = @collect (vtab over _ from i to i+l) homMask{vtab} @@ -126,7 +126,7 @@ def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void)} = { } def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void) if simd_bittab} = { def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}} - def k = vcount{VI} + def [k]_ = VI @for (x in *VI~~x0, r in *ty_u{k}~~r0 over cdiv{n,k}) r = bitsel{x} } @@ -175,7 +175,7 @@ def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = { # Do first few values with a scalar loop # Avoids the cost of ever loading the table into vectors for n<=48 i:u64 = 32; if (n<=48) i=n - def k = vcount{VI}; def uk = ty_u{k}; def ik = ty_s{k} + def [k]_ = VI; def uk = ty_u{k}; def ik = ty_s{k} {rw,rv} := undef{tup{u64,uk}} # Bit results, used if rbit if (rbit) rw = 0 @for (x over j to i) { @@ -295,8 +295,8 @@ def acc{unr, init:T} = { else a1 = F{a1} } } -def isI64{x:T if eltype{T}==f64 and hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}} -def isI64{x:T if eltype{T}==f64 and hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53)) +def isI64{x:T=[_](f64) if hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}} +def isI64{x:T=[_](f64) if hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53)) def maskBlend{b:T, x:T, M} = x def maskBlend{b:T, x:T, M if M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}} @@ -306,10 +306,10 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { x:= *E~~x0 min1:E = *x max1:E = *x - if (has_simd and not (E==f64 and not (hasarch{'AARCH64'} or hasarch{'SSE4.1'}))) { + if (has_simd and (not E==f64 or hasarch{'AARCH64'} or hasarch{'SSE4.1'})) { def bulk = arch_defvw/width{E} def VT = [bulk]E - def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2} + def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2} def minA = acc{2, VT**min1} def maxA = acc{2, VT**min1} @muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) { @@ -599,8 +599,8 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it } def try_vec_memb{..._} = {} -def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done - if hasarch{'SSE4.2'} and T==u32} = { +def try_vec_memb{T==u32, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done + if hasarch{'SSE4.2'}} = { # Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints) # i-h>>sh = i+((1<>sh = (((i+1)<>sh # We maintain io = (i+1)< { v:=load{*V~~x, i}; ++i; v } } else { i := iota{V} - if (isreg{x}) i += V**cast_i{eltype{V},x} - ii := V**vcount{V} + if (isreg{x}) i += V**cast_i{T,x} + ii := V**k {} => { v:=i; i+=ii; v } } } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 8e63d014..103df555 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -7,24 +7,24 @@ include './vecfold' def preserve_negative_zero = 0 # SSE2 versions avoid any 64-bit integer comparsions -def anySNaN{M, x:T if eltype{T}==u64} = { +def anySNaN{M, x:[_](u64)} = { homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}} } -def anySNaN{M, x:T if T==[2]u64 and hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { +def anySNaN{M, x:T==[2]u64 if hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = { topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}} } -def anyNonChar{M, x:T if isvec{T} and eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} -def anyNonChar{M, x:T if isvec{T} and hasarch{'X86_64'}} = { +def anyNonChar{M, x:[_](u64)} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}} +def anyNonChar{M, x:T=[_]_ if hasarch{'X86_64'}} = { def H = re_el{u32, T} def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16} topAny{M{T~~ne}} } -def cvtNarrow{T, x:X if width{T}==elwidth{X}} = cvt{T, x} -def cvtNarrow{T, x:X if width{T}< elwidth{X}} = narrow{T, x} -def cvtWiden{T, x:X if elwidth{T}==elwidth{X}} = cvt{eltype{T}, x} -def cvtWiden{T, x:X if elwidth{T}> elwidth{X}} = widen{T, x} +def cvtNarrow{T, x:[_]X if width{T}==width{X}} = cvt{T, x} +def cvtNarrow{T, x:[_]X if width{T}< width{X}} = narrow{T, x} +def cvtWiden{ [_]T, x:[_]X if width{T}==width{X}} = cvt{T, x} +def cvtWiden{V=[_]T, x:[_]X if width{T}> width{X}} = widen{V, x} fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { assert{len>0} @@ -35,14 +35,11 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E} # fold with either Max or Bitwise Or, truncating/zero-extending to TE - def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}} - def foldTotal{TE, x:T if hasarch{'AARCH64'}} = { - if (elwidth{T}==64) { - if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} - else vfold{max, narrow{TE, x}} - } else { - vfold{max, x} - } + def foldTotal{TE, x:[_]T} = cast_i{TE, vfold{|, x}} + def foldTotal{TE, x:[_]T if hasarch{'AARCH64'}} = { + if (width{T}!=64) vfold{max, x} + else if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} + else vfold{max, narrow{TE, x}} } # show{XV, EV, CHR, B} @@ -79,8 +76,8 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { return{0xffff_ffff} # float }} - def getAcc{EV, x:T} = { - ((EV ** ~(eltype{EV})~~1) & EV~~x) ^ EV~~(x >> (elwidth{T}-1)) + def getAcc{EV=[_]E, x:[_]T} = { + ((EV ** ~E~~1) & EV~~x) ^ EV~~(x >> (width{T}-1)) } if (isint{X}) { # i16, i32 @@ -89,7 +86,10 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { } } else { # f64 r2:= EV2**0 - @muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) { + @muLoop{ + bulk, hasarch{'AARCH64'}+1, + {} => { r1 = half{r2,0}|half{r2,1} } + }(v0 in tup{XV,xp}, M in 'm' over is to len) { def int = { def {int, wdn} = { if (hasarch{'AARCH64'} and length{is}==2) { @@ -99,15 +99,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { } else { def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0} def wdn = each{{v} => cvtWiden{XV, v}, ints} - if (vcount{type{select{ints,0}}} == bulk) { - def intp = { - if (length{ints}==1) select{ints, 0} - else pair{ints} - } - tup{intp, wdn} - } else if (length{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) { - tup{select{ints, 0}, wdn} - } else assert{0} + def intp = match (...ints) { + {i:[(bulk)]_} => i + {i if hasarch{'X86_64'} and not hasarch{'AVX2'}} => i + {i:T=[(bulk)]_, j:T} => pair{ints} + } + tup{intp, wdn} } } @@ -124,7 +121,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = { } int } - def acc = { if (length{is}==2) r2 else r1 } + def acc = if (length{is}==2) r2 else r1 acc|= M{getAcc{type{acc}, int}} } diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index cc1a6309..dc347da2 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -1,6 +1,6 @@ ### SSSE3 ### def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} -def vshl{a:T, b:T, n if hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)} +def vshl{a:V=[_]T, b:V, n if hasarch{'SSSE3'}} = V~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(width{T}/8)} ### SSE4.1 ### @@ -8,16 +8,16 @@ def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packu def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} # conversion -def widen{T==[8]u16, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} -def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} -def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} -def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} -def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} -def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} -def widen{T==[2]f64, x:X if hasarch{'SSE4.1'} and w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} +def widen{T==[8]u16, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x} +def widen{T==[4]u32, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x} +def widen{T==[4]u32, x:X==[8]u16 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x} +def widen{T==[2]u64, x:X==[16]u8 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x} +def widen{T==[2]u64, x:X==[8]u16 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x} +def widen{T==[2]u64, x:X==[4]u32 if hasarch{'SSE4.1'}} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x} +def widen{T==[2]f64, x:X=[_]E if hasarch{'SSE4.1'} and w128i{X} and width{E}<32} = widen{T, widen{[4]i32, x}} -def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} -def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} +def narrow{(i8 ), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}} +def narrow{(i16), x:X if hasarch{'SSE4.1'} and w128i{X,32}} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}} # mask stuff def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index eff66f82..14fc934f 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -1,25 +1,25 @@ # compact casting for the annoying intrinsic type system -def v2i{x:T if w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x +def v2i{x:V=[_]T if w128{V}} = if(isint{T}) x else [16]u8 ~~ x def v2f{x:T if w128{T}} = [4]f32 ~~ x def v2d{x:T if w128{T}} = [2]f64 ~~ x # load & store -def loadLow{ptr:P, w if w128{eltype{P}} and w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr} -def loadLow{ptr:P, w if w128{eltype{P}} and w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr} -def loadLow{ptr:P, w if w128{eltype{P}} and w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def loadLow{ptr:P, w if w128{eltype{P}} and w==128} = load{ptr} +def loadLow{ptr:*T, w if w128{T} and w== 16} = T ~~ emit{[16]u8, '_mm_loadu_si16', ptr} +def loadLow{ptr:*T, w if w128{T} and w== 32} = T ~~ emit{[16]u8, '_mm_loadu_si32', ptr} +def loadLow{ptr:*T, w if w128{T} and w== 64} = T ~~ emit{[16]u8, '_mm_loadu_si64', ptr} +def loadLow{ptr:*T, w if w128{T} and w==128} = load{ptr} -def storeLow{ptr:P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} -def storeLow{ptr:P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x} +def storeLow{ptr:*P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} +def storeLow{ptr:*P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} +def storeLow{ptr:*P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} +def storeLow{ptr:*P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x} # float comparison -def unord{a:T,b:T if T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} -def unord{a:T,b:T if T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} +def unord{a:T,b:T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b} +def unord{a:T,b:T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b} @@ -51,21 +51,21 @@ def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}} def topMask{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} def topMask{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} def homMask{x:T if w128{T}} = topMask{x} -def homMaskX{a:T if elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}} +def homMaskX{a:[_]T if width{T}==16} = tup{2, homMask{re_el{u8,a}}} def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}} def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0 def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff -def topAny{x:T if w128i{T}} = topMask{x} != 0 -def topAll{x:T if w128i{T}} = topMask{x} == (1<elwidth{X}} = { def s{v} = s{mzipLo{v, v}} - def s{v:V if V==T} = v + def s{v:(T)} = v s{x} >> (elwidth{T} - elwidth{X}) } def widen{T==[2]f64, x:X if w128s{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}} -def widen{T==[2]f64, x:X if X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} -def widen{T==[2]f64, x:X if X==[4]f32} = emit{T, '_mm_cvtps_pd', x} +def widen{T==[2]f64, x:X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x} +def widen{T==[2]f64, x:X==[4]f32} = emit{T, '_mm_cvtps_pd', x} def narrow{T==i16, x:([4]i32)} = packs{x,x} def narrow{T==i8, x:([8]i16)} = packs{x,x} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 45d8dbfd..244bd49a 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -27,9 +27,9 @@ def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x} # Square kernel where width is a full vector def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x} -def load2{a:T, b:T} = pair{load{a}, load{b}} -def store2{a:T, b:T, v:T2 if w128i{eltype{T}} and w256{T2}} = { - each{{p:P, i} => store{p, 0, eltype{P}~~half{v,i}}, tup{a,b}, iota{2}} +def load2{a:*T, b:*T} = pair{load{a}, load{b}} +def store2{a:*T, b:*T, v:T2 if w128i{T} and w256{T2}} = { + each{{p, i} => store{p, 0, T~~half{v,i}}, tup{a,b}, iota{2}} } def load_k {VT, src, l, w if w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} def store_k{VT, dst, x, l, h if w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} @@ -37,8 +37,7 @@ def load_k {VT, src, l, w if w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*V def store_k{VT, dst, x, l, h if w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} # Transpose kernel of size kw,kh in size w,h array -def kernel{src:P, dst:P, kw, kh, w, h} = { - def T = eltype{P} +def kernel{src:*T, dst:*T, kw, kh, w, h} = { def n = (kw*kh*width{T}) / 256 # Number of vectors def xvs = load_k{[kw]T, src, n, w} def xt = unpack_to{n==kh, n/2, xvs} # Transpose n by n @@ -105,7 +104,7 @@ def transpose_with_kernel{T, k, kh, call_base, rp:*T, xp:*T, w, h, ws, hs} = { assert{k == kh} def VT = [k]T def line_vecs = line_bytes / (width{VT}/8) - def store_line{p:P, vs} = each{{i,v} => store{p, i, eltype{P}~~v}, iota{line_vecs}, vs} + def store_line{p:*T, vs} = each{{i,v} => store{p, i, T~~v}, iota{line_vecs}, vs} def get_lines{loadx} = { def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli index 92a57f45..0961a50f 100644 --- a/src/singeli/src/vecfold.singeli +++ b/src/singeli/src/vecfold.singeli @@ -1,12 +1,12 @@ # Fold associative/commutative operation across a register -def vfold{F, x:T if w128{T} and hasarch{'X86_64'}} = { +def vfold{F, x:V=[_]T if w128{V} and hasarch{'X86_64'}} = { c:= x - def EW = elwidth{T} + def EW = width{T} if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} - if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} } + if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{T, v}, cast_i{T, v>>8}} } else extract{c, 0} } def vfold{F, x:T if w256{T} and hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}