From 6c89982bbc1eec964cc22ed15654a8ec6bbc770f Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 15 Jan 2023 22:49:15 +0200 Subject: [PATCH] unrolled Singeli loop option --- build/singeliSubmodule | 2 +- src/singeli/src/base.singeli | 66 ++++++++++++++++---------------- src/singeli/src/debug.singeli | 71 +++++++++++++++++++++++++++++++++++ src/singeli/src/mask.singeli | 43 +++++++++++++++++++-- src/singeli/src/neon.singeli | 3 +- src/singeli/src/sse3.singeli | 4 -- 6 files changed, 148 insertions(+), 41 deletions(-) create mode 100644 src/singeli/src/debug.singeli diff --git a/build/singeliSubmodule b/build/singeliSubmodule index 19f3f30e..fb214069 160000 --- a/build/singeliSubmodule +++ b/build/singeliSubmodule @@ -1 +1 @@ -Subproject commit 19f3f30e156b78c310fe76727bd8ed8579364542 +Subproject commit fb21406949d5b613ca5dc896708bd698fcf4bc28 diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 1839f5bd..884d43af 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -60,6 +60,16 @@ def anyInt{x & isreg{x}|isconst{x}} = isint{x} def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x} def trunc{T, x & knum{x}} = cast{T, x} +def tern{c, T, F & anyInt{c}} = { + if(c) T + else F +} +def tern{c, t:T, f:T & anyInt{c}} = { + res:T = f + if (c) res = t + res +} + # reinterpret vector element type to signed/unsigned def ty_s{w} = primtype{'i', w} def ty_u{w} = primtype{'u', w} @@ -89,8 +99,13 @@ def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T}) # test if vector has a specific width & element type def lvec{T, n, w} = 0 def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 +# type stats +def minvalue{T & isunsigned{T}} = 0 +def maxvalue{T & isunsigned{T}} = (1<b, a, b} + + + +# tuple operations +def broadcast{T, v & isprim{T}} = v +def iota{n & knum{n}} = @collect(i to n) i +def broadcast{n, v & knum{n}} = @collect(n) v + +def arch_defvw = tern{hasarch{'X86_64'}, 256, 128} + + def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def f{i,l & i==l} = 0 @@ -132,36 +160,10 @@ def forNZ{vars,begin,end,block} = { } } -def minvalue{T & isunsigned{T}} = 0 -def maxvalue{T & isunsigned{T}} = (1<b, a, b} - - -# tuple operations -def broadcast{T, v & isprim{T}} = v -def iota{n & knum{n}} = @collect(i to n) i -def broadcast{n, v & knum{n}} = @collect(n) v - - -# debug stuff -def printfType{T} = tern{isfloat{T}, '%.17g', merge{'%', tern{width{T}==64, 'l', ''}, tern{issigned{T}, 'd', 'u'}}} -def print{x & match{kind{x},'symbol'}} = { emit{void, 'printf', '"%s"', merge{'"', x, '"'}}; x } -def print{x:T & anyNum{x}} = { emit{void, 'printf', merge{'"', printfType{T}, '"'}, x}; x } -def println{x} = { print{x}; print{'\n'}; x } - -def arch_defvw = tern{hasarch{'X86_64'}, 256, 128} \ No newline at end of file +def makeregs{v, n} = @collect(n) { reg:=v } diff --git a/src/singeli/src/debug.singeli b/src/singeli/src/debug.singeli new file mode 100644 index 00000000..6d027709 --- /dev/null +++ b/src/singeli/src/debug.singeli @@ -0,0 +1,71 @@ +def assert{x:u1} = { if (not x) emit{void, '__builtin_trap'} } + +def test_assert = assert # test_assert is guaranteed to either not exist, or always trap on bad input + +def printf{...vs0} = { + def nsym = ~each{ksym, vs0} + def vs = join{each{{a,b}=>{ + if (b) tup{' ', a} + else tup{a} + }, vs0, nsym & shiftright{tup{0}, nsym}}} + + def listfmt{hex, s, e, vs} = { + if (tuplen{vs}==0) { + tup{merge{s, e}, tup{}} + } else { + def f = each{{v} => runfmt{hex, v}, vs} + def f2 = each{{f1, i} => { + if (i+1 == tuplen{f}) f1 + else merge{f1, tup{tup{',%c', tup{32}}}} + }, f, iota{tuplen{f}}} + tup{ + tup{s, tup{}}, + ...join{f2}, + tup{e, tup{}} + } + } + } + + def runfmt{hex, x} = { + if (ksym{x}) { + each{{c} => { + if (c==' ') tup{'%c', tup{32}} + else tup{c, tup{}} + }, symchars{x}} + } else if (ktup{x}) { + if (tuplen{x}==2 and match{tupsel{0,x}, 'x'}) runfmt{1, tupsel{1,x}} + else listfmt{hex, '{', '}', x} + } else if (kreg{x} or kcon{x}) { + def T = type{x} + if (match{typekind{T},'primitive'}) { + def q = quality{T} + def w = width{T} + def u = (w>1) & (q=='u') + def spec = { + if (q=='f') '%.18g' + else merge{tern{u & hex, '0x', ''}, '%', tern{w==64, 'l', ''}, tern{u, tern{hex, 'x', 'u'}, 'd'}} + } + tup{tup{spec, tup{x}}} + } else if (match{typekind{T},'vector'}) { + listfmt{hex, '[', ']', each{{i} => extract{x, i}, iota{vcount{T}}}} + } else { + tup{tup{'(unknown typekind)', tup{}}} + } + } else if (knum{x}) { + if ((x>>0) == x) { + if (x>=minvalue{i64} and x<=maxvalue{i64}) runfmt{0, i64~~x} + else if (x>=minvalue{u64} and x<=maxvalue{u64}) runfmt{0, u64~~x} + else runfmt{0, f64~~x} + } else runfmt{0, f64~~x} + } else { + tup{tup{'(unknown type)', tup{}}} + } + } + + def is = join{each{{v} => runfmt{0,v}, vs}} + + def e0s = join{each{{x}=>tupsel{0,x}, is}} + def e1s = join{each{{x}=>tupsel{1,x}, is}} + emit{void, 'printf', merge{'"', e0s, '"'}, ...e1s} +} +def lprintf{...vs} = printf{...vs, '\n'} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 189effd8..e36ff230 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -31,13 +31,12 @@ def maskAfter{n} = { def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}} def mask{X, mode=='to sign bits'} = maskOf{X,n} def mask{mode=='count'} = n + def mask{x & istup{x} & tuplen{x}==1} = tup{mask{tupsel{0,x}}} def mask{x==0} = 1 } -# store low packed elements of x to P - def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w} # store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M @@ -59,6 +58,9 @@ def loadBatch{ptr:P, n, T} = { widen{T, loadLow{*to_el{E0, T} ~~ rpos, vcount{T}*width{E0}}} } +def loadBatch {ptr:P, ns, T & istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns} +def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs} + def maskedLoop{bulk, l, step} = maskedLoop{bulk, 0, l, step} @@ -78,4 +80,39 @@ def maskedLoopPositive{bulk, l:L, step} = { ++i } step{i, maskAfter{l - i*bulk}} -} \ No newline at end of file +} + +# masked unrolled loop +# bulk: vector count +# unr: unroll amount +# l: length +# step: {is, M}=>operation +# 'is' is a tuple of bulk indexes to process, usable as an arg to loadBatch/storeBatch to get a tuple of vectors +# fromunr (optional): {}=>{transition from unrolled to non-unrolled} +def muLoop{bulk, unr, l, step, fromunr} = { + if (unr==1) { + maskedLoop{bulk, l, {i, M} => step{tup{i}, M}} + } else { + m:u64 = l/bulk + if (m > 0) { + i:u64 = 0 + if (unr <= m) { + while ((i+unr) <= m) { + def is = each{{j}=>i+j, iota{unr}} + step{each{{j}=>i+j, iota{unr}}, maskNone} + i+= unr + } + fromunr{} + } + if (unr==2) { + if (i!=m) step{tup{i}, maskNone} + } else { + @for(j from i to m) step{tup{j}, maskNone} + } + } + + left:= l & (bulk-1) + if (left!=0) step{tup{m}, maskAfter{left}} + } +} +def muLoop{bulk, unr, l, step} = muLoop{bulk, unr, l, step, {}=>0} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index e19d47ad..11fd2a34 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -49,13 +49,14 @@ def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M} & width{T}==width{M} & elwidth{T def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m} def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a} -def addpw{a:T & nveci{T} & elwidth{T}<=32} = emit{ty_dbl{T}, ntyp{'vpaddl', T}, a} def __not{a:T & nvecu{T}} = T~~emit{to_el{u8,T}, ntyp{'vmvn', to_el{u8,T}}, a} def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a} def floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a} def ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a} def abs{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vabs', T}, a} def absu{a:T & nveci{T}} = ty_u{abs{a}} +def addpw { x:T & nveci{T} & elwidth{T}<=32 } = emit{ty_dbl{T}, ntyp{'vpaddl', T}, x} # add pairwise widening +def addpwa{a:D, x:T & nveci{T} & elwidth{T}<=32 & D==ty_dbl{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate def __eq{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vceq', T}, a, b} def __ge{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcge', T}, a, b} diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 6c72c55c..44ee0c03 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -143,10 +143,6 @@ def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0} # the lone SSE2 extract def extract{x:T, i & w128i{T,16} & knum{i}} = emit{eltype{T}, '_mm_extract_epi16', x, i} -# debug stuff -local def printGen{x, s, n} = emit{void, 'printf', merge{'"', apply{merge,each{{c}=>{if(c>0) merge{',',s}; else s}, iota{n}}}, '"'}, ...each{{c}=>extract{x,c}, iota{n}}} -def print{x:T & isvec{T}} = printGen{x, printfType{eltype{T}}, vcount{T}} - ### SSE4.1; TODO either move to own file or rename this file