diff --git a/src/singeli/README.md b/src/singeli/README.md new file mode 100644 index 00000000..1e3a4aee --- /dev/null +++ b/src/singeli/README.md @@ -0,0 +1,355 @@ +## Overview + +Types used here follow `[VET][ISUF]?\d?`. +First character: + - `V`: some vector type + - `E`: some scalar type + - `T`: either scalar or vector type + +Second character, if present: + - `F`: float + - `S`: signed integer + - `U`: unsigned integer (may include `u1`) + - `I`: any signedness integer (may include `u1`) + +Equally-named types should match. A number is added to the type to separate unequal types. + +`mt{T}` is a mask version of the argument - either `ty_u{T}` or `[vcount{T}]u1` depending on the target. + +The SIMD operations listed aren't guaranteed to be supported on all targets, nor list all possible argument types. + +## General definitons + +- `has_simd` + `0` or `1` depending on whether SIMD types are available. + +- `arch_defvw` + Target vector width for the current target architecture. + If `has_simd==0`, should not be used. Otherwise, vectors of all element types (except `u1`) whose width is this must be supported, but other widths (both larger and smaller) may be too. + +- `reinterpret` is available as `~~` + +- `exportN{f, 'name1', 'name2', ...}` - export single function under many names +- `exportT{'name', fs}` - export list of functions as the specified name +- `iota{knum}` - tuple `tup{0,1,...,knum-1}` +- `broadcast{knum,v}` (`knum**v`) - tuple with `knum` items, all `v` +- `tern{c, t, f}` - ternary - if `c`, return `t`, else `f` +- `eachx{F, ...args}` - `each` but broadcasting non-tuples +- `undef{T}` - return register of type `T` with undefined value; maps over a tuple of types +- `undef{T, n}` - a tuple of `n` different undefined registers +- `oneVal{xs}` - assert that `xs` is a tuple of equivalent (as per `==`) items, and return the first +- `oneType{xs}` - `oneVal` but over the types of the items + +## Loops & branches + +- `@for` - generate body once, with indices `s,s+1,...,e-2,e-1` +- `@forNZ` - `@for`, but assumed to run at least once +- `@for_backwards` - `@for`, but indices go backwards `e-1,e-2,...,s+1,s` +- `@forUnroll{exp, unr}` - generate a main loop body that takes a tuple of `unr` indices to process. Handle the tail via processing one index at a time; if `exp==1`, that's done via `unr` generated bodies, otherwise it's a loop +- `@unroll(n)` - generate body `n` times +- `@collect(n)` - generate body `n` times, and collect the results as a tuple + +- `makeBranch{Ts, G=={...x:Ts}=>{}} == {...x:Ts}=>{}` - make a code segment that takes arguments of types `Ts`, and can be jumped to by invoking the result of the `makeBranch` +- `makeOptBranch{enable, Ts, G}` - `makeBranch` but only generated & invokable when `enable==1` + +## Scalar operations + +- `popc{x:EI} : uint` - population count +- `popcRand{x:EI} : uint` - population count; under valgrind, return a random value in the set of possible ones if the input is partly undefined +- `ctz{x:EI} : uint` - count trailing zeroes; if `x==0`, the result is undefined +- `clz{x:EI} : uint` - count leading zeroes; if `x==0`, the result is undefined +- `clzc{x:EI} : uint` - `width{EI}-clz{x}`; if `x==0`, the result is undefined + +- `zlow{n, x}` - zero low `n` bits of `x` +- `tail{n, x}` - keep low `n` bits of `x` +- `bit{n, x}` - get `n`-th bit of `x` + +- `lb{n}` - base-2 log of a constant power of two +- `base{b, x}` - convert tuple to number in little-endian base `b` +- `tree_fold{G, xs}` - fold tuple `xs` with a tree-like pattern with `G` + +## Hints + +- `rare{x:u1} : u1` - Expect that `x` is `0`. Returns `x`. +- `likely{x:u1} : u1` - Expect that `x` is `1`. Returns `x`. +- `assert{x:u1} : void` - assume `x` is `1`. If `x` is a compile-time constant, fails to compile if it's not `1`. If at runtime it is not `1`, undefined behavior happens (unless debug mode, via either `include './debug'` or CBQN-wide `DEBUG` mode) + +## Type operations + +Of the below functions which take & return vector types of the same width, they can instead take a vector value & reinterpret it to the result type. + +- `w_n{E, w} == E2` - a scalar type with the quality of `E` and width `w` +- `w_d{E} == E2`, `w_h{E} == E2` - double/halve the width of a scalar type +- `n_d{V} == V2`, `n_h{V} == V2` - double/halve element count in vector (keeps the same element width) +- `el_d{V} == V2`, `el_h{V} == V2` - double/halve element width in vector (keeps the same count) +- `el_m{V} == V2`, `el_s{V} == V2` - vector with the same total width, but element width doubled (`el_m` - elements merged) or halved (`el_s` - elements split) + +- `ty_u{T} == TU` - unsigned integer version of the type +- `ty_s{T} == TS` - signed integer version of the type +- `ty_f{T} == TF` - float version of the type +- `re_el{E, V} == V2` - reinterpret vector to one with the specified element - `eltype{V2}==E and width{V2}==width{V}` + +- `minvalue{EI}`, `maxvalue{EI}` - smallest & biggest value of the specified type + +## Arithmetic + +Some may also support one scalar argument or arguments with different widths. + +- `__neg{a}` - `-a` +- `__add{a:V, b:V} : V` - `a + b` +- `__sub{a:VI, b:VI} : VI` - `a - b` +- `__div{a:V, b:V} : V` - `a / b` +- `__mul{a:V, b:V} : V` - `a * b` +- `mulHi{a:VI, b:VI} : VI` - high half of multiplication + +- `__eq`, `__ne`, `__ge`, `__gt`, `__le`, `__lt` - comparison; `{a:V, b:V} : mt{V}` + +- `__not{a:VI} : VI` - `~a` +- `__and{a:VI, b:VI} : VI` - `a&b` +- `__or{a:VI, b:VI} : VI` - `a|b` +- `__xor{a:VI, b:VI} : VI` - `a^b` +- `andnot{a:VI, b:VI} : VI` - `a&~b` +- `__shl{a:VI, b:TI} : VI` - `a<>b`; arithmetic shift right +- `__shr{a:VU, b:TI} : VU` - `a>>b`; logical shift right + +- `abs{a:V} : V` - absolute value +- `absu{a:VI} : ty_u{VI}` - absolute value, reinterpreted as unsigned +- `adds{a:VI, b:VI} : VI` - saturating add +- `subs{a:VI, b:VI} : VI` - saturating subtract + +- `rcpE{a:VF} : VF` - estimate of `1/a`; precision depends +- `rsqrtE{a:VF} : VF` - estimate of `sqrt{1/a}`; precision depends + +- `sqrt{a:VT} : VT` - square root +- `floor{a:VF} : VF` - floor +- `ceil{a:VF} : VF` - ceiling +- `max{a:V, b:V} : V` - minimum value; arch-specific behavior on NaNs +- `min{a:V, b:V} : V` - maximum value; arch-specific behavior on NaNs + +- `shl{U, a:V, n}` - shift vector elements within blocks of `U` +- `shr{U, a:V, n}` - shift vector elements within blocks of `U` +- `unord{a:VF, b:VF} : mt{VI}` - `(a==NaN) | (b==NaN)` + +- `andAllZero{a:VI, b:VI} : u1` - whether `a&b` is all zeroes + +## Structural operations + +- `broadcast{V, a:eltype{V}} : V` (`V**a`) - broadcast (splat) the scalar to all elements of the result +- `make{V, ...vs} : V` or `make{V, tup{...vs}}` - initialize a vector with the specified elements +- `iota{V} : V` - `make{V, range{vcount{V}}}` +- `extract{a:V, n} : eltype{V}` - extract the n-th element +- `insert{a:V, i, b:eltype{V}} : V` - a copy of `a` with the `i`-th element replaced with `b` +- `half{a:V, n} : count_half{V}` - extract either the low (`n==0`) or high (`n==1`) half of `a` +- `pair{a:V, b:V} : count_dbl{V}` - make a vector from halves +- `undefPromote{V2, a:V} : V` - promote `a` to a wider type, leaving the upper part undefined +- `vshl{a:V, b:V, n} : V` - `vshl{[0,1,2,3], [4,5,6,7], 1} → [1,2,3,4]` +- `sel{VI1, a:V, b:VI2} : V2` - shuffle `a` by indices `b` in `VI1`-long lanes; arch-specific behavior on out-of-bounds values + +- `zip{a:T,b:T} : tup{T, T}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}` +- `mzip{a:T,b:T} : tup{el_dbl{T}, el_dbl{T}}` - reinterpreted `zip{a, b}` +- `zipLo{a:T,b:T} : T` - `tupsel{0, zip{a, b}}` +- `zipHi{a:T,b:T} : T` - `tupsel{1, zip{a, b}}` +- `mzipLo{a:T,b:T} : T` - `tupsel{0, mzip{a, b}}` +- `mzipHi{a:T,b:T} : T` - `tupsel{1, mzip{a, b}}` + +## Mask stuff + +Homogeneous definitions (i.e. ones with `hom` in their name) assume that each element in the mask type has all its bits equal. + +- `homAll{a:VI} : u1` - whether all elements are set +- `homAny{a:VI} : u1` - whether any element is set +- `topAny{a:VI} : u1` - whether all elements have their top bit set +- `topAll{a:VI} : u1` - whether any element has its top bit set +- `homBlend{f:V, t:V, m:mt{V}} : V` - blend by `m`, setting to `f` where `0` and `t` where `1` +- `topBlend{f:V, t:V, m:V} : V` - blend by top bit of `m` +- `homMask{a:VI} : uint` - integer mask of whether each element is set (assumes each element has all its bits equal) +- `topMask{a:VI} : uint` - integer mask of the top bit of each element +- `homMaskX{a:VI} : tup{knum, uint}` - integer mask where each element is represented by `knum` bits (possibly more efficient to calculate than `homMask`) +- `ctzX{tup{knum, uint}}` - count trailing zeroes from a result of `homMaskX` + +## Load/store + +Alignment requirements are target-specific, but at most one element. + +For unaligned scalar loads & stores, `loadu` & `storeu` should be used. + +- `loadu{p:*E} : E` - load scalar from unaligned memory +- `storeu{p:*E, a:V} : void` - store scalar to unaligned memory +- `load{p:*V} : V` - load full vector +- `store{p:*V, a:V} : void` - store full vector +- `loadLow{p:*V, w} : V` - load to low `w` bits +- `storeLow{p:*V, w, a:V}` - store low `w` bits +- `homMaskStore{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements +- `topMaskStore{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements +- `homMaskStoreF` - `homMaskStore` but may touch masked-off elements and thus be supported on more types +- `topMaskStoreF` - `topMaskStore` but may touch masked-off elements and thus be supported on more types + + + +## Conversion + +For float conversions, the used rounding mode is unspecified. + +- `narrow{eltype{V2}, a:V} : V2` - narrow elements of `a` to desired type. + For integer-to-integer conversion, completely undefined result if the values don't exactly fit. + Provides `vcount{V2} >= vcount{V}`. +- `widen{V2, a:V} : V2` - widen elements of `a` to desired type. + Discards elements of `a` past `vcount{V2}`. +- `cvt{eltype{V2}, a:V} : V2` - convert element type at the same width. + Requires `vcount{V}==vcount{V2} and elwidth{V}==elwidth{V2}`. + + + + +## x86-specific + +- `mul32{a:VI, b:VI} : VI` - multiply, reading only low 32 bits +- `blend{L, a:V, b:V, m}` - blend `L`-sized blocks via the immediate +- `shuf{L, x:T, n} : T` - shuffle by immediate in `L`-sized lanes +- `packQ` - pack 128-bit lanes (`packs`/`packus`) for 16-bit & 32-bit elements +- `packQQ` - `packQ` but also defined for 64-bit elements, assuming the high halves are zeroes +- `packs` - 128-bit `packs`/`packus` +- `unpackQ` - unpack in 128-bit lanes (like `mzip`) +- `shuf16Hi`, `shuf16Lo` - 16-bit shuffles with immediate +- `shufHalves` + +## NEON-specific + +### folds +- `fold_add{x:V} : eltype{V}` +- `fold_addw{x:V} : ty_dbl{eltype{V}}` +- `fold_max{x:V} : eltype{V}` +- `fold_min{x:V} : eltype{V}` +- `vfold{F, x:V} : eltype{V}` for F in `min`, `max`, `__add` + +### arithmetic +- `addp` - add pairwise +- `addpw` - add pairwise, widening +- `addpwa` - add pairwise, widening; accumulate +- `addwLo` - widening add +- `subwLo` - widening subtract +- `mulwLo`, `mulwHi`, `mulw` - widening multiply +- `andnz` - element-wise (a&b)!=0? ~0 : 0 + +- `bitAll` - are all bits 1s +- `bitAny` - is any bit a 1 +- `bitBlend` - blend by bits +- `clz` - count leading zeroes +- `cls` - count leading sign bits +- `copyLane` +- `mla` - multiply-add +- `mls` - multiply-subtract +- `ornot` - a|~b +- `packLo`, packHi +- `popc` - popcount of each byte +- `rbit` - reverse bits in bytes +- `rev` - reverse elements in lanes +- `shlm`, shrm - fun shifts with defaults +- `shrn` - narrowing shift right +- `trn1`, `trn2` - 2×2 transposes from pairs of elements across the two arguments + + +## mask.singeli + +- `maskOf{TU, n}` - get a mask of type `TU` whose first `n` elements are all `1`s, and the remaining are `0`s. `0≤n≤vcount{TU}` +- `maskNone` - a mask generator that matches all items +- `maskAfter{n}` - a mask generator that patches the first `n` items +- `loadBatch{p:*E, n, V}` - load the `n`-th batch of `vcount{V}` elements from `*E`; if `E` isn't `eltype{V}`, the result is sign- or zero-extended. +- `storeBatch{p:*E, n, x:V, M}` - store equivalent of the `loadBatch`; if `E` isn't `eltype{V}`, `x` is narrowed via `narrow{}` + +To test whether a mask object `M` is `maskNone` or `maskAfter`, `M{0}` can be used - `maskNone{0}` is `0`, but `maskAfter{n}{0}` is `1`. + +### Loops + +- `@maskedLoop{bulk}` - loop that generates its body twice, once with a `maskNone` mask, and once with a `maskAfter{n}` one to handle the tail. + +- `@muLoop{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end. + Unrolling is handled by passing a tuple of indices to process as the index variable (the tail generated bodies get a tuple of the one index) + +- `@muLoop{bulk, unr, fromunr}` - `fromunr` is additionally ran after exiting the unrolled loop (such that you can have separate accumulators for the unrolled bit, and convert them to single-vector acccumulators for the tail). + + +Tuples can be used in the iterated variable list for various things: +``` +p:*T - regular pointer +tup{VT,p:*T} - loadBatch/storeBatch vector data +tup{'b',p:P} - b_getBatch (bits.singeli) +tup{'b',VT,p:P} - loadBatchBit (bits.singeli) +tup{'g',VT,p:*T} - gives a generator, used as g{} to loadBatch and g{newValue} to storeBatch +tup{'g',p:*T} - the above, but without load support +'m' - get the mask object - either maskNone or some maskAfter{n} +``` + +Stores via those will implicitly do a masked store when required. + +Loads will load past the end, so `M in 'm'` must be used to mask off elements if they're used for something other than the above stores. + +For `muLoop`, `M in 'm'` still gives a single generator, but it's only ever `maskAfter{n}` when there's only one index. + +Example usage for a loop that adds `u16` and bit boolean elements to a `u32` accumulator, early-exiting on overflow: +``` +fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop + def bulk = 8 + def VT = [8]u32 + + @maskedLoop{bulk}( + r in tup{VT, r}, + x in tup{VT, x}, + b in tup{'b', VT, bits}, + M in 'm' + over i to len + ) { + # x, r, and b now have type VT + r1:= r + x - b # subtracting b because it's all 1s for a one + if (homAny{M{r1 < r}}) return{0} # detect overflow + r = r1 # will be mask-stored if needed + } + 1 +} + +fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x unrolled + def bulk = 8 + def VT = [8]u32 + + @muLoop{bulk,2}( + r in tup{'g', VT, r}, + x in tup{VT, x}, + b in tup{'b', VT, bits}, + M in 'm' + over i to len + ) { + def r0 = r{} + def r1 = each{-, each{+,r0,x}, b} + if (homAny{M{tree_fold{|, each{<,r1,r0}}}}) return{0} + r{r1} + } + 1 +} +``` + + \ No newline at end of file diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index efdaa693..eabe0c1a 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -19,7 +19,7 @@ def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f6 def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v} def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v} -def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*v_half{eltype{P}} ~~ ptr, w}} +def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}} def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr} def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}} @@ -91,9 +91,9 @@ def floor{a:[4]f64} = emit{[4]f64, '_mm256_floor_pd', a} def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion -def half{x:T, i & w256{T} & knum{i}} = v_half{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} -def half{x:T, i==0 & w256{T}} = v_half{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} -def pair{a:T,b:T & width{T}==128} = v_dbl{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} +def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} +def half{x:T, i==0 & w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} +def pair{a:T,b:T & width{T}==128} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x} def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 484d49aa..65a52263 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -156,17 +156,17 @@ def widen{T==[ 4]u64, x:X & X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; de def narrow{T, x:X & w256i{X,32} & width{T}==8} = { a:= packQ{x, x} b:= packQ{a, a} - to_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}} + re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}} } -def narrow{T, x:X & w256i{X,32} & width{T}==16} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} -def narrow{T, x:X & w256i{X,16} & width{T}== 8} = to_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X & w256i{X,32} & width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} +def narrow{T, x:X & w256i{X,16} & width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}} def narrow{T, x:X & w256f{X,64} & T>1<<2) | (iota{32}&1)}}} -def narrow{T, x:X & w256u{X,64} & T== u8} = to_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}} +def narrow{T, x:X & w256u{X,64} & T==u32} = re_el{T, sel{[8]i32, x, make{[8]i32, 2*iota{8}}}} +def narrow{T, x:X & w256u{X,64} & T==u16} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}} +def narrow{T, x:X & w256u{X,64} & T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}} def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index d056f53f..020b8789 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -27,6 +27,8 @@ def load{p:P & isptr{P}} = load{p, 0} def store{p:P, v:V & isptr{P} & eltype{P}==v} = store{p, 0, v} def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} def storeu{p:T, v:eltype{T} & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} +def loadu{p:T & elwidth{T}==8} = load{p} +def storeu{p:T, v:eltype{T} & elwidth{T}==8} = store{p, v} def reinterpret{T, x:X & T==X} = x def exportN{f, ...ns} = each{{n} => export{n, f}, ns} @@ -126,35 +128,37 @@ def tern{c, t:T, f:T & anyInt{c}} = { res } -# reinterpret vector element type to signed/unsigned -def ty_s{w} = primtype{'i', w} -def ty_u{w} = primtype{'u', w} - -def ty_u{T & isprim{T}} = ty_u{width{T}} -def ty_s{T & isprim{T}} = ty_s{width{T}} - -def ty_u{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}}) -def ty_s{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}}) -def ty_u{x:T} = ty_u{T}~~x -def ty_s{x:T} = ty_s{T}~~x - -# reinterpret vector as one with element type E -def to_el{E, T & isvec{T}} = [width{T}/width{E}]E -def to_el{E, x:T} = to_el{E,T} ~~ x - -# change width of primitive type def to_w{T, w} = primtype{quality{T}, w} -# double/halve primitive or element type width -def ty_dbl {T & isprim{T}} = to_w{T, width{T}*2} -def ty_half{T & isprim{T}} = to_w{T, width{T}/2} -def ty_dbl {T & isvec{T}} = to_el{ty_dbl {eltype{T}}, T} -def ty_half{T & isvec{T}} = to_el{ty_half{eltype{T}}, T} -# double/halve vector count -def v_dbl {T & isvec{T}} = [vcount{T}*2](eltype{T}) -def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T}) -# double element width without changing count -def el_dbl {T & isvec{T}} = [vcount{T}](ty_dbl{eltype{T}}) -def el_half{T & isvec{T}} = [vcount{T}](ty_half{eltype{T}}) + +def re_el{E, V} = [width{V}/width{E}]E +def re_el{E, x:V} = re_el{E,V} ~~ x + +local def qualChange{q} = { + def f{w & knum{w}} = primtype{q, w} + def f{T & isprim{T}} = primtype{q, width{T}} + def f{T & isvec{T}} = re_el{f{eltype{T}}, T} + def f{x:T} = f{T}~~x +} +def ty_u = qualChange{'u'} +def ty_s = qualChange{'i'} +def ty_f = qualChange{'f'} + +def w_n{T, w & isprim{T}} = primtype{quality{T}, w} +def w_d{T & isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width +def w_h{T & isprim{T}} = to_w{T, width{T}/2} + +def n_d{T & isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count +def n_h{T & isvec{T}} = [vcount{T}/2](eltype{T}) + +def el_d{T & isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count +def el_h{T & isvec{T}} = [vcount{T}](w_h{eltype{T}}) + +def el_m{T & isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width +def el_s{T & isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x + + + + # test if vector has a specific width & element type def lvec{T, n, w} = 0 @@ -169,15 +173,15 @@ def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1 def { abs,andAllZero,andnz,b_getBatch,clmul,cvt,extract,floor,fold_addw,half, homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, - loadLow,make,mul12,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, + loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow, topBlend,topMask,topMaskStore,topMaskStoreF,unord,unpackQ,vfold,widen, zip,zipHi,zipLo } -def mzip {a:T, b:T} = each{{v}=>ty_dbl{T}~~v, zip{a, b}} -def mzipLo{a:T, b:T} = ty_dbl{T} ~~ zipLo{a, b} -def mzipHi{a:T, b:T} = ty_dbl{T} ~~ zipHi{a, b} +def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}} +def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b} +def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b oper &~ andnot infix none 35 def widen{T, x:X & T==X} = x @@ -281,10 +285,6 @@ def forUnroll{exp,unr}{vars,begin,end,iter} = { } } } -def forXUnroll{unr}{vars,begin,end,iter} = { - @forUnroll{unr}(is from begin to end) each{{i} => iter{i, vars}, is} -} - def makeBranch{Ts, F} = { def args = undef{Ts} diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index b21f36de..a7ef5df2 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -23,7 +23,7 @@ def for_vec_overlap{vl}{vars,begin==0,n,iter} = { } # Shift as u16, since x86 is missing 8-bit shifts -def shr16{v:V, n} = V~~(to_el{u16, v} >> n) +def shr16{v:V, n} = V~~(re_el{u16, v} >> n) # Forward or backwards in-place max-scan # Assumes a whole number of vectors and minimum 0 @@ -69,7 +69,7 @@ if (hasarch{'AVX2'}) { {i} => sel{H, v, i} } def getsel{v:V & lvec{V, 32, 8}} = { - def H = v_half{V} + def H = n_h{V} vtop := V**(vcount{V}/2) hs := each{bind{shuf, [4]u64, v}, tup{4b3232, 4b1010}} {i} => homBlend{...each{{h}=>sel{H,h,i}, hs}, V~~i 16) { - b := V~~(to_el{u16, i0} << (7 - lb{vl/2})) + b := V~~(re_el{u16, i0} << (7 - lb{vl/2})) ind = topBlend{ind, isel{ui1}, b} if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} } @@ -260,7 +260,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { tup{wv, wv2} = each{un, tr_half{wv, un{load{wg, 1}}}} } } - def ms{v}{h} = getsel{to_el{I, if (lanes) half{v,h} else tupsel{h,v}}} + def ms{v}{h} = getsel{re_el{I, if (lanes) half{v,h} else tupsel{h,v}}} def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef' def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef' # Offset at end @@ -276,7 +276,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { if (this) @for_vec_overlap{vl} (j to xn) { xv:= load{*V~~(x+j), 0} s := U**bb{iota{isub}} # Select sequential bytes within each U - def ltx{se,ind} = lt{xv, V~~se{to_el{I,ind}}} + def ltx{se,ind} = lt{xv, V~~se{re_el{I,ind}}} @unroll (j to klog) { m := s | tupsel{klog-1-j,bits} s = homBlend{m, s, ltx{selw, m}} @@ -295,7 +295,7 @@ def bin_search_vec{T, up, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = { r -= off rn := if (T==i8) r else if (T==i16) half{narrow{u8, r}, 0} - else extract{to_el{i64, narrow{u8, r}}, 0} + else extract{re_el{i64, narrow{u8, r}}, 0} store{*type{rn}~~(*i8~~rp+j), 0, rn} } } diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index ce0fde1c..82765fbc 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -75,7 +75,7 @@ def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = { def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { b:= make{T, 1< ty_s{v ^ (v>>15)}, rp} - def RU = to_el{u16,T} + def RU = re_el{u16,T} tup{packQ{rp}, tup{'~andAllZero', RU~~tree_fold{|, bad}, RU**0xff80}} } } @@ -61,8 +61,8 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i16==eltype{T} & hasar tup{rl, tup{'anyne', rh, rl>>15}} } def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = { - max:= to_el{f32, (ty_u{T})**0x4efffffe} - def cf32{x:X} = emit{to_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} + max:= re_el{f32, (ty_u{T})**0x4efffffe} + def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x} f32mul:= cf32{w} * cf32{x} tup{w*x, tup{'homAny', M{abs{f32mul} >= max}}} # TODO fallback to the below if the above fails @@ -70,7 +70,7 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasar # def wp = unpackQ{w, T**0} # def xp = unpackQ{x, T**0} # def rp = each{mul32, wp, xp} - # def T2 = to_el{i64, T} + # def T2 = re_el{i64, T} # def bad = each{{v} => { # ((T2~~v + T2**0x80000000) ^ T2**(cast{i64,1}<<63)) > T2**cast_i{i64, (cast{u64,1}<<63) | 0xFFFFFFFF} # }, rp} @@ -78,7 +78,7 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasar } def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = { - def r12 = mul12{w, x} + def r12 = mulw{w, x} rl:= packLo{r12} rh:= packHi{r12} tup{rl, tup{'homAny', M{rh != (rl >> (elwidth{T}-1))}}} @@ -110,7 +110,7 @@ def runner{u, R, F} = { def run{F, M, w:VW, x:VX & c & R==u32 & (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1 r:= F{ty_u{w}, ty_u{x}} - tup{to_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} + tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}} } run } diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index a3a1043e..0a8afb45 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -48,7 +48,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { } else { # bitarr ≡ i8/i16/i32arr def T = [bulk]X def sh{c} = c << (width{X}-1) - def sh{c & X==u8} = T ~~ (to_el{u16,c}<<7) + def sh{c & X==u8} = T ~~ (re_el{u16,c}<<7) def mask{x:X & hasarch{'X86_64'}} = topMask{x} def mask{x:X & hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 557b8840..7a862627 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -48,7 +48,7 @@ def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n def storeBatch{ptr:P, n, x:T, M} = { def rpos = ptr + n*vcount{T} def E0 = eltype{P} - def TF = to_el{E0, T} + def TF = re_el{E0, T} xu:= narrow{E0, x} if (M{0}) homMaskStoreF{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}} @@ -60,7 +60,7 @@ def loadBatch{ptr:P, n, T} = { def rpos = ptr + n*vcount{T} def E0 = eltype{P} - widen{T, loadLow{*to_el{E0, T} ~~ rpos, vcount{T}*width{E0}}} + widen{T, loadLow{*re_el{E0, T} ~~ rpos, vcount{T}*width{E0}}} } def loadBatch {ptr:P, ns, T & istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns} @@ -73,13 +73,6 @@ def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got def hCast{T,p:P & same{T,eltype{P}}} = p def hCast{T,p:P & same{P,*void}} = *T~~p -# masked loop arguments -# p:*T - regular pointer, loaded by the batch index(es) -# tup{VT,p:*T} - loadBatch/storeBatch vector data -# tup{'b',p:P} - b_getBatch -# tup{'b',VT,p:P} - loadBatchBit -# tup{'g',VT,p:*T} - gives a generator supporting g{} for loadBatch and g{newValue} for storeBatch -# tup{'g',p:*T} - the above, but without load support def mlExec{i, iter, vars0, bulk, M} = { def vproc2{T,p:P} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} def vproc2{S=='b', p:P} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 4e558e6a..8aedf7d8 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -42,16 +42,15 @@ def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, def adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} def subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b} -def addw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vaddl', T}, a, b} -def subw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vsubl', T}, a, b} -def mulw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vmull', T}, a, b} - -def mulw2{a:T,b:T & w128i{T}} = emit{ty_dbl{T}, ntyp0{'vmull_high', T}, a, b} -def mul12{a:T,b:T & w128{T}} = tup{mulw{half{a,0}, half{b,0}}, mulw2{a,b}} +def addwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b} +def subwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b} +def mulwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b} +def mulwHi{a:T,b:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b} +def mulw {a:T,b:T & w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}} def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s0 & s8} = { def H=el_half{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed +def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a< elwidth{X}*2} = widen{T, widen{ty_half{T}, x}} +def widen{T, x:X & w64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}} def widen{T, x:X & w64{X} & isfloat{eltype{T}}!=isfloat{eltype{X}} & elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}} def widen{T, x:X & w128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} -def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}} -def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{el_half{X}, ntyp0{'vmovn', X}, x} +def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}} +def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x} def narrow{T, x:X & w128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T} c == VT**e} } -def isNegZero{x:T} = to_el{u64,x} == to_el{u64, T ** -f64~~0} +def isNegZero{x:T} = re_el{u64,x} == re_el{u64, T ** -f64~~0} fn searchNormalizable{}(x:*f64, len:u64) : u64 = { search{f64, x, len, {c:VT} => isNegZero{c} | (c!=c)} } diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 2cba92db..dfffe487 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -59,7 +59,7 @@ def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { @maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) { cw:= wrapChk{cw0, VI,xlf, M} is:= (if (ext>1) i<>width{S}}) + def make_top{S} = re_el{S,U}**(if (T>width{S}}) top := each{make_top, replicate{{S}=>S