Merge pull request #108 from mlochbaum/new-singeli

Singeli syntax updates
This commit is contained in:
dzaima 2024-05-10 20:56:17 +03:00 committed by GitHub
commit b2eb26c2e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 650 additions and 650 deletions

View File

@ -148,10 +148,10 @@ Some may also support one scalar argument or arguments with different widths.
<!-- -->
- `zip{a:T,b:T} : tup{T, T}` - `zip{[0,1,2,3], [4,5,6,7]} → tup{[0,4,1,5], [2,6,3,7]}`
- `mzip{a:T,b:T} : tup{el_dbl{T}, el_dbl{T}}` - reinterpreted `zip{a, b}`
- `zipLo{a:T,b:T} : T` - `tupsel{0, zip{a, b}}`
- `zipHi{a:T,b:T} : T` - `tupsel{1, zip{a, b}}`
- `mzipLo{a:T,b:T} : T` - `tupsel{0, mzip{a, b}}`
- `mzipHi{a:T,b:T} : T` - `tupsel{1, mzip{a, b}}`
- `zipLo{a:T,b:T} : T` - `select{zip{a, b}, 0}`
- `zipHi{a:T,b:T} : T` - `select{zip{a, b}, 1}`
- `mzipLo{a:T,b:T} : T` - `select{mzip{a, b}, 0}`
- `mzipHi{a:T,b:T} : T` - `select{mzip{a, b}, 1}`
## Mask stuff
@ -353,4 +353,4 @@ bmi2.singeli
clmul.singeli
clmul
-->
-->

View File

@ -1,54 +1,54 @@
# compact casting for the annoying intrinsic type system
def v2i{x:T & w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x
def v2f{x:T & w256{T}} = [8]f32 ~~ x
def v2d{x:T & w256{T}} = [4]f64 ~~ x
def v2i{x:T if w256{T}} = if(isint{eltype{T}}) x else [32]u8 ~~ x
def v2f{x:T if w256{T}} = [8]f32 ~~ x
def v2d{x:T if w256{T}} = [4]f64 ~~ x
def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
def undefPromote{T, x:X if w128{X} and w256{T} and eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
# load & store
def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}}
def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr}
def loadLow{ptr:P, w if w256{eltype{P}} and w<=128} = undefPromote{eltype{P}, loadLow{*n_h{eltype{P}} ~~ ptr, w}}
def loadLow{ptr:P, w if w256{eltype{P}} and w==256} = load{ptr}
def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x}
def storeLow{ptr:P, w, x:T if w256{T} and w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:P, w, x:T if w256{T} and w==256} = store{*T~~ptr, 0, x}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
local def f64cmpAVX{a,b,n} = [4]u64 ~~ emit{[4]f64, '_mm256_cmp_pd', a, b, n}
def unord{a:T,b:T & T==[8]f32} = f32cmpAVX{a,b,3}
def unord{a:T,b:T & T==[4]f64} = f64cmpAVX{a,b,3}
def unord{a:T,b:T if T==[8]f32} = f32cmpAVX{a,b,3}
def unord{a:T,b:T if T==[4]f64} = f64cmpAVX{a,b,3}
# f32 arith
def rsqrtE{a:[8]f32} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcpE{a:[8]f32} = emit{[8]f32, '_mm256_rcp_ps', a}
def rsqrtE{a:([8]f32)} = emit{[8]f32, '_mm256_rsqrt_ps', a}
def rcpE{a:([8]f32)} = emit{[8]f32, '_mm256_rcp_ps', a}
# conversion
def half{x:T, i & w256{T} & knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
def half{x:T, i==0 & w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def pair{a:T,b:T & width{T}==128} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
def half{x:T, i if w256{T} and knum{i}} = n_h{T} ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
def half{x:T, i==0 if w256{T}} = n_h{T} ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def pair{a:T,b:T if width{T}==128} = n_d{T} ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
def widen{T==[4]f64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x}
def widen{T==[4]f64, x:X & X==[4]f32} = emit{T, '_mm256_cvtps_pd', x}
def widen{T==[4]f64, x:X & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T, x:X & w256{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
def widen{T==[4]f64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_pd', x}
def widen{T==[4]f64, x:X if X==[4]f32} = emit{T, '_mm256_cvtps_pd', x}
def widen{T==[4]f64, x:X if w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T, x:X if w256{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}}
# structural operations
def topBlend{f:T, t:T, m:M & w256{T,32} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & w256{T,64} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}
def topBlend{f:T, t:T, m:M if w256{T,32} and w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M if w256{T,64} and w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def homBlend{f:T, t:T, m:M if w256{T}} = topBlend{f, t, m}
# mask stuff
def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
def andAllZero{x:T, y:T if w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
def topMask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def topMask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
def homMask{x:T & w256{T}} = topMask{x}
def topMask{x:T if w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def topMask{x:T if w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
def homMask{x:T if w256{T}} = topMask{x}
def homAny{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
def homAll{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
def homAny{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
def homAll{x:T if w256i{T} and elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
def topAny{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} != 0
def topAll{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} != 0
def topAll{x:T if w256i{T} and elwidth{T}>=32} = topMask{x} == (1<<vcount{T})-1

View File

@ -1,97 +1,97 @@
# questionable pack
def unpackQ{a:[32]i8, b:[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:[16]i16, b:[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
def unpackQ{a:[ 8]i32, b:[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
def unpackQ{a:[ 4]i64, b:[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}}
def unpackQ{a:T,b:T if T==[32]i8 } = { tup{emit{[16]i16, '_mm256_unpacklo_epi8', a, b}, emit{[16]i16, '_mm256_unpackhi_epi8', a, b}}}
def unpackQ{a:T,b:T if T==[16]i16} = { tup{emit{[ 8]i32, '_mm256_unpacklo_epi16', a, b}, emit{[ 8]i32, '_mm256_unpackhi_epi16', a, b}}}
def unpackQ{a:T,b:T if T==[ 8]i32} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi32', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi32', a, b}}}
def unpackQ{a:T,b:T if T==[ 4]i64} = { tup{emit{[ 4]i64, '_mm256_unpacklo_epi64', a, b}, emit{[ 4]i64, '_mm256_unpackhi_epi64', a, b}}}
# inverse of questionable pack; these saturate the argument
def packQ{a:T,b:T & T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T & T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T & T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
def packQ{a:T,b:T if T==[16]i16} = emit{[32]i8, '_mm256_packs_epi16', a, b}
def packQ{a:T,b:T if T==[ 8]i32} = emit{[16]i16, '_mm256_packs_epi32', a, b}
def packQ{a:T,b:T if T==[16]u16} = emit{[32]u8, '_mm256_packus_epi16', a, b}
def packQ{a:T,b:T if T==[ 8]u32} = emit{[16]u16, '_mm256_packus_epi32', a, b}
# super questionable pack - assumes high halves are zero
def packQQ{a:T,b:T & T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{a:T,b:T if T==[4]i64} = emit{[8]i32, '_mm256_shuffle_epi32', a, 4b1120} | emit{[8]i32, '_mm256_shuffle_epi32', b, 4b2011}
def packQQ{{a, b}} = packQQ{a, b}
# arith
def mulh {a:T,b:T & [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b}
def mulh {a:T,b:T & [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b}
def mul32{a:T,b:T & [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
def mul32{a:T,b:T & [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
def mulh {a:T,b:T if [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b}
def mulh {a:T,b:T if [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b}
def mul32{a:T,b:T if [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments
def mul32{a:T,b:T if [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments
# structural operations
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
def shl{S==[16]u8, x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n if w256{T} and knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
def blend{L==[4]u64, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
def blend{L==[8]u16, a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
def blend{L==[8]u32, a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
def blend{L==[4]u64, a:T, b:T, m if w256{T} and knum{m}} = T ~~ emit{[ 4]f64, '_mm256_blend_pd', v2d{a}, v2d{b}, m}
def topBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def homBlend{f:T, t:T, m:M & w256{T, 8} & w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & w256{T, 16} & w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
def topBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def homBlend{f:T, t:T, m:M if w256{T, 8} and w256i{M, 8}} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M if w256{T, 16} and w256i{M,16}} = T ~~ topBlend{[32]i8~~f, [32]i8~~t, [32]i8~~m}
def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_permute2x128_si256', v2i{x}, v2i{y}, n}
def shuf{L, x:T, n if lvec{L,4,32} and w256{T} and knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n if lvec{L,4,64} and w256{T} and knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
def shufHalves{x:T, y:T, n if w256{T} and knum{n}} = T ~~ emit{[4]i64, '_mm256_permute2x128_si256', v2i{x}, v2i{y}, n}
def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
def sel{L, x:T, i:I if w256{T} and lvec{L,8,32} and w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
def sel{L, x:T, i:I if w256{T} and lvec{L,16,8} and w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
# masked store; F variants may not be a single instruction
def topMaskStore{p:P, m:M, v:T & w256i{M, 32} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T & w256i{M, 64} & w256{T,elwidth{M}} & eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:P, m:M, v:T & w256i{M} & w256{T,elwidth{M}} & eltype{P}==T} = topMaskStore{p, m, v}
def topMaskStore{p:P, m:M, v:T if w256i{M, 32} and w256{T,elwidth{M}} and eltype{P}==T} = emit{void, '_mm256_maskstore_epi32', *i32~~p, m, [8]i32~~v}
def topMaskStore{p:P, m:M, v:T if w256i{M, 64} and w256{T,elwidth{M}} and eltype{P}==T} = emit{void, '_mm256_maskstore_pd', *f64~~p, m, [4]f64~~v}
def homMaskStore{p:P, m:M, v:T if w256i{M} and w256{T,elwidth{M}} and eltype{P}==T} = topMaskStore{p, m, v}
def topMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T & w256i{M} & elwidth{T}<=16 & w256{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def topMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}>=32} = topMaskStore{p,m,v}
def homMaskStoreF{p:P, m:M, v:T if w256i{M} and elwidth{T}<=16 and w256{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
# mask stuff
def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
def topMask{x:T & w256{T, 16}} = {
def topMask{x:T if w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
def topMask{x:T if w256{T, 16}} = {
msk:u32 = topMask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
(msk&255) | (msk>>8)
}
def homAny{x:T & w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
def homAll{x:T & w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
def topAny{x:T & w256i{T}} = topMask{x} != 0
def topAll{x:T & w256i{T}} = topMask{x} == (1<<vcount{T})-1
def homMask{a:T, b:T & w256i{T,16}} = homMask{shuf{[4]u64, packQ{ty_s{a},ty_s{b}}, 4b3120}}
def homAny{x:T if w256i{T}} = ~emit{u1, '_mm256_testz_si256', v2i{x}, v2i{x}}
def homAll{x:T if w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
def topAny{x:T if w256i{T}} = topMask{x} != 0
def topAll{x:T if w256i{T}} = topMask{x} == (1<<vcount{T})-1
def homMask{a:T, b:T if w256i{T,16}} = homMask{shuf{[4]u64, packQ{ty_s{a},ty_s{b}}, 4b3120}}
def topAny{x:T & w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
def topAny{x:T & w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
def homAny{x:T & w256i{T} & elwidth{T}>=32} = topAny{x}
def topAny{x:T if w256i{T,32}} = ~emit{u1, '_mm256_testz_ps', v2f{x}, v2f{x}}
def topAny{x:T if w256i{T,64}} = ~emit{u1, '_mm256_testz_pd', v2d{x}, v2d{x}}
def homAny{x:T if w256i{T} and elwidth{T}>=32} = topAny{x}
def topAny{x:T & w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0}
def topAll{x:T & w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}
def topAny{x:T if w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0}
def topAll{x:T if w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}
# conversion
def widen{T==[16]u16, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x}
def widen{T==[ 8]u32, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x}
def widen{T==[ 8]u32, x:X & X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X & X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x}
def widen{T==[ 4]u64, x:X & X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X & X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x}
def widen{T==[ 4]u64, x:X & X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X & X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x}
def widen{T==[ 4]u64, x:X & X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X & X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x}
def widen{T==[16]u16, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi16', x}; def widen{T==[16]i16, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi16', x}
def widen{T==[ 8]u32, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi32', x}; def widen{T==[ 8]i32, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi32', x}
def widen{T==[ 8]u32, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi32', x}; def widen{T==[ 8]i32, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi32', x}
def widen{T==[ 4]u64, x:X if X==[16]u8} = emit{T, '_mm256_cvtepu8_epi64', x}; def widen{T==[ 4]i64, x:X if X==[16]i8} = emit{T, '_mm256_cvtepi8_epi64', x}
def widen{T==[ 4]u64, x:X if X==[8]u16} = emit{T, '_mm256_cvtepu16_epi64', x}; def widen{T==[ 4]i64, x:X if X==[8]i16} = emit{T, '_mm256_cvtepi16_epi64', x}
def widen{T==[ 4]u64, x:X if X==[4]u32} = emit{T, '_mm256_cvtepu32_epi64', x}; def widen{T==[ 4]i64, x:X if X==[4]i32} = emit{T, '_mm256_cvtepi32_epi64', x}
def narrow{T, x:X & w256i{X,32} & width{T}==8} = {
def narrow{T, x:X if w256i{X,32} and width{T}==8} = {
a:= packQ{x, x}
b:= packQ{a, a}
re_el{T, sel{[8]u32, b, make{[8]i32, 0,4,0,4,0,4,0,4}}}
}
def narrow{T, x:X & w256i{X,32} & width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
def narrow{T, x:X & w256i{X,16} & width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
def narrow{T, x:X if w256i{X,32} and width{T}==16} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
def narrow{T, x:X if w256i{X,16} and width{T}== 8} = re_el{T, shuf{[4]u64, packQ{x, x}, 4b3120}}
def narrow{T, x:X & w256f{X,64} & T<i32} = narrow{T, narrow{i32, x}}
def narrow{T, x:X & w256f{X,64} & T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def narrow{T, x:X if w256f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
def narrow{T, x:X if w256f{X,64} and T==i32} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def narrow{T, x:X & w256u{X,64} & T==u32} = re_el{T, sel{[8]i32, x, make{[8]i32, 2*iota{8}}}}
def narrow{T, x:X & w256u{X,64} & T==u16} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}}
def narrow{T, x:X & w256u{X,64} & T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}}
def narrow{T, x:X if w256u{X,64} and T==u32} = re_el{T, sel{[8]i32, x, make{[8]i32, 2*iota{8}}}}
def narrow{T, x:X if w256u{X,64} and T==u16} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, (iota{32}>>1<<2) | (iota{32}&1)}}}
def narrow{T, x:X if w256u{X,64} and T== u8} = re_el{T, sel{[16]i8, narrow{u32,x}, make{[32]i8, 4*iota{32}}}}
def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
def cvt2{T, x:X if T==i32 and X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def cvt2{T, x:X if T==f64 and X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}

View File

@ -4,30 +4,30 @@ local {
if (isfloat{T}) (if (width{T}==32) 'ps' else 'pd')
else merge{'epi', fmtnat{width{T}}}
}
def suf{V & isvec{V}} = suf{eltype{V}}
def suf{V if isvec{V}} = suf{eltype{V}}
def pref{w} = merge{'_mm', if (w==128) '' else fmtnat{w}, '_'}
def pref{V & isvec{V}} = pref{width{V}}
def pref{V if isvec{V}} = pref{width{V}}
}
local def re_mask{M, sub} = {
def l = vcount{M}; def w = max{32,l}
sub{fmtnat{l}, fmtnat{w}, ty_u{w}}
}
def reinterpret{M, a:T & ismask{M} & width{T}==width{M}} = {
def reinterpret{M, a:T if ismask{M} and width{T}==width{M}} = {
re_mask{M, {l,w,W} => emit{M, merge{'_cvtu',w,'_mask',l}, promote{W, a}}}
}
def reinterpret{T, a:M & ismask{M} & width{T}==width{M}} = {
def reinterpret{T, a:M if ismask{M} and width{T}==width{M}} = {
re_mask{M, {l,w,W} => cast_i{T, emit{W, merge{'_cvtmask',l,'_u',w}, a}}}
}
def maskStore{p:*V, m:M, v:V & ismask{M} & isvec{V} & vcount{M}==vcount{V}} = {
def maskStore{p:*V, m:M, v:V if ismask{M} and isvec{V} and vcount{M}==vcount{V}} = {
emit{void, merge{pref{V}, 'mask_storeu_', suf{V}}, p, m, v}
}
def topMaskReg{x:V} = emit{[vcount{V}]u1, merge{pref{V},'mov',suf{V},'_mask'}, x}
def topMask{x:T & isvec{T} & 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x}
def homMask{x:T & isvec{T} & 512==width{T}} = topMask{x}
def topMask{x:T if isvec{T} and 512==width{T}} = ty_u{vcount{T}}~~topMaskReg{x}
def homMask{x:T if isvec{T} and 512==width{T}} = topMask{x}
def maskToHom{T, x:M & ismask{M} & isvec{T} & vcount{M}==vcount{T}} = {
def maskToHom{T, x:M if ismask{M} and isvec{T} and vcount{M}==vcount{T}} = {
emit{T, merge{pref{T},'movm_',suf{T}}, x}
}

View File

@ -18,40 +18,40 @@ def istup = ktup
def isunsigned{T} = isint{T} & ~issigned{T}
def isvec {T} = 0; def isvec {T & istype{T}} = same{typekind{T},'vector'}
def isprim{T} = 0; def isprim{T & istype{T}} = same{typekind{T},'primitive'}
def isptr {T} = 0; def isptr {T & istype{T}} = same{typekind{T},'pointer'}
def isvec {T} = 0; def isvec {T if istype{T}} = same{typekind{T},'vector'}
def isprim{T} = 0; def isprim{T if istype{T}} = same{typekind{T},'primitive'}
def isptr {T} = 0; def isptr {T if istype{T}} = same{typekind{T},'pointer'}
def elwidth{T} = width{eltype{T}}
oper &~ andnot infix none 35
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
def andnot{a, b if anyNum{a} and anyNum{b}} = a & ~b
def load{p:P, n & isvec{eltype{P}}} = assert{0}
def store{p:P, n, v & isvec{eltype{P}}} = assert{0}
def load{p:P & isptr{P}} = load{p, 0}
# def store{p:P, v & isptr{P}} = store{p, 0, v}
def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p}
def storeu{p:T, v:eltype{T} & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v}
def loadu{p:T & issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p}
def storeu{p:T, v:eltype{T} & issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}}
def loadu{p:T & elwidth{T}==8} = load{p}
def storeu{p:T, v:eltype{T} & elwidth{T}==8} = store{p, v}
def load{p:P, n if isvec{eltype{P}}} = assert{0}
def store{p:P, n, v if isvec{eltype{P}}} = assert{0}
def load{p:P if isptr{P}} = load{p, 0}
# def store{p:P, v if isptr{P}} = store{p, 0, v}
def loadu{p:T if isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p}
def storeu{p:T, v:(eltype{T}) if isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v}
def loadu{p:T if issigned{eltype{T}}} = loadu {*ty_u{eltype{T}} ~~ p}
def storeu{p:T, v:(eltype{T}) if issigned{eltype{T}}} = storeu{*ty_u{eltype{T}} ~~ p, ty_u{v}}
def loadu{p:T if elwidth{T}==8} = load{p}
def storeu{p:T, v:(eltype{T}) if elwidth{T}==8} = store{p, v}
def reinterpret{T, x:X & T==X} = x
def reinterpret{T, x:X if T==X} = x
def exportN{f, ...ns} = each{export{.,f}, ns}
def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} }
def exportT{name, fs} = { v:*type{select{fs,0}} = fs; export{name, v} }
# hints
def rare{x & knum{x}} = x
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
def assert{x & x==0} = assert{'failed assertion'}
def assert{x & x==1} = 1
def rare{x if knum{x}} = x
def rare{x:(u1)} = emit{u1, '__builtin_expect', x, 0}
def assert{x if x==0} = assert{'failed assertion'}
def assert{x if x==1} = 1
def unreachable{} = emit{void, 'si_unreachable'}
def assert{x:u1} = { if (not x) emit{void, 'si_unreachable'} }
def assert{x:(u1)} = { if (not x) emit{void, 'si_unreachable'} }
# various checks
def oneVal{{h, ...t}} = {
@ -65,24 +65,24 @@ def anyNum{x} = isconst{x} | knum{x}
def anyNum{x:T} = isprim{T}
def anyInt{x} = 0
def anyInt{x & knum{x}} = (x>>0) == x
def anyInt{x & isreg{x}|isconst{x}} = isint{x}
def anyInt{x if knum{x}} = (x>>0) == x
def anyInt{x if isreg{x}|isconst{x}} = isint{x}
# vector width/type checks
def w64 {T} = 0; def w64 {T & isvec{T}} = width{T}==64
def w128{T} = 0; def w128{T & isvec{T}} = width{T}==128
def w256{T} = 0; def w256{T & isvec{T}} = width{T}==256
def w64 {T,w} = 0; def w64 {T,w & w64{T}} = elwidth{T}==w
def w128{T,w} = 0; def w128{T,w & w128{T}} = elwidth{T}==w
def w256{T,w} = 0; def w256{T,w & w256{T}} = elwidth{T}==w
def w64 {T} = 0; def w64 {T if isvec{T}} = width{T}==64
def w128{T} = 0; def w128{T if isvec{T}} = width{T}==128
def w256{T} = 0; def w256{T if isvec{T}} = width{T}==256
def w64 {T,w} = 0; def w64 {T,w if w64{T}} = elwidth{T}==w
def w128{T,w} = 0; def w128{T,w if w128{T}} = elwidth{T}==w
def w256{T,w} = 0; def w256{T,w if w256{T}} = elwidth{T}==w
# width+type checks
def genchk{B, F} = {
def r{T} = 0
def r{T & B{T}} = F{eltype{T}}
def r{T if B{T}} = F{eltype{T}}
def r{T,w} = 0
def r{T,w & B{T}} = F{eltype{T}} & (elwidth{T}==w)
def r{T & ~isvec{T}} = 0
def r{T,w if B{T}} = F{eltype{T}} & (elwidth{T}==w)
def r{T if ~isvec{T}} = 0
r
}
def w256i = genchk{w256, isint}; def w128i = genchk{w128, isint}; def w64i = genchk{w64, isint}
@ -92,14 +92,14 @@ def w256f = genchk{w256, isfloat}; def w128f = genchk{w128, isfloat}; de
def trunc{T, x:U & isint{T} & isint{U} & T<=U} = emit{T, '', x}
def trunc{T, x & knum{x}} = cast{T, x}
def trunc{T, x:U if isint{T} and isint{U} and T<=U} = emit{T, '', x}
def trunc{T, x if knum{x}} = cast{T, x}
def tern{c, T, F & anyInt{c}} = {
def tern{c, T, F if anyInt{c}} = {
if(c) T
else F
}
def tern{c, t:T, f:T & anyInt{c}} = {
def tern{c, t:T, f:T if anyInt{c}} = {
res:T = f
if (c) res = t
res
@ -111,47 +111,47 @@ def re_el{E, V} = [width{V}/width{E}]E
def re_el{E, x:V} = re_el{E,V} ~~ x
local def qualChange{q} = {
def f{w & knum{w}} = primtype{q, w}
def f{T & isprim{T}} = primtype{q, width{T}}
def f{T & isvec{T}} = re_el{f{eltype{T}}, T}
def f{w if knum{w}} = primtype{q, w}
def f{T if isprim{T}} = primtype{q, width{T}}
def f{T if isvec{T}} = re_el{f{eltype{T}}, T}
def f{x:T} = f{T}~~x
}
def ty_u = qualChange{'u'}
def ty_s = qualChange{'i'}
def ty_f = qualChange{'f'}
def w_n{T, w & isprim{T}} = primtype{quality{T}, w}
def w_d{T & isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width
def w_h{T & isprim{T}} = to_w{T, width{T}/2}
def w_n{T, w if isprim{T}} = primtype{quality{T}, w}
def w_d{T if isprim{T}} = to_w{T, width{T}*2} # double/halve primitive type width
def w_h{T if isprim{T}} = to_w{T, width{T}/2}
def n_d{T & isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count
def n_h{T & isvec{T}} = [vcount{T}/2](eltype{T})
def n_d{T if isvec{T}} = [vcount{T}*2](eltype{T}) # double/halve vector count
def n_h{T if isvec{T}} = [vcount{T}/2](eltype{T})
def el_d{T & isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count
def el_h{T & isvec{T}} = [vcount{T}](w_h{eltype{T}})
def el_d{T if isvec{T}} = [vcount{T}](w_d{eltype{T}}) # double/halve element width, preserving count
def el_h{T if isvec{T}} = [vcount{T}](w_h{eltype{T}})
def el_m{T & isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width
def el_s{T & isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x
def el_m{T if isvec{T}} = re_el{w_d{eltype{T}}, T}; def el_m{x:T} = re_el{T}~~x # double/halve element width, preserving width
def el_s{T if isvec{T}} = re_el{w_h{eltype{T}}, T}; def el_s{x:T} = re_el{T}~~x
# type stats
def minvalue{T & isunsigned{T}} = 0
def maxvalue{T & isunsigned{T}} = (1<<width{T})-1
def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
def minvalue{T if isunsigned{T}} = 0
def maxvalue{T if isunsigned{T}} = (1<<width{T})-1
def minvalue{T if issigned{T}} = - (1<<(width{T}-1))
def maxvalue{T if issigned{T}} = (1<<(width{T}-1))-1
# tuple operations
def iota{n & knum{n}} = range{n}
def broadcast{T, v & isprim{T}} = v
def broadcast{n, v & knum{n}} = each{{_}=>v, range{n}}
def collect{vars,begin,end,iter & knum{begin} & knum{end}} = {
def iota{n if knum{n}} = range{n}
def broadcast{T, v if isprim{T}} = v
def broadcast{n, v if knum{n}} = each{{_}=>v, range{n}}
def collect{vars,begin,end,iter if knum{begin} and knum{end}} = {
each{iter{., vars}, range{end-begin}+begin}
}
# convert tuple to number in little-endian base b
def base{b,l} = if (0==tuplen{l}) 0 else tupsel{0,l}+b*base{b,slice{l,1}}
def base{b,l} = if (0==length{l}) 0 else select{l,0}+b*base{b,slice{l,1}}
@ -162,7 +162,7 @@ def fast_BMI2{} = if (SLOW_PDEP) 0 else hasarch{'BMI2'}
# test if vector has a specific width & element type
def lvec{T, n, w} = 0
def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1
def lvec{T, n, w if isvec{T} and vcount{T}==n and elwidth{T}==w} = 1
# base cases
def {
@ -176,8 +176,8 @@ def {
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX
def homMask{...vs & tuplen{vs}>1} = {
def n = tuplen{vs}
def homMask{...vs if length{vs}>1} = {
def n = length{vs}
def T = oneType{vs}
def RT = ty_u{max{8,vcount{T}*n}}
def a = promote{RT, homMask{...slice{vs,0,n/2}}}
@ -185,13 +185,13 @@ def homMask{...vs & tuplen{vs}>1} = {
(b<<(n/2 * vcount{T})) | a
}
if (hasarch{'X86_64'}) {
if_inline (hasarch{'X86_64'}) {
include 'arch/iintrinsic/basic'
include './sse2'
include './sse'
include './avx'
include './avx2'
} else if (hasarch{'AARCH64'}) {
} else if_inline (hasarch{'AARCH64'}) {
include 'arch/neon_intrin/basic'
include './neon'
} else {
@ -204,15 +204,15 @@ def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b}
def packQ{{a, b}} = packQ{a, b}
def pair{{a, b}} = pair{a, b}
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = x
def cvt{T, x:X & T==eltype{X}} = x
def widen{T, x:X if T==X} = x
def narrow{T, x:X if T==eltype{X}} = x
def undefPromote{T, x:X if T==X} = x
def cvt{T, x:X if T==eltype{X}} = x
def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}}
def make{T, ...xs & isvec{T}} = vec_make{T, ...xs}
def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}}
def absu{a:T & isvec{T}} = ty_u{abs{a}}
def broadcast{T, v if isvec{T}} = vec_broadcast{T, promote{eltype{T},v}}
def make{T, ...xs if isvec{T}} = vec_make{T, ...xs}
def iota{T if isvec{T}} = make{T, ...iota{vcount{T}}}
def absu{a:T if isvec{T}} = ty_u{abs{a}}
def floor = __floor
def ceil = __ceil
@ -226,29 +226,29 @@ def sqrt = __sqrt
# more arith
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
def min{a, b if anyNum{a} and anyNum{b}} = tern{a<b, a, b}
def max{a, b if anyNum{a} and anyNum{b}} = tern{a>b, a, b}
def cdiv{a,b} = (a+b-1)/b # ceiling divide
def cdiv{a,b & knum{a} & knum{b}} = ceil{a/b}
def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x}
def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x}
def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x}
def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x}
def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x}
def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x}
def cdiv{a,b if knum{a} and knum{b}} = ceil{a/b}
def popc{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_popcountll', x}
def popc{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_popcount', x}
def ctz{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_ctzll', x}
def ctz{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_ctz', x}
def clz{x:T if isint{T} and width{T}==64} = emit{ux, '__builtin_clzll', x}
def clz{x:T if isint{T} and width{T}<=32} = emit{ux, '__builtin_clz', x}
# count-leading-zeros complement, less type-dependent
def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x}
def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x}
def clzc{x:T if isint{T} and width{T}==64} = 64-clz{x}
def clzc{x:T if isint{T} and width{T}<=32} = 32-clz{x}
def ceil_log2{n} = clzc{n-1}
def truncBits{n, v & n<=8} = cast_i{u8, v}
def truncBits{n, v & n==16} = cast_i{u16, v}
def truncBits{n, v & n==32} = cast_i{u32, v}
def truncBits{n, v & n==64} = cast_i{u64, v}
def truncBits{n, v if n<=8} = cast_i{u8, v}
def truncBits{n, v if n==16} = cast_i{u16, v}
def truncBits{n, v if n==32} = cast_i{u32, v}
def truncBits{n, v if n==64} = cast_i{u64, v}
# base-2 log of a constant power of two
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
def lb{n if knum{n} and (n>>1<<1) == n and n>0} = lb{n>>1}+1
def lb{n==1} = 0
def zlow{n,x} = (x >> n) << n # zero out n least significant bits
@ -256,11 +256,11 @@ def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
def bit {k,x} = x & (1<<k) # get the k-th bit
# range logic
def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count)
def inRangeLen{x:TS, start, count if issigned{eltype{TS}}} = { # ∊ [start;start+count)
def TU = ty_u{TS}
(TU~~(x-TS**start)) < TU**count
}
def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = { # ∊ [start;start+count)
def inRangeLen{x:TU, start, count if isunsigned{eltype{TU}}} = { # ∊ [start;start+count)
def TS = ty_s{TU}
def h = 1 << (elwidth{TU}-1)
(TS~~(x-TU**(start-h))) < TS**(count-h)
@ -270,8 +270,8 @@ def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} # ∊ [start;
def load{p,i & kgen{p}} = p{i}
def store{p,i,x & kgen{p}} = p{i,x}
def load{p,i if kgen{p}} = p{i}
def store{p,i,x if kgen{p}} = p{i,x}
def tptr{l,s} = { # create "pointer" generator with given load & store definitions
def r{i} = l{i}
def r{i,x} = s{i,x}
@ -319,17 +319,17 @@ def makeOptBranch{enable, Ts, F} = if (enable) makeBranch{Ts, F} else 'not defin
def tree_fold{F, x} = {
def h = tuplen{x}>>1
F{tree_fold{F, slice{x,0,h}}, tree_fold{F, slice{x,h,tuplen{x}}}}
def h = length{x}>>1
F{tree_fold{F, slice{x,0,h}}, tree_fold{F, slice{x,h,length{x}}}}
}
def tree_fold{F, {x}} = x
def eachx{F, ...args} = {
def l = tree_fold{max, each{{x} => if(ktup{x}) tuplen{x} else 0, args}}
def l = tree_fold{max, each{{x} => if(ktup{x}) length{x} else 0, args}}
each{F, ...each{{x} => if (istup{x}) x else l**x, args}}
}
def undef{T, n & istype{T}} = @collect(n) undef{T}
def undef{Ts & istup{Ts}} = each{undef, Ts}
def undef{T, n if istype{T}} = @collect(n) undef{T}
def undef{Ts if istup{Ts}} = each{undef, Ts}
def undef{x:T} = undef{T}
def undef{T & istype{T}} = { reg:=undefined{T} }
def undef{T if istype{T}} = { reg:=undefined{T} }

View File

@ -40,21 +40,21 @@ fn max_scan{T, up}(x:*T, len:u64) : void = {
}
def getsel{...x} = assert{'shuffling not supported', show{...x}}
if (hasarch{'AVX2'}) {
def getsel{h:H & lvec{H, 16, 8}} = {
if_inline (hasarch{'AVX2'}) {
def getsel{h:H if lvec{H, 16, 8}} = {
sel{H, pair{h,h}, .}
}
def getsel{v:V & lvec{V, 32, 8}} = {
def getsel{v:V if lvec{V, 32, 8}} = {
def H = n_h{V}
vtop := V**(vcount{V}/2)
hs := each{shuf{[4]u64, v, .}, tup{4b3232, 4b1010}}
{i} => homBlend{...each{sel{H,.,i}, hs}, V~~i<vtop}
}
def getsel{v:V & lvec{V, 8, 32}} = sel{V, v, .}
def getsel{v:V if lvec{V, 8, 32}} = sel{V, v, .}
}
# Move evens to half 0 and odds to half 1
def uninterleave{x:V & hasarch{'AVX2'}} = {
def uninterleave{x:V if hasarch{'AVX2'}} = {
def vl = vcount{V}; def bytes = width{eltype{V}}/8
def i = 2*iota{vl/4}
def i2= join{table{+, bytes*merge{i,i+1}, iota{bytes}}}
@ -74,7 +74,7 @@ def get_rtype{len} = {
}
def rtype_arr{gen} = {
def t = each{gen, rtypes}
a:*(type{tupsel{0,t}}) = t
a:*(type{select{t,0}}) = t
}
# Write the last index of v at t+v, for each unique v in w
@ -108,10 +108,10 @@ fn write_indices{I,T}(t:*I, w:*T, n:u64) : void = {
}
setlabel{break}
}
fn write_indices{I,T & width{I}==8}(t:*I, w:*T, n:u64) : void = {
fn write_indices{I,T if width{I}==8}(t:*I, w:*T, n:u64) : void = {
@for (w over j to n) store{t, w, cast_i{I, j+1}}
}
def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
def bins_lookup{I, T, up, w:*T, wn:(u64), x:*T, xn:(u64), rp:(*void)} = {
# Build table
def tc = 1<<width{T}
t0:*I = talloc{I, tc}
@ -129,7 +129,7 @@ def bins_lookup{I, T, up, w:*T, wn:u64, x:*T, xn:u64, rp:*void} = {
tfree{t0}
}
def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = {
def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done if hasarch{'AVX2'}} = {
assert{wn < 128} # Total must fit in i8
def vl = 32
def T = i8
@ -200,7 +200,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = {
}
# Binary search within vector registers
def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn if hasarch{'AVX2'}} = {
def up = prim != '⍒'
def search = (prim=='∊') | (prim=='⊐')
assert{wn > 1}; assert{wn < maxwn}
@ -230,7 +230,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def un = uninterleave
def tr_half{a, b} = each{shufHalves{a,b,.}, tup{16b20, 16b31}}
def un{{a,b}} = tr_half{un{a},un{b}}
if (not lanes) tupsel{1,wv} = load{wg, 1}
if (not lanes) select{wv,1} = load{wg, 1}
wv = un{wv}
if (ex>=2 and wn >= 2*svl) {
assert{lanes} # Different transpose pattern needed
@ -238,7 +238,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
tup{wv, wv2} = each{un, tr_half{wv, un{load{wg, 1}}}}
}
}
def ms{v}{h} = getsel{re_el{I, if (lanes) half{v,h} else tupsel{h,v}}}
def ms{v}{h} = getsel{re_el{I, if (lanes) half{v,h} else select{v,h}}}
def selw = ms{wv}{0}; def selw1 = if (ex>=1) ms{wv}{1} else 'undef'
def selw2 = if (ex>=2) each{ms{wv2}, iota{2}} else 'undef'
# Offset at end
@ -257,7 +257,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
def cmpx{cmp}{se,ind} = cmp{xv, V~~se{re_el{I,ind}}}
def ltx = cmpx{lt}; def eqx = cmpx{==}
@unroll (j to klog) {
m := s | tupsel{klog-1-j,bits}
m := s | select{bits,klog-1-j}
s = homBlend{m, s, ltx{selw, m}}
}
r := if (isub==1) s else s>>(lb{isub}+wd-wi)
@ -295,7 +295,7 @@ def bin_search_vec{prim, T, w:*T, wn, x:*T, xn, rp, maxwn & hasarch{'AVX2'}} = {
}
}
if (hasarch{'AVX2'}) {
if_inline (hasarch{'AVX2'}) {
fn avx2_search_bin{prim, T, maxwn}(rp:*(if (prim=='∊') u64 else i8), w:*void, wn:u64, x:*void, xn:u64) : void = {
bin_search_vec{prim, T, *T~~w, wn, *T~~x, xn, rp, maxwn}
}
@ -345,8 +345,8 @@ def bin_search_branchless{up, w, wn, x, n, res, rtype} = {
fn bins{T, up}(w:*void, wn:u64, x:*void, xn:u64, rp:*void, rty:u8) : void = {
def param = tup{up, *T~~w, wn, *T~~x, xn, rp}
def lookup{k} = {
if (rty == k) bins_lookup{tupsel{k,rtypes}, T, ...param}
else if (k+1 < tuplen{rtypes}) lookup{k+1}
if (rty == k) bins_lookup{select{rtypes,k}, T, ...param}
else if (k+1 < length{rtypes}) lookup{k+1}
}
# For >=8 i8 values, vector bit-table is as good as binary search
def wn_vec = if (T==i8) 8 else 2*256/width{T}
@ -377,7 +377,7 @@ fn saturate{F,T,...up}(dst:*void, src:*void, n:u64) : void = {
def a = minvalue{T}; af := cast_i{F,a}
def b = maxvalue{T}; bf := cast_i{F,b}
@for (d in *T~~dst, xf in *F~~src over n) {
x := if (F==f64) (if (tupsel{0,up}) floor else ceil){xf} else xf
x := if (F==f64) (if (select{up,0}) floor else ceil){xf} else xf
d = cast_i{T, x}
if (x<af) d = a
if (x>bf) d = b

View File

@ -1,26 +1,26 @@
def ones{T, n} = (cast{T,1}<<cast{T,n}) - 1
def b_get{x:*u64, n:(ux)} = {
def b_get{x:(*u64), n:(ux)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatchLo{sz, x:*u64, n:(ux) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
def b_getBatchLo{sz, x:*u64, n:(ux) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
def b_getBatchLo{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2})
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4})
def b_getBatchLo{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatch{sz, x:*u64, n:(ux) & sz==2} = b_getBatchLo{sz, x, n} & 3
def b_getBatch{sz, x:*u64, n:(ux) & sz==4} = b_getBatchLo{sz, x, n} & 15
def b_getBatch{sz, x:*u64, n:(ux) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_getBatch{sz, x:(*u64), n:(ux) if sz==2} = b_getBatchLo{sz, x, n} & 3
def b_getBatch{sz, x:(*u64), n:(ux) if sz==4} = b_getBatchLo{sz, x, n} & 15
def b_getBatch{sz, x:(*u64), n:(ux) if sz>=8} = load{*ty_u{sz}~~x, n}
def b_set{x:*u64, n:(ux), v:u1} = {
def b_set{x:(*u64), n:(ux), v:(u1)} = {
m:u64 = cast{u64,1}<<(n&63)
p:u64 = load{x,n>>6}
if (v) store{x,n>>6,p | m}
else store{x,n>>6,p & ~m}
}
def b_setBatch{sz, x:*u64, n:(ux), v} = {
def b_setBatch{sz, x:(*u64), n:(ux), v} = {
vc:u64 = promote{u64,v}
am:u64 = 64/sz
w:u64 = load{x,n/am}
@ -30,7 +30,7 @@ def b_setBatch{sz, x:*u64, n:(ux), v} = {
store{x, n/am, w}
}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==4} = {
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==4} = {
x8:= *u8 ~~ x
#w:u64 = cast_i{u64, load{x8,n/2}}
@ -49,12 +49,12 @@ def b_setBatch{sz, x:*u64, n:(ux), v & sz==4} = {
store{x8, n/2, cast_i{u8,w}}
}
def b_setBatch{sz, x:*u64, n:(ux), v & sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}}
def b_setBatch{sz, x:*u64, n:(ux), v & sz==64} = store{ x, n, cast_i{u64,v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz== 8} = store{*u8 ~~ x, n, cast_i{u8, v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==16} = store{*u16 ~~ x, n, cast_i{u16,v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==32} = store{*u32 ~~ x, n, cast_i{u32,v}}
def b_setBatch{sz, x:(*u64), n:(ux), v if sz==64} = store{ x, n, cast_i{u64,v}}
def spreadBits{T==[32]u8, a:u32} = {
def spreadBits{T==[32]u8, a:(u32)} = {
def idxs = iota{32}
b:= [8]u32**a
c:= [32]u8~~b
@ -63,41 +63,41 @@ def spreadBits{T==[32]u8, a:u32} = {
e == (d&e)
}
def spreadBits{T==[16]u8, a:u16 & hasarch{'AARCH64'}} = {
def spreadBits{T==[16]u8, a:(u16) if hasarch{'AARCH64'}} = {
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
}
def spreadBits{T==[16]u8, a:u16 & hasarch{'X86_64'}} = {
def spreadBits{T==[16]u8, a:(u16) if hasarch{'X86_64'}} = {
b:= [16]u8~~[8]u16**a
exp:= [16]u8~~shuf{[4]i32, shuf16Lo{mzipLo{b, b}, 4b1100}, 4b1100}
(exp & make{[16]u8, 1<<(iota{16}&7)}) != [16]u8**0
}
def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
def spreadBits{T, a if vcount{T} <= elwidth{T} and quality{eltype{T}}=='u'} = {
b:= make{T, 1<<iota{vcount{T}}}
b == (b & T ~~ re_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}
def loadBatchBit{T, x:*u64, n:(ux)} = { # vector with type T with each element being either all 0s or 1s
def loadBatchBit{T, x:(*u64), n:(ux)} = { # vector with type T with each element being either all 0s or 1s
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
}
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
def loaduBitRaw{x:*u64, i} = {
def loaduBitRaw{x:(*u64), i} = {
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
}
def loaduBit{x:*u64, i, n} = {
def loaduBit{x:(*u64), i, n} = {
assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))}
loaduBitRaw{x, i}
}
def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}}
def loaduBitTrunc{x:(*u64), i, n if knum{n}} = truncBits{n, loaduBit{x, i, n}}
def loadBatchBit{T, x:*u64, is & ktup{is}} = {
# def len = tuplen{is}
def loadBatchBit{T, x:(*u64), is if ktup{is}} = {
# def len = length{is}
# def count = vcount{T}
# assert{count*len <= 64}
# bits:= b_getBatchLo{count*len, x, tupsel{0,is}}
# bits:= b_getBatchLo{count*len, x, select{is,0}}
# @collect(i to len) spreadBits{T, truncBits{count, bits>>(i*count)}}
each{loadBatchBit{T, x, .}, is}
}

View File

@ -1,4 +1,4 @@
def pdep{x:u64, m:u64} = emit{u64, '_pdep_u64', x, m}
def pdep{x:u32, m:u32} = emit{u32, '_pdep_u32', x, m}
def pext{x:u64, m:u64} = emit{u64, '_pext_u64', x, m}
def pext{x:u32, m:u32} = emit{u32, '_pext_u32', x, m}
def pdep{x:(u64), m:(u64)} = emit{u64, '_pdep_u64', x, m}
def pdep{x:(u32), m:(u32)} = emit{u32, '_pdep_u32', x, m}
def pext{x:(u64), m:(u64)} = emit{u64, '_pext_u64', x, m}
def pext{x:(u32), m:(u32)} = emit{u32, '_pext_u32', x, m}

View File

@ -1,13 +1,13 @@
def bcall{T, f, x} = emit{T, 'BCALL', f, x}
def from_B{T, x & T==f64} = bcall{T, 'o2fG', x}
def from_B{T, x & T<=i32 & issigned{T}} = bcall{T, 'o2iG', x}
def from_B{T, x & T<=u32 & isunsigned{T}} = bcall{T, 'o2cG', x}
def from_B{T, x if T==f64} = bcall{T, 'o2fG', x}
def from_B{T, x if T<=i32 and issigned{T}} = bcall{T, 'o2iG', x}
def from_B{T, x if T<=u32 and isunsigned{T}} = bcall{T, 'o2cG', x}
def q_f64{x} = bcall{u1, 'q_f64', x}
def q_chr{x} = bcall{u1, 'q_c32', x}
def q_chr{T,x & T==u8 } = bcall{u1, 'q_c8', x}
def q_chr{T,x & T==u16} = bcall{u1, 'q_c16', x}
def q_chr{T,x & T==u32} = bcall{u1, 'q_c32', x}
def q_chr{T,x if T==u8 } = bcall{u1, 'q_c8', x}
def q_chr{T,x if T==u16} = bcall{u1, 'q_c16', x}
def q_chr{T,x if T==u32} = bcall{u1, 'q_c32', x}
def cbqn_c32Tag{} = emit{u64, '', 'C32_TAG'}
def cbqn_tagTag{} = emit{u64, '', 'TAG_TAG'}
@ -21,18 +21,18 @@ def cbqn_nspTag{} = emit{u64, '', 'NSP_TAG'}
def cbqn_objTag{} = emit{u64, '', 'OBJ_TAG'}
def cbqn_arrTag{} = emit{u64, '', 'ARR_TAG'}
def cbqn_elType{T & T==u1 } = 0
def cbqn_elType{T & T==i8 } = 1
def cbqn_elType{T & T==i16} = 2
def cbqn_elType{T & T==i32} = 3
def cbqn_elType{T & T==f64} = 4
def cbqn_elType{T & T==u8 } = 5
def cbqn_elType{T & T==u16} = 6
def cbqn_elType{T & T==u32} = 7
def cbqn_elType{T if T==u1 } = 0
def cbqn_elType{T if T==i8 } = 1
def cbqn_elType{T if T==i16} = 2
def cbqn_elType{T if T==i32} = 3
def cbqn_elType{T if T==f64} = 4
def cbqn_elType{T if T==u8 } = 5
def cbqn_elType{T if T==u16} = 6
def cbqn_elType{T if T==u32} = 7
def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'}
def talloc{T, len} = emit{*T, 'TALLOCP', fmt_type{T}, len}
def tfree{ptr} = emit{void, 'TFREE', ptr}
def fmt_type{T} = merge{quality{T}, fmtnat{width{T}}}
def fmt_type{T & isptr{T}} = merge{'*',fmt_type{eltype{T}}}
def fmt_type{T if isptr{T}} = merge{'*',fmt_type{eltype{T}}}

View File

@ -1,2 +1,2 @@
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
def clmul{a:T, b:T, imm if w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
def clmul{a, b} = clmul{a, b, 0}

View File

@ -4,11 +4,11 @@ include './f64'
include './bitops'
def fillbits{dst:*u64, len:(ux), v } = { emit{void, 'fillBits', dst, len, v }; return{}; }
def fillbits{dst:*u64, len:(ux), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
def fillbits{dst:(*u64), len:(ux), v } = { emit{void, 'fillBits', dst, len, v }; return{}; }
def fillbits{dst:(*u64), len:(ux), v, x} = { emit{void, 'fillBitsDec', dst, len, v, x}; return{}; }
def cmp_err{x} = { emit{void, 'cmp_err'}; return{}; }
fn cmpIX(dst:*u64, len:ux, x:u64, v:u1) : void = {
fn cmpIX(dst:(*u64), len:ux, x:(u64), v:(u1)) : void = {
nan:u1 = q_f64{x}
if (~(nan | q_chr{x})) cmp_err{x}
fillbits{dst, len, v&~nan, x}
@ -16,13 +16,13 @@ fn cmpIX(dst:*u64, len:ux, x:u64, v:u1) : void = {
def eqne{op} = same{op,__eq}|same{op,__ne}
def pathAS{dst, len, T, op, x & issigned{T}} = {
def R{f & eqne{op}} = {
def pathAS{dst, len, T, op, x if issigned{T}} = {
def R{f if eqne{op}} = {
if (rare{floor{f}!=f}) fillbits{dst, len, op{0,1}, x} # also includes check for NaN/sNaN
ftrunc{i64,f}
}
def R{f & same{op,__lt}|same{op,__ge}} = ftrunc{i64,ceil{f}}
def R{f & same{op,__gt}|same{op,__le}} = ftrunc{i64,floor{f}}
def R{f if same{op,__lt} or same{op,__ge}} = ftrunc{i64,ceil{f}}
def R{f if same{op,__gt} or same{op,__le}} = ftrunc{i64,floor{f}}
xf:f64 = interp_f64{x}
xi64:i64 = R{xf}
@ -40,7 +40,7 @@ def pathAS{dst, len, T, op, x & issigned{T}} = {
xT
}
def pathAS{dst, len, T, op, x & T==f64} = {
def pathAS{dst, len, T, op, x if T==f64} = {
if (rare{~q_f64{x}}) {
if (~eqne{op}) if (~q_chr{x}) cmp_err{x}
fillbits{dst, len, op{0,1}, x}
@ -48,7 +48,7 @@ def pathAS{dst, len, T, op, x & T==f64} = {
from_B{T,x}
}
def pathAS{dst, len, T, op, x & isunsigned{T}} = {
def pathAS{dst, len, T, op, x if isunsigned{T}} = {
if (rare{~q_chr{x}}) {
if (~eqne{op}) if (~q_f64{x}) cmp_err{x}
fillbits{dst, len, op{1,0}, x}
@ -60,18 +60,18 @@ def pathAS{dst, len, T, op, x & isunsigned{T}} = {
def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:*u64, len:(ux)} = {
def any2bit{VT, unr, op0, wS, wV, xS, xV, dst:(*u64), len:(ux)} = {
def bulk = vcount{VT}*unr
xi:ux = 0
def T = eltype{VT}
def op = match (op0) {
{_ & ~hasarch{'X86_64'} | hasarch{'AVX512F'}} => op0
{(__le) & issigned{T}} => __gt
{(__ge) & issigned{T}} => __lt
{(__lt) & isunsigned{T}} => __ge
{(__gt) & isunsigned{T}} => __le
{(__ne) & isint{T}} => __eq
{_ if not hasarch{'X86_64'} or hasarch{'AVX512F'}} => op0
{(__le) if issigned{T}} => __gt
{(__ge) if issigned{T}} => __lt
{(__lt) if isunsigned{T}} => __ge
{(__gt) if isunsigned{T}} => __le
{(__ne) if isint{T}} => __eq
{_} => op0
}
def mask = if (same{op0, op}) homMask else ({...x} => ~homMask{...x})

View File

@ -3,7 +3,7 @@ include './mask'
include './cbqnDefs'
include './bitops'
def copyFromBits{T, loadFn, rp, l:u64} = {
def copyFromBits{T, loadFn, rp, l:(u64)} = {
def bulk = vcount{T}
def TU = ty_u{T}

View File

@ -2,9 +2,9 @@ include './base'
include 'util/tup'
include './vecfold'
if (hasarch{'SSE2'}) {
if_inline (hasarch{'SSE2'}) {
fn sum_vec{T}(v:T) = vfold{+, fold{+, unpackQ{v, T**0}}}
def fold_addw{v:T & eltype{T}==i8} = sum_vec{T}(v)
def fold_addw{v:T if eltype{T}==i8} = sum_vec{T}(v)
}
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}
@ -46,7 +46,7 @@ fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = {
total := trunc{usz, r0} # To compute last count
def count_each{js, num} = {
j := @collect (k to num) trunc{T, js+k}
c := copy{tuplen{j}, [vec]uT ** 0}
c := copy{length{j}, [vec]uT ** 0}
e := each{{j}=>V**j, j}
@for (xv over b) each{{c,e} => c -= xv == e, c, e}
def add_sum{c, j} = {

View File

@ -1,4 +1,4 @@
include 'debug/printf' # printf & lprintf
def assert{x:u1} = { if (not x) emit{void, '__builtin_trap'} }
def assert{x:(u1)} = { if (not x) emit{void, '__builtin_trap'} }
def test_assert = assert # test_assert is guaranteed to either not exist, or always trap on bad input

View File

@ -6,15 +6,15 @@ include './mask'
include 'util/tup'
def rootty{T & isprim{T}} = T
def rootty{T & isvec{T}} = eltype{T}
def rootty{T if isprim{T}} = T
def rootty{T if isvec{T}} = eltype{T}
def is_s{X} = issigned{rootty{X}}
def is_u{X} = isunsigned{rootty{X}}
def ty_sc{O, R} = R # keep floats as-is
def ty_sc{O, R & is_s{O} & is_u{R}} = ty_s{R}
def ty_sc{O, R & is_u{O} & is_s{R}} = ty_u{R}
def ty_sc{O, R if is_s{O} and is_u{R}} = ty_s{R}
def ty_sc{O, R if is_u{O} and is_s{R}} = ty_u{R}
def bqn_or{a, b} = (a+b)-(a*b)
@ -22,18 +22,18 @@ def bqn_or{a, b} = (a+b)-(a*b)
# + & -
def arithChk1{F==__add, M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}}
def arithChk1{F==__sub, M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}}
def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r}
def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r}
def arithChk1{F==__add, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r}
def arithChk1{F==__sub, M, w:T, x:T, r:T if isvec{T} and tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r}
def arithChk2{F, M, w:T, x:T & is_s{T} & (same{F,__add} | same{F,__sub})} = {
def arithChk2{F, M, w:T, x:T if is_s{T} and (same{F,__add} or same{F,__sub})} = {
r:= F{w,x}
tup{r, arithChk1{F, M, w, x, r}}
}
# ×
def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i8==eltype{T} & hasarch{'X86_64'}} = {
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i8==eltype{T} and hasarch{'X86_64'}} = {
def wp = unpackQ{w, T ~~ (T**0 > w)}
def xp = unpackQ{x, T ~~ (T**0 > x)}
def rp = each{__mul, wp, xp}
@ -46,12 +46,12 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i8==eltype{T} & hasarc
tup{packQ{rp}, tup{'~andAllZero', RU~~tree_fold{|, bad}, RU**0xff80}}
}
}
def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i16==eltype{T} & hasarch{'X86_64'}} = {
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i16==eltype{T} and hasarch{'X86_64'}} = {
rl:= __mul{w,x}
rh:= mulh{w,x}
tup{rl, tup{'anyne', rh, rl>>15}}
}
def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasarch{'X86_64'}} = {
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and i32==eltype{T} and hasarch{'X86_64'}} = {
max:= re_el{f32, (ty_u{T})**0x4efffffe}
def cf32{x:X} = emit{re_el{f32,X}, tern{T==[8]i32, '_mm256_cvtepi32_ps', '_mm_cvtepi32_ps'}, x}
f32mul:= cf32{w} * cf32{x}
@ -68,7 +68,7 @@ def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & i32==eltype{T} & hasar
# tup{packQQ{each{{v} => v & T2**0xFFFFFFFF, rp}}, tup{'homAny', tree_fold{|,bad}}} this doesn't use M
}
def arithChk2{F, M, w:T, x:T & same{F,__mul} & isvec{T} & hasarch{'AARCH64'}} = {
def arithChk2{F, M, w:T, x:T if same{F,__mul} and isvec{T} and hasarch{'AARCH64'}} = {
def r12 = mulw{w, x}
rl:= packLo{r12}
rh:= packHi{r12}
@ -82,24 +82,24 @@ def runner{u, R, F} = {
def run{F, M, w, x} = { show{'todo', c, R, F, w, x}; emit{void,'__builtin_abort'}; w }
def run{F, M, w:T, x:T & c & R!=u32} = {
def run{F, M, w:T, x:T if c and R!=u32} = {
arithChk2{F, M, w, x}
}
def run{F, M, w, x & u} = tup{F{w, x}, tup{'none'}} # trivial base implementation
def run{F, M, w, x if u} = tup{F{w, x}, tup{'none'}} # trivial base implementation
def toggleTop{x:X} = x ^ X**(1<<(elwidth{X}-1))
def run{F==__sub, M, w:VU, x:VU & c & is_u{VU}} = { # 'b'-'a'
def run{F==__sub, M, w:VU, x:VU if c and is_u{VU}} = { # 'b'-'a'
def VS = ty_s{VU}
run{F, M, VS~~toggleTop{w}, VS~~toggleTop{x}}
}
def run{F, M, w:VU, x:VS & c & is_u{VU} & is_s{VS}} = { # 'a'+3, 'a'-3
def run{F, M, w:VU, x:VS if c and is_u{VU} and is_s{VS}} = { # 'a'+3, 'a'-3
def {res, ok} = run{F, M, VS~~toggleTop{w}, x}
tup{toggleTop{VU~~res}, ok}
}
def run{F==__add, M, w:VS, x:VU & c & is_s{VS} & is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3
def run{F==__add, M, w:VS, x:VU if c and is_s{VS} and is_u{VU}} = run{F, M, x, w} # 3+'a' → 'a'+3
def run{F, M, w:VW, x:VX & c & R==u32 & (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1
def run{F, M, w:VW, x:VX if c and R==u32 and (same{F,__add} | same{F,__sub})} = { # 'a'+1, 'a'-1
r:= F{ty_u{w}, ty_u{x}}
tup{re_el{R, VW}~~r, tup{'homAny', M{r > type{r}**1114111}}}
}
@ -107,11 +107,11 @@ def runner{u, R, F} = {
}
# homAny, topAny already give masked vals; anyne doesn't, and ~andAllZero assumes no masking
def runChecks_any{F, vals} = { F{tree_fold{|, each{tupsel{1,.}, vals}}} }
def runChecks_any{F, vals} = { F{tree_fold{|, each{select{.,1}, vals}}} }
def runChecks{type=='homAny', vals, M} = runChecks_any{homAny, vals}
def runChecks{type=='topAny', vals, M} = runChecks_any{topAny, vals}
def runChecks{type=='none', vals, M} = 0
def runChecks{type=='~andAllZero', vals, M & ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}}
def runChecks{type=='~andAllZero', vals, M if ~M{0}} = ~tree_fold{&, each{andAllZero, ...slice{flip{vals}, 1}}}
def runChecks{type=='anyne', vals, M} = {
def i{vals} = {
def {_,xs,ys} = flip{vals}
@ -124,8 +124,8 @@ def runChecks{type=='anyne', vals, M} = {
def arithProcess{F, run, overflow, M, is, cw, cx, TY} = {
def {values, checks} = flip{each{{w1, x1} => run{F, M, w1, x1}, cw, cx}}
def ctype = oneVal{each{tupsel{0,.}, checks}}
if (rare{runChecks{ctype, checks, M}}) overflow{tupsel{0,is}*vcount{TY}}
def ctype = oneVal{each{select{.,0}, checks}}
if (rare{runChecks{ctype, checks, M}}) overflow{select{is,0}*vcount{TY}}
each{{c} => TY~~c, values}
}
@ -182,12 +182,12 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
def run = runner{(R==f64) | (mode>=2), R, F}
def getW{v} = trunc{W, v}
def getW{v & W==f64} = interp_f64{v}
def getW{v if W==f64} = interp_f64{v}
cw:= ty_sc{W, TY}**getW{w}
def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
def cws = tuplen{is}**cw
def cws = length{is}**cw
sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}}
}

View File

@ -39,9 +39,9 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
} else { # bitarr ≡ i8/i16/i32arr
def T = [bulk]X
def sh{c} = c << (width{X}-1)
def sh{c & X==u8} = T ~~ (re_el{u16,c}<<7)
def mask{x:X & hasarch{'X86_64'}} = topMask{x}
def mask{x:X & hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}}
def sh{c if X==u8} = T ~~ (re_el{u16,c}<<7)
def mask{x:X if hasarch{'X86_64'}} = topMask{x}
def mask{x:X if hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}}
# TODO compare with doing the comparison in vector registers
badBits:= T ** ~(X~~1)

View File

@ -1,15 +1,15 @@
def ceil{x:f64} = emit{f64, 'ceil', x}
def floor{x:f64} = emit{f64, 'floor', x}
def abs{x:f64} = emit{f64, 'fabs', x}
def ceil{x:(f64)} = emit{f64, 'ceil', x}
def floor{x:(f64)} = emit{f64, 'floor', x}
def abs{x:(f64)} = emit{f64, 'fabs', x}
def NaN = 0.0/0.0
def isNaN{x:f64} = x!=x
def qNaN{x:u64} = (x<<1) == (cast{u64, 0x8ff8} << 49)
def isNaN{x:(f64)} = x!=x
def qNaN{x:(u64)} = (x<<1) == (cast{u64, 0x8ff8} << 49)
def ftrunc{T, x:f64 & i8==T} = emit{i8, '', x}
def ftrunc{T, x:f64 & i16==T} = emit{i16, '', x}
def ftrunc{T, x:f64 & i32==T} = emit{i32, '', x} # maybe explicitly use _mm_cvtsd_si32?
def ftrunc{T, x:f64 & i64==T} = emit{i64, '', x}
def ftrunc{T== i8, x:(f64)} = emit{T, '', x}
def ftrunc{T==i16, x:(f64)} = emit{T, '', x}
def ftrunc{T==i32, x:(f64)} = emit{T, '', x} # maybe explicitly use _mm_cvtsd_si32?
def ftrunc{T==i64, x:(f64)} = emit{T, '', x}
def fext{x} = emit{f64, '', x}
def interp_f64{x:u64} = emit{f64, 'interp_f64', x}
def interp_f64{x:(u64)} = emit{f64, 'interp_f64', x}

View File

@ -1,10 +1,10 @@
include './base'
include './mask'
def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}}
def opsh32{op}{v:[2]f64, perm} = op{v, shuf{[4]u32, v, perm}}
def mix{op, v:[4]f64 & hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} }
def mix{op, v:[2]f64 & hasarch{'X86_64'}} = opsh32{op}{v, 4b1032}
def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}}
def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}}
def mix{op, v:([4]f64) if hasarch{'AVX'}} = { def sh=opsh64{op}; sh{sh{v, 4b2301}, 4b1032} }
def mix{op, v:([2]f64) if hasarch{'X86_64'}} = opsh32{op}{v, 4b1032}
def reduce_pairwise{op, plog, x:*T, len, init:T} = {
# Pairwise combination to shorten dependency chains

View File

@ -15,21 +15,21 @@ def memset{p:pT, v, l} = {
# These hashes are stored in tables and must be invertible!
# Murmur3
def hash_val{x0:u32} = {
def hash_val{x0:(u32)} = {
x := x0
x ^= x >> 16; x *= 0x85ebca6b
x ^= x >> 13; x *= 0xc2b2ae35
x ^= x >> 16; x
}
def hash_val{x0:u64} = {
def hash_val{x0:(u64)} = {
x := x0
x ^= x >> 33; x *= 0xff51afd7ed558ccd
x ^= x >> 33; x *= 0xc4ceb9fe1a85ec53
x ^= x >> 33; x
}
# CRC32
if (hasarch{'SSE4.2'}) require{'x86intrin.h'}
def hash_val{x:u32 & hasarch{'SSE4.2'}} = {
if_inline (hasarch{'SSE4.2'}) require{'x86intrin.h'}
def hash_val{x:(u32) if hasarch{'SSE4.2'}} = {
emit{u32, '_mm_crc32_u32', 0x973afb51, x}
}
@ -39,7 +39,7 @@ def hash_val{x:u32 & hasarch{'SSE4.2'}} = {
# Allocates the maximum at the start, resizes downwards within the existing allocation
def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
def ws = each{width,Ts}
def wt = tupsel{0,ws}
def wt = select{ws,0}
each{assert, slice{ws,0,-1} >= slice{ws,1}} # Doesn't do alignment
# Variables updated on resize
sz := usz~~1 << logsz
@ -61,7 +61,7 @@ def hash_alloc{logsz, msz, ext, Ts, v0s, has_radix, ordered} = {
sh -= m; sz <<= m
set_thresh{}
cc = 0 # Collision counter
k:tupsel{0,Ts} = 0; --k # Index where to move an element to
k:select{Ts,0} = 0; --k # Index where to move an element to
each{{p,v} => { p -= dif; memset{p, v, dif} }, ptrs, v0s}
def {hash, ...vals} = ptrs; def {h0, ...v0r} = v0s
s := sz+ext

View File

@ -3,25 +3,25 @@ local def maskInit1{w} = {
merge{(w/8-1)**255, (1<<x)-1, (w/8)**0}
}, iota{8}}}
}
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n & width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n & width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
mask256_1:*u8 = maskInit1{256}; def maskOfBit{T,n if width{T}==256} = load{*[32]u8 ~~ (mask256_1 + (n>>3)^31 + 64*(n&7))}
mask128_1:*u8 = maskInit1{128}; def maskOfBit{T,n if width{T}==128} = load{*[16]u8 ~~ (mask128_1 + (n>>3)^15 + 32*(n&7))}
mask256:*i64 = merge{4 ** -1, 4 ** 0}
local def maskOfImpl{T, n, w} = load{*ty_u{T} ~~ (*u8~~mask256 + 32 - n*(elwidth{T}/8))}
# get homogeneous mask of first n items; 0 ≤ n ≤ vcount{T}
def maskOf{T,n & width{T}==256} = maskOfImpl{T, n, 256}
def maskOf{T,n & width{T}==128} = maskOfImpl{T, n, 128}
def maskOf{T,n & width{T}== 64} = maskOfImpl{T, n, 64}
def maskOf{T,n if width{T}==256} = maskOfImpl{T, n, 256}
def maskOf{T,n if width{T}==128} = maskOfImpl{T, n, 128}
def maskOf{T,n if width{T}== 64} = maskOfImpl{T, n, 64}
def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~homAll{x==y}
def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = homAny{M{x!=y}}
def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y
def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
def anyne{x:T, y:T, M if M{0}==0 and isvec{T}} = ~homAll{x==y}
def anyne{x:T, y:T, M if M{0}==1 and isvec{T}} = homAny{M{x!=y}}
def anyne{x:T, y:T, M if M{0}==0 and anyInt{x}} = x!=y
def anyne{x:T, y:T, M if M{0}==1 and anyInt{x}} = M{x^y} != 0
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = {
def anynePositive{x:T, y:T, M if M{0}==0} = anyne{x, y, M}
def anynePositive{x:T, y:T, M if M{0}==1 and isvec{T}} = {
def {n,m} = homMaskX{x==y}
def E = tern{type{m}==u64, u64, u32}
(promote{E,~m} << (width{E}-M{'count'}*n)) != 0
@ -30,8 +30,8 @@ def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = {
def maskNone{x} = x
def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x}
def maskAfter{n} = {
def mask{x:X & isvec{X}} = x & (X~~maskOf{X,n})
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X if isvec{X}} = x & (X~~maskOf{X,n})
def mask{x:X if anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}}
def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{X, mode=='to homogeneous bits'} = maskOf{X,n}
@ -42,7 +42,7 @@ def maskAfter{n} = {
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
def loadLowBatch{T, ptr:P, w, n if eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
def storeBatch{ptr:P, n, x:T, M} = {
@ -63,18 +63,18 @@ def loadBatch{ptr:P, n, T} = {
widen{T, loadLow{*re_el{E0, T} ~~ rpos, vcount{T}*width{E0}}}
}
def loadBatch {ptr:P, ns, T & istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns}
def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs}
def loadBatch {ptr:P, ns, T if istup{ns}} = each{{n } => loadBatch {ptr, n, T }, ns}
def storeBatch{ptr:P, ns, xs, M if istup{ns}} = each{{n,x} => storeBatch{ptr, n, x, M}, ns, xs}
# "harmless" pointer cast that'll only cast void*
def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got ',p}}
def hCast{T,p:P & same{T,eltype{P}}} = p
def hCast{T,p:P & same{P,*void}} = *T~~p
def hCast{T,p:*T} = p
def hCast{T,p:(*void)} = *T~~p
def mlExec{i, iter, vars0, bulk, M} = {
def vproc{p:P & isptr{P}} = p
def vproc{p:P if isptr{P}} = p
def vproc{('m')} = tptr{{_}=>M, '!'}
def vproc{{T,p:P}} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}}

View File

@ -9,7 +9,7 @@ fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T}
def VT = [bulk]T
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk}
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
sr{each{abs, cx}}
}
len

View File

@ -1,157 +1,157 @@
def nvec{T} = 0
def nvec{T & isvec{T}} = (width{T}==64) | (width{T}==128)
def nvec{T if isvec{T}} = (width{T}==64) | (width{T}==128)
def nvec{T,w} = 0
def nvec{T,w & nvec{T}} = elwidth{T}==w
def nvec{T,w if nvec{T}} = elwidth{T}==w
def nveci = genchk{nvec, isint}
def nvecs = genchk{nvec, issigned}
def nvecu = genchk{nvec, isunsigned}
def nvecf = genchk{nvec, isfloat}
def reinterpret{T, v & same{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v }
def reinterpret{T, v if same{'pointer',typekind{T}} and ktup{v}} = { tmp:T=v }
def nty{T} = {
def q = quality{T}
merge{if (q=='i') 's' else q, fmtnat{width{T}}}
}
def nty{T & isvec{T}} = nty{eltype{T}}
def ntyp{S, ...S2, T & w128{T}} = merge{S, 'q', ...S2, '_', nty{T}}
def ntyp{S, ...S2, T & w64{T}} = merge{S, ...S2, '_', nty{T}}
def nty{T if isvec{T}} = nty{eltype{T}}
def ntyp{S, ...S2, T if w128{T}} = merge{S, 'q', ...S2, '_', nty{T}}
def ntyp{S, ...S2, T if w64{T}} = merge{S, ...S2, '_', nty{T}}
def ntyp0{S, T} = merge{S, '_', nty{T}}
def addwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b}
def subwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b}
def mulwLo{a:T,b:T & w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b}
def mulwHi{a:T,b:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b}
def mulw {a:T,b:T & w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}}
def addwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vaddl', T}, a, b}
def subwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vsubl', T}, a, b}
def mulwLo{a:T,b:T if w64i{T}} = emit{el_d{T}, ntyp{'vmull', T}, a, b}
def mulwHi{a:T,b:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmull_high', T}, a, b}
def mulw {a:T,b:T if w128{T}} = tup{mulwLo{half{a,0}, half{b,0}}, mulwHi{a,b}}
def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
def shrn{a:T, s if w128i{T} and elwidth{T}>8} = { def H=el_h{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
def shrm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes))
def shlm{a:T, s, d:T if nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a<<s) | (d & (mask of new zeroes))
def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M,elwidth{T}} & width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m}
def bitBlend{f:T, t:T, m:M if nvec{T} and nvecu{M,elwidth{T}} and width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def homBlend{f:T, t:T, m:M if nvec{M}} = bitBlend{f, t, m}
def addpw { x:T & nveci{T} & elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D, x:T & nveci{T} & elwidth{T}<=32 & D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
def mla{a:T, x:T, y:T & nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
def mls{a:T, x:T, y:T & nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
def rbit{x:T & nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
def rev{w, x:T & w==elwidth{T}} = x
def rev{w==16, x:T & elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
def rev{w==32, x:T & elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
def rev{w==64, x:T & elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
def popc{x:T & nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
def clz{x:T & nvecu{T} & elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
def cls{x:T & nveci{T} & elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
def addpw { x:T if nveci{T} and elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening
def addpwa{a:D, x:T if nveci{T} and elwidth{T}<=32 and D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate
def mla{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y
def mls{a:T, x:T, y:T if nvec{T}} = emit{T, ntyp{'vmls', T}, a, x, y} # a - x*y
def rbit{x:T if nvecu{T,8}} = emit{T, ntyp{'vrbit', T}, x}
def rev{w, x:T if w==elwidth{T}} = x
def rev{w==16, x:T if elwidth{T}<16} = emit{T, ntyp{'vrev16', T}, x} # reverse the order of elements in each w-bit window
def rev{w==32, x:T if elwidth{T}<32} = emit{T, ntyp{'vrev32', T}, x}
def rev{w==64, x:T if elwidth{T}<64} = emit{T, ntyp{'vrev64', T}, x}
def popc{x:T if nvecu{T,8}} = emit{T, ntyp{'vcnt', T}, x}
def clz{x:T if nvecu{T} and elwidth{T}<=32} = emit{T, ntyp{'vclz', T}, x}
def cls{x:T if nveci{T} and elwidth{T}<=32} = ty_u{T}~~emit{ty_s{T}, ntyp{'vcls', T}, x}
def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
def fold_addw{a:T & nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a}
def fold_min {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T & nvec{T} & ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def vfold{F, x:T & nvec{T} & ~nveci{T,64} & same{F, min}} = fold_min{x}
def vfold{F, x:T & nvec{T} & ~nveci{T,64} & same{F, max}} = fold_max{x}
def vfold{F, x:T & nvec{T} & same{F, +}} = fold_add{x}
def fold_add {a:T if nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
def fold_addw{a:T if nveci{T}} = emit{w_d{eltype{T}}, ntyp{'vaddlv', T}, a}
def fold_min {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T if nvec{T} and ~nveci{T,64}} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x}
def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x}
def storeLow{ptr:P, w, x:T & nvec{T} & w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def storeLow{ptr:P, w, x:T & nvec{T} & w==width{T}} = store{*T~~ptr, 0, x}
def storeLow{ptr:P, w, x:T if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def storeLow{ptr:P, w, x:T if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def loadLow{ptr:P, w & w<=64} = { # a broadcast load
def loadLow{ptr:P, w if w<=64} = { # a broadcast load
def T=eltype{P}
def L=re_el{ty_u{w}, T}
T ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
}
def loadLow{ptr:P, w & w==elwidth{P}} = load{ptr}
def loadLow{ptr:P, w if w==elwidth{P}} = load{ptr}
def undefPromote{T, x:X & w64{X} & w128{T} & eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯
def half{x:T, n==0 & w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x}
def half{x:T, n==1 & w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x}
def pair{a:T, b:T & w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
def copyLane{dst:D, di, src:S, si & w64{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
def copyLane{dst:D, di, src:S, si & w128{D} & nvec{S} & eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def broadcastSel{x:T, i & nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
def vshl{a:T, b:T, n & knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
def undefPromote{T, x:X if w64{X} and w128{T} and eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯
def half{x:T, n==0 if w128{T}} = emit{n_h{T}, ntyp0{'vget_low', T}, x}
def half{x:T, n==1 if w128{T}} = emit{n_h{T}, ntyp0{'vget_high', T}, x}
def pair{a:T, b:T if w64{T}} = emit{n_d{T}, ntyp0{'vcombine', T}, a, b}
def copyLane{dst:D, di, src:S, si if w64{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopy_lane', S}, dst, di, src, si}
def copyLane{dst:D, di, src:S, si if w128{D} and nvec{S} and eltype{D}==eltype{S}} = emit{D, ntyp{'vcopyq_lane', S}, dst, di, src, si}
def broadcastSel{x:T, i if nvec{T}} = emit{T, ntyp{'vdup', tern{w128{T},'_laneq','_lane'}, T}, x, i}
def vshl{a:T, b:T, n if knum{n}} = emit{T, ntyp{'vext', T}, a, b, n}
def zipLo{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zipHi{a:T, b:T & nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def zip{a:T, b:T & nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def zipLo{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip1', T}, a, b}
def zipHi{a:T, b:T if nvec{T}} = emit{T, ntyp{'vzip2', T}, a, b}
def zip{a:T, b:T if nvec{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def packLo{x:T, y:T & nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def packHi{x:T, y:T & nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
def packLo{x:T, y:T if nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def packHi{x:T, y:T if nvec{T}} = { def H=el_s{T}; emit{H, ntyp{'vuzp2', H}, H~~x, H~~y} }
def packLo{{x, y}} = packLo{x, y}
def packHi{{x, y}} = packHi{x, y}
def trn1{x:T, y:T & nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
def trn2{x:T, y:T & nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
def trn1{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn1', T}, x, y}
def trn2{x:T, y:T if nvec{T}} = emit{T, ntyp{'vtrn2', T}, x, y}
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
def sel{L, x:T, i:I if lvec{L,16,8} and w128{T} and nvec{I, 8}} = re_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, re_el{eltype{I},x}, ty_u{i}}}
local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers
def cvt{T==f64, x:X & nveci{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_f64', X}, x}
def cvt{T==i64, x:X & nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_s64', X}, x}
def cvt{T==u64, x:X & nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_u64', X}, x}
def cvt{T==f64, x:X if nveci{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_f64', X}, x}
def cvt{T==i64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_s64', X}, x}
def cvt{T==u64, x:X if nvecf{X,64}} = emit{[vcount{X}]T, ntyp{'vcvt', '_u64', X}, x}
def widen{T, x:X & w64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x}
def widen{T, x:X & w64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}}
def widen{T, x:X & w64{X} & isfloat{eltype{T}}!=isfloat{eltype{X}} & elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}}
def widen{T, x:X & w128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}}
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x}
def widen{T, x:X if w64{X} and eqqi{eltype{T},eltype{X}} and elwidth{T}> elwidth{X}*2} = widen{T, widen{el_s{T}, x}}
def widen{T, x:X if w64{X} and isfloat{eltype{T}}!=isfloat{eltype{X}} and elwidth{T}>elwidth{X}} = cvt{eltype{T}, widen{[vcount{T}](to_w{eltype{X},elwidth{T}}), x}}
def widen{T, x:X if w128{X} and vcount{X}>vcount{T}} = widen{T, half{x,0}}
def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}}
def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x}
def narrow{T, x:X & w128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2< elwidth{X}} = narrow{T, undefPromote{el_s{X}, narrow{w_h{eltype{X}}, x}}}
def narrow{T, x:X if w128{X} and eqqi{T,eltype{X}} and width{T}*2==elwidth{X}} = emit{el_h{X}, ntyp0{'vmovn', X}, x}
def narrow{T, x:X if w128{X} and isfloat{T}!=isfloat{eltype{X}} and width{T}<elwidth{X}} = narrow{T, cvt{to_w{T, elwidth{X}}, x}}
def narrowUpper{lowRes:L, x:X & w64i{L} & w128{X} & el_d{L}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
def narrowUpper{lowRes:L, x:X if w64i{L} and w128{X} and el_d{L}==X} = emit{[vcount{L}*2](eltype{L}), ntyp0{'vmovn_high', X}, lowRes, x}
def narrowPair{a:T, b:T} = narrowUpper{narrow{w_h{eltype{T}}, a}, b}
def narrowPair{a:T, b:T & isint{eltype{T}}} = packLo{a, b}
def narrowPair{a:T, b:T if isint{eltype{T}}} = packLo{a, b}
def widenUpper{x:T & w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T & w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
def widenUpper{x:T if w128i{T}} = emit{el_m{T}, ntyp0{'vmovl_high', T}, x}
def widen{x:T if w128{T}} = tup{widen{el_m{T}, x}, widenUpper{x}}
def bitAny{x:T} = fold_max{re_el{u32, x}}!=0
def bitAll{x:T} = fold_min{re_el{u32, x}}==0xffff_ffff
def topAny{x:T & nvec{T}} = fold_min{ty_s{x}}<0
def topAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0
def homAny{x:T & nvec{T}} = bitAny{x}
def homAll{x:T & nvec{T}} = bitAll{x}
def topAny{x:T if nvec{T}} = fold_min{ty_s{x}}<0
def topAll{x:T if nvec{T}} = fold_max{ty_s{x}}<0
def homAny{x:T if nvec{T}} = bitAny{x}
def homAll{x:T if nvec{T}} = bitAll{x}
def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
def homMask{x:T if nvecu{T} and elwidth{T}>=vcount{T}} = {
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
}
def homMask{x:T & nvecu{T} & T==[16]u8} = {
def homMask{x:T if nvecu{T} and T==[16]u8} = {
t:= [8]u16~~sel{[16]u8, x, make{[16]u8, 0,8,1,9,2,10,3,11,4,12,5,13,6,14,7,15}}
fold_add{t & make{[8]u16, (1<<iota{8})*0x0101}}
}
def homMask{a:T,b:T & T==[16]u8} = {
def homMask{a:T,b:T if T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
fold_add{addpw{addpw{addp{a&m, b&m}}}<<make{[4]u32,iota{4}*8}}
}
def homMask{a:T,b:T,c:T,d:T & T==[16]u8} = {
def homMask{a:T,b:T,c:T,d:T if T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
t1:= addp{a&m, b&m}
t2:= addp{c&m, d&m}
t3:= addp{t1, t2}
extract{[2]u64~~addp{t3,t3},0}
}
def homMask{...as & tuplen{as}>1 & elwidth{type{tupsel{0,as}}}>=32} = homMask{...each{{i}=>narrowPair{tupsel{i*2,as},tupsel{i*2+1,as}}, iota{tuplen{as}/2}}}
def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = {
def homMask{...as if length{as}>1 and elwidth{type{select{as,0}}}>=32} = homMask{...each{{i}=>narrowPair{select{as,i*2},select{as,i*2+1}}, iota{length{as}/2}}}
def homMask{a:T,b:T if vcount{T}*2<=elwidth{T}} = {
def n = vcount{T}
truncBits{n*2, fold_add{shrm{a,elwidth{T}-n,b} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
}
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def andAllZero{x:T, y:T if nveci{T}} = ~bitAny{x&y}
def homMaskX{a:T & eltype{T}!=u64} = {
def homMaskX{a:T if eltype{T}!=u64} = {
def h = elwidth{T}/2
tup{h, truncBits{vcount{T}*h, extract{[1]u64~~shrn{el_m{T}~~a, h}, 0}}}
}
def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def homMaskStoreF{p:P, m:M, v:T if nveci{M} and nvec{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}

View File

@ -20,9 +20,9 @@ def scan_core{upd, set, scan, rp:pT, wp:W, s:(usz)} = {
k = e
}
}
def indrep_by_sum{T, rp:*T, wp, s:(usz), js, inc} = {
def indrep_by_sum{T, rp:(*T), wp, s:(usz), js, inc} = {
def scan{ptr, len} = @for (ptr over len) js=ptr+=js
def scan{ptr, len & width{T}<=32} = {
def scan{ptr, len if width{T}<=32} = {
def scanfn = merge{'si_scan_pluswrap_u',fmtnat{width{T}}}
p := *ty_u{eltype{type{ptr}}}~~ptr
emit{void, scanfn, p, p, len, js}; js=load{ptr,len-1}
@ -45,7 +45,7 @@ fn ind_by_scan_i32{W}(xv:*void, rp:*i32, s:usz) : void = {
}
}
def rep_by_scan{T, wp, xv:*void, rv:*void, s} = {
def rep_by_scan{T, wp, xv:(*void), rv:(*void), s} = {
xp := *T~~xv; js := *xp; px := js
def inc{j} = {sx:=px; px=load{xp,j}; px-sx}
indrep_by_sum{T, *T~~rv, wp, s, js, inc}
@ -59,7 +59,7 @@ exportT{'si_replicate_scan', flat_table{rep_by_scan, ind_types, dat_types}}
# Constant replicate
if (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) {
if_inline (not (hasarch{'AVX2'} | hasarch{'AARCH64'})) {
fn rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = {
rep_by_scan{T, cast_i{usz,wv}, x, r, cast_i{usz, wv*n}}
@ -101,14 +101,14 @@ def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
rcsh4_dat:*i8 = join{join{each{get_shuf_data{., 4}, rcsh4_dom}}}
rcsh4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
def read_shuf_vecs{l, ellw:u64, shp:P} = { # tuple of byte selectors in 1<<ellw
def read_shuf_vecs{l, ellw:(u64), shp:P} = { # tuple of byte selectors in 1<<ellw
def V = eltype{P}
def double{x:X & hasarch{'AVX2'}} = {
def double{x:X if hasarch{'AVX2'}} = {
s:=shuf{[4]u64, x, 4b3120}; s+=s
r:=each{bind{~~,[32]i8},unpackQ{s, s + X**1}}
r
}
def double{x:X & hasarch{'AARCH64'}} = {
def double{x:X if hasarch{'AARCH64'}} = {
s:= x+x
zip{s, s + X**1}
}
@ -116,7 +116,7 @@ def read_shuf_vecs{l, ellw:u64, shp:P} = { # tuple of byte selectors in 1<<ellw
def sh = each{{v}=>{r:=v}, l**V**0}
def tlen{e} = cdiv{l, e} # Length for e bytes, rounded up
def set{i} = { tupsel{i,sh} = each{load{shp,.}, i} }
def set{i} = { select{sh,i} = each{load{shp,.}, i} }
def ext{e} = {
def m = tlen{2*e}; def n = tlen{e} # m<n
if (ellw <= lb{e}) set{slice{iota{n},m}}
@ -126,7 +126,7 @@ def read_shuf_vecs{l, ellw:u64, shp:P} = { # tuple of byte selectors in 1<<ellw
sh
}
def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { # onreps{inputVector, {nextOutputVector} => ...}
def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:(u64)} = { # onreps{inputVector, {nextOutputVector} => ...}
def step = vcount{V}
nv := n / step
j:u64 = 0
@ -148,13 +148,13 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { # onreps{inputVect
}
}
if (hasarch{'AVX2'}) {
if_inline (hasarch{'AVX2'}) {
def rep_iter_from_sh{sh}{x, gen} = {
def l = tuplen{sh}
def l = length{sh}
def h = l>>1
def fs{v, s} = gen{sel{[16]i8, v, s}}
a := shuf{[4]u64, x, 4b1010}; each{fs{a,.}, slice{sh,0,h}}
if (l%2) fs{x, tupsel{h, sh}}
if (l%2) fs{x, select{sh, h}}
b := shuf{[4]u64, x, 4b3232}; each{fs{b,.}, slice{sh,-h}}
}
@ -168,7 +168,7 @@ if (hasarch{'AVX2'}) {
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
}
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:(u64)} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
} else if (hasarch{'AARCH64'}) {
@ -176,7 +176,7 @@ if (hasarch{'AVX2'}) {
each{{s} => gen{sel{[16]u8, x, s}}, sh}
}
def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:u64} = {
def rep_const_shuffle{V, wv==2, xv0:*V, rv0:*V, n:(u64)} = {
def E = ty_u{eltype{V}}
rv:= *E~~rv0
@for (x in *E~~xv0 over i to n) { # autovectorized well enough, probably
@ -193,15 +193,15 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
def step = vcount{V} # Bytes written
def wvb = wv << ellw
def hs = (h*step) / wvb # Actual step size in argument elements
def shufbase{i & hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010}
def shufbase{i & hasarch{'AARCH64'}} = load{*V~~(x+i)}
def shufbase{i if hasarch{'AVX2'}} = shuf{[4]u64, load{*V~~(x+i)}, 4b1010}
def shufbase{i if hasarch{'AARCH64'}} = load{*V~~(x+i)}
def shufrun{a, s} = sel{[16]i8, a, s} # happens to be the same across AVX2 & NEON
i:u64 = 0
re := r + n*wvb - h*step
while (r <= re) {
a := shufbase{i}
@unroll (j to h) store{*V~~r, j, shufrun{a, tupsel{j,sh}}}
@unroll (j to h) store{*V~~r, j, shufrun{a, select{sh,j}}}
i += hs << ellw
r += hs*wvb
}
@ -211,7 +211,7 @@ fn rep_const_shuffle_partial4(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
def end = makelabel{}
@unroll (j to h) {
s = shufrun{a, tupsel{j,sh}}
s = shufrun{a, select{sh,j}}
if (r > re) goto{end}
store{*V~~r, 0, s}
r+= step
@ -230,17 +230,17 @@ fn rcsh_sub{wv, V}(ellw:u64, x:*i8, r:*i8, n:u64, sh:*V) : void = {
}
fn rep_const_shuffle_any(wv:u64, ellw:u64, x:*i8, r:*i8, n:u64) : void = {
if (wv > tupsel{-1,rcsh_vals}) {
if (wv > select{rcsh_vals,-1}) {
return{rep_const_shuffle_partial4(wv, ellw, x, r, n)}
}
n <<= ellw
ri := wv - tupsel{0,rcsh_vals}
ri := wv - select{rcsh_vals,0}
sh := *sdtype~~rcsh_data + load{rcsh_offs,ri}
def try{k} = { if (wv==k) rcsh_sub{k, sdtype}(ellw, x, r, n, sh) }
each{try, rcsh_vals}
}
def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = {
def rep_const_broadcast{T, kv, loop, wv:(u64), x:*T, r:*T, n:(u64)} = {
assert{kv > 0}
def V = [arch_defvw/width{T}]T
@for (x over n) {

View File

@ -1,7 +1,7 @@
include './base'
if (hasarch{'X86_64'}) {
if (hasarch{'PCLMUL'}) include './clmul'
if (hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}) include './avx512'
if_inline (hasarch{'X86_64'}) {
if_inline (hasarch{'PCLMUL'}) include './clmul'
if_inline (hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}) include './avx512'
}
include './mask'
include './f64'
@ -12,7 +12,7 @@ fn scan_scal{T, op}(x:*T, r:*T, len:u64, m:T) : void = {
@for (x, r over len) r = m = op{m, x}
}
def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
def scan_loop{T, init, x:*T, r:*T, len:(u64), scan, scan_last} = {
def step = arch_defvw/width{T}
def V = [step]T
p:= V**init
@ -23,7 +23,7 @@ def scan_loop{T, init, x:*T, r:*T, len:u64, scan, scan_last} = {
q:= len & (step-1)
if (q!=0) homMaskStoreF{rv+e, maskOf{V, q}, scan_last{load{xv,e}, p}}
}
def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
def scan_post{T, init, x:*T, r:*T, len:(u64), op, pre} = {
def last{v, p} = op{pre{v}, p}
def scan{v, p} = {
n:= last{v, p}
@ -35,7 +35,7 @@ def scan_post{T, init, x:*T, r:*T, len:u64, op, pre} = {
# Associative scan ?` if a?b?a = a?b = b?a, used for ⌊⌈
def scan_idem = scan_scal
fn scan_idem{T, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
fn scan_idem{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
scan_post{T, init, x, r, len, op, make_scan_idem{T, op}}
}
@ -54,7 +54,7 @@ export{'si_scan_min_i32', scan_idem_id{i32, min}}; export{'si_scan_max_i32', sca
# Assumes identity is 0
def scan_assoc{op} = {
def shl0{v, k} = shl{[16]u8, v, k/8} # Lanewise
def shl0{v:V, k==128 & hasarch{'AVX2'}} = {
def shl0{v:V, k==128 if hasarch{'AVX2'}} = {
# Broadcast end of lane 0 to entire lane 1
l:= V~~make{[8]i32,0,0,0,-1,0,0,0,0} & spread{v}
sel{[8]i32, l, make{[8]i32, 3*(3<iota{8})}}
@ -65,7 +65,7 @@ def scan_plus = scan_assoc{+}
# Associative scan
def scan_assoc_0 = scan_scal
fn scan_assoc_0{T, op & hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
fn scan_assoc_0{T, op if hasarch{'X86_64'}}(x:*T, r:*T, len:u64, init:T) : void = {
# Prefix op on entire AVX register
scan_post{T, init, x, r, len, op, scan_plus}
}
@ -80,7 +80,7 @@ fn scan_neq{}(p:u64, x:*u64, r:*u64, nw:u64) : void = {
p = -(r>>63) # repeat sign bit
}
}
fn clmul_scan_ne_any{& hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = {
def V = [2]u64
m := V**mark
def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total
@ -101,10 +101,10 @@ fn clmul_scan_ne_any{& hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64,
storeLow{rv+e, 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
}
}
fn scan_neq{& hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = {
fn scan_neq{if hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = {
clmul_scan_ne_any{}(*void~~x, *void~~r, init, nw, -(u64~~1))
}
fn scan_neq{& hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = {
fn scan_neq{if hasarch{'AVX512BW', 'VPCLMULQDQ', 'GFNI'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = {
def V = [8]u64
def sse{a} = make{[2]u64, a, 0}
carry := sse{init}
@ -136,7 +136,7 @@ fn bcs{T}(x:*u64, r:*T, l:u64) : void = {
c:T = 0
@for (r over i to l) { c+= cast_i{T, bitp_get{x,i}}; r = c }
}
fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
fn bcs{T if hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
def U = ty_u{T}
def w = width{T}
def vl= 256 / w
@ -149,7 +149,7 @@ fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
def sums{n} = (if (n==0) tup{0}; else { def s=sums{n-1}; merge{s,s+1} })
def widen{v:T} = unpackQ{shuf{[4]u64, v, 4b3120}, T**0}
def sumlanes{x:u32} = {
def sumlanes{x:(u32)} = {
b:= [8]u32**x >> make{[8]u32, 4*tail{1, iota{8}}}
s:= sel8{[32]u8~~b, ii32>>3 + bit{2}}
p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes
@ -157,12 +157,12 @@ fn bcs{T & hasarch{'AVX2'}}(x:*u64, r:*T, l:u64) : void = {
d+= sel8{d, bit{2}*(1+bit{3}>>2)-1}
d + sel8{d, bit{3}-1}
}
def step{x:u32, i, store1} = {
def step{x:(u32), i, store1} = {
d:= sumlanes{x}
if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100}
j:= (w/8)*i
def out{v, k} = each{out, widen{v}, 2*k+iota{2}}
def out{v0:[vl]T, k} = {
def out{v0:([vl]T), k} = {
v := V~~v0 + c
# Update carry at the lane boundary
if (w!=32 or tail{1,k}) {
@ -202,7 +202,7 @@ def addChk{a:T, b:T} = {
def bad = emit{u1, '__builtin_add_overflow', a, b, mem}
tup{bad, load{mem}}
}
def addChk{a:T, b:T & T==f64} = tup{0, a+b}
def addChk{a:T, b:T if T==f64} = tup{0, a+b}
def widenFull{E, xs} = {
merge{...each{{x:X} => {
@ -219,9 +219,9 @@ def widenFull{E, xs} = {
}, xs}}
}
def floor{x & knum{x}} = x - (x%1)
def maxabsval{T & issigned{T}} = -minvalue{T}
def maxsafeint{T & issigned{T}} = maxvalue{T}
def floor{x if knum{x}} = x - (x%1)
def maxabsval{T if issigned{T}} = -minvalue{T}
def maxsafeint{T if issigned{T}} = maxvalue{T}
def maxsafeint{T==f64} = 1<<53
fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
@ -239,7 +239,7 @@ fn plus_scan{X, R, O}(x:*X, c:R, r:*R, len:u64) : O = {
len
}
# Sum as many vector registers as possible; modifies c and i
def simd_plus_scan_part{X, R}{x:*X, c:(R), r:*R, len:u64, i:u64} = {
def simd_plus_scan_part{X, R}{x:(*X), c:(R), r:(*R), len:(u64), i:(u64)} = {
def b = max{width{R}/2, width{X}}
def bulk = arch_defvw/b
@ -275,12 +275,12 @@ def simd_plus_scan_part{X, R}{x:*X, c:(R), r:*R, len:u64, i:u64} = {
def s1{v0,v1} = tup{v0,v1+toLast{v0}}
def cr = eachx{+, widenFull{R, s1{...s0}}, cv}
cv = toLast{tupsel{-1, cr}}
cv = toLast{select{cr, -1}}
assert{type{cv} == oneType{cr}}
assert{vcount{type{cv}} * tuplen{cr} == bulk}
assert{vcount{type{cv}} * length{cr} == bulk}
each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}}
each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{length{cr}}}
i = i2
}

View File

@ -1,11 +1,11 @@
# Used by scan.singeli and bins.singeli
def sel8{v:V, t} = sel{[16]u8, v, make{re_el{i8,V}, t}}
def sel8{v:V, t & w256{V} & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}}
def sel8{v:V, t if w256{V} and istup{t} and length{t}==16} = sel8{v, merge{t,t}}
def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}}
def shuf{T, v, n if istup{n}} = shuf{T, v, base{4,n}}
local def rev{t} = { def l=tuplen{t}; def j=l-1; tupsel{j-range{l}, j-t} }
local def rev{t} = { def l=length{t}; def j=l-1; select{j-t, j-range{l}} }
local def rev{up,t} = if (up) t else rev{t}
def sel8{v, t, up} = sel8{v, rev{up,t}}
@ -19,14 +19,14 @@ def spread{a:VT, ...up} = {
}
# Set all elements with the last element of the input
def toLast{n:VT, up & hasarch{'X86_64'} & w128{VT}} = {
def toLast{n:VT, up if hasarch{'X86_64'} and w128{VT}} = {
def l{v, w} = l{zip{up,v}, 2*w}
def l{v, w & hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)}
def l{v, w if hasarch{'SSSE3'}} = sel8{v, up*(16-w/8)+iota{16}%(w/8)}
def l{v, w==32} = shuf{[4]i32, v, 4**(up*3)}
def l{v, w==64} = shuf{[4]i32, v, (2*up) + tup{0,1,0,1}}
l{n, elwidth{VT}}
}
def toLast{n:VT, up & hasarch{'AVX2'} & w256{VT}} = {
def toLast{n:VT, up if hasarch{'AVX2'} and w256{VT}} = {
if (elwidth{VT}<=32) sel{[8]i32, spread{n,up}, [8]i32**(up*7)}
else shuf{[4]u64, n, 4**(up*3)}
}
@ -51,17 +51,17 @@ def make_scan_idem{T, op, up} = {
def id = make{V, merger{c**get_id{op,T}, (width{V}/w-c)**0}}
(if (up) shl else shr){[16]u8, v, k/8} | id
}
def shb{v, k & hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
def shb{v, k & k>=32} = shuf{[4]u32, v, shift{k/32,4}}
def shb{v, k & k==128 & hasarch{'AVX2'}} = {
def shb{v, k if hasarch{'SSSE3'}} = sel8{v, shift{k/8,16}}
def shb{v, k if k>=32} = shuf{[4]u32, v, shift{k/32,4}}
def shb{v, k if k==128 and hasarch{'AVX2'}} = {
# After lanewise scan, broadcast end of lane 0 to entire lane 1
sel{[8]i32, spread{v,up}, make{[8]i32, rev{up,3*(3<iota{8})}}}
}
prefix_byshift{op, shb}
}
def make_scan_idem{T==f64, op, up} = {
def sc{a:T & vcount{T}==2} = op{a, zip{~up,a}}
def sc{a:T & hasarch{'AVX2'} & w256{T}} = {
def sc{a:T if vcount{T}==2} = op{a, zip{~up,a}}
def sc{a:T if hasarch{'AVX2'} and w256{T}} = {
def sh{s, a} = op{a, shuf{[4]u64, a, rev{up,s}}}
sh{tup{0,1,1,1},sh{tup{0,0,2,2},a}}
}

View File

@ -5,8 +5,8 @@ include './hashtab'
def findFirst{C, M, F, ...v1} = {
def exit = makelabel{}
def args = undef{M{...each{tupsel{0, .}, v1}}}
def am = tuplen{tupsel{0,v1}}
def args = undef{M{...each{select{., 0}, v1}}}
def am = length{select{v1,0}}
each{{last, ...v2} => {
if (last or C{...v2}) {
each{=, args, M{...v2}}
@ -18,7 +18,7 @@ def findFirst{C, M, F, ...v1} = {
F{...args}
}
def search{E, x, n:u64, OP} = {
def search{E, x, n:(u64), OP} = {
def bulk = arch_defvw/width{E}
def VT = [bulk]E
def end = makeBranch{
@ -60,7 +60,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
0
}
if (hasarch{'X86_64'} | hasarch{'AARCH64'}) {
if_inline (hasarch{'X86_64'} | hasarch{'AARCH64'}) {
export{'simd_search_u8', searchOne{u64, u8}}
export{'simd_search_u16', searchOne{u64, u16}}
export{'simd_search_u32', searchOne{u64, u32}}
@ -79,7 +79,7 @@ def simd_bittab = hasarch{'SSSE3'}
def bittab_init{tab, z} = {
@for (t in *TI~~tab over 256) t = z
}
def bittab_init{tab, z & simd_bittab} = {
def bittab_init{tab, z if simd_bittab} = {
init:= VI**z
@unroll (t in *VI~~tab over 256/vcount{VI}) t = init
}
@ -111,7 +111,7 @@ def readbytes{vtab}{} = {
}
# Look up bits from table
def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = {
def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void)} = {
x:= *u8~~x0
t:= *TI~~tab
r:= *u64~~r0
@ -124,7 +124,7 @@ def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void} = {
x+=k; rem-=k; ++r
}
}
def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & simd_bittab} = {
def bittab_lookup{x0:(*void), n:(u64), r0:(*void), tab:(*void) if simd_bittab} = {
def {bitsel, _} = bittab_selector{readbytes{*VI~~tab}}
def k = vcount{VI}
@for (x in *VI~~x0, r in *ty_u{k}~~r0 over cdiv{n,k}) r = bitsel{x}
@ -139,7 +139,7 @@ def bittab_lookup{x0:*void, n:u64, r0:*void, tab:*void & simd_bittab} = {
# - 'mask': Mark Firsts of x0
# - 'unique': Deduplicate of x0
# - 'index': First index of value x at r0+x
def do_bittab{x0:*void, n:u64, tab:*void, u:u8, t, mode, r0} = {
def do_bittab{x0:(*void), n:(u64), tab:(*void), u:(u8), t, mode, r0} = {
def rbit = mode == 'mask'
def rval = mode == 'unique'
def rind = mode == 'index'
@ -291,15 +291,15 @@ def acc{unr, init:T} = {
def op{S=='get'} = a0v
def op{S=='tr', F} = { a0v = tree_fold{F, a1} }
def op{S=='upd', is, F} = {
if (tuplen{is}==1) a0 = F{a0}
if (length{is}==1) a0 = F{a0}
else a1 = F{a1}
}
}
def isI64{x:T & eltype{T}==f64 & hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
def isI64{x:T & eltype{T}==f64 & hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
def isI64{x:T if eltype{T}==f64 and hasarch{'AARCH64'}} = x == cvt{f64, cvt{i64, x}}
def isI64{x:T if eltype{T}==f64 and hasarch{'SSE4.1'}} = (x==floor{x}) & (abs{x}<=T**(1<<53))
def maskBlend{b:T, x:T, M} = x
def maskBlend{b:T, x:T, M & M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}}
def maskBlend{b:T, x:T, M if M{0}} = homBlend{b, x, M{T, 'to homogeneous bits'}}
fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
assert{n>0}
@ -600,7 +600,7 @@ fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:it
def try_vec_memb{..._} = {}
def try_vec_memb{T, hash, sz, sh, maxh, has_maxh, swap, rp, fp, n, done
& hasarch{'SSE4.2'} & T==u32} = {
if hasarch{'SSE4.2'} and T==u32} = {
# Hash h wants bin h>>sh, so the offset for h in slot i is (in infite-precision ints)
# i-h>>sh = i+((1<<sh-1)-h)>>sh = (((i+1)<<sh-1)-h)>>sh
# We maintain io = (i+1)<<sh-1

View File

@ -1,4 +1,4 @@
if (hasarch{'AVX2'}) {
if_inline (hasarch{'AVX2'}) {
include './base'
include './cbqnDefs'
@ -9,11 +9,11 @@ include 'util/tup'
# def:T - masked original content
# b:B - pointer to data to index; if width{B}<elwidth{T}, padding bytes are garbage read after wanted position
# idx - actual (unscaled) index list
def gather{d:T, b:B, idx:[8]i32, M & w256{T,32}} = {
def gather{d:T, b:B, idx:([8]i32), M if w256{T,32}} = {
if (M{0}) T ~~ emit{[8]i32, '_mm256_mask_i32gather_epi32', d, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
else T ~~ emit{[8]i32, '_mm256_i32gather_epi32', *void~~b, idx, elwidth{B}/8}
}
def gather{d:T, b:B, idx:[4]i32, M & w256{T,64}} = {
def gather{d:T, b:B, idx:([4]i32), M if w256{T,64}} = {
if (M{0}) T ~~ emit{[4]i64, '_mm256_mask_i32gather_epi64', d, *void~~b, idx, M{T,'to sign bits'}, elwidth{B}/8}
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *void~~b, idx, elwidth{B}/8}
}
@ -70,8 +70,8 @@ def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})
def bs{b, c, x} = cshuf{x, c}
def bs{b, c, x & tuplen{b}>0} = {
tupsel{0,b}{each{bs{slice{b,1}, c, .}, x}}
def bs{b, c, x if length{b}>0} = {
select{b,0}{each{bs{slice{b,1}, c, .}, x}}
}
def i = iota{logv}
@ -164,4 +164,4 @@ fn avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
}
export{'avx2_select_bool128', avx2_select_bool128}
}
}

View File

@ -1,14 +1,14 @@
include './base'
if (hasarch{'X86_64'}) {
if (hasarch{'PCLMUL'}) include './clmul'
if (hasarch{'BMI2'}) include './bmi2'
if (hasarch{'AVX512F'}) include './avx512'
if_inline (hasarch{'X86_64'}) {
if_inline (hasarch{'PCLMUL'}) include './clmul'
if_inline (hasarch{'BMI2'}) include './bmi2'
if_inline (hasarch{'AVX512F'}) include './avx512'
}
include './mask'
include 'util/tup'
def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones
def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x}
def popcRand{x:T if isint{T} and width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones
def popcRand{x:T if isint{T} and width{T}<=32} = emit{u8, 'rand_popc64', x}
# Table from l bits to w-bit indices, shifted left by s, and G applied afterwards
def maketab{l,w,s,G} = {
@ -64,7 +64,7 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
} else {
if (has_simd) {
def bufw = bufn * tw
def vc = tern{hasarch{'X86_64'} & (bufw==128), 128, arch_defvw} / tw;
def vc = tern{hasarch{'X86_64'} and bufw==128, 128, arch_defvw} / tw;
def R = [vc]T
@unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc }
assert{bufw % width{R} == 0} # to make sure the below doesn't read out-of-bounds on the stack
@ -118,7 +118,7 @@ def topper{T, U, k, x} = {
itab_4_16:*u64 = maketab{4,16} # 16 elts, 128B
def thresh{c==0, T==i8 } = 32
def thresh{c==0, T==i16} = 16
fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
fn slash{c==0, T if T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def tw = width{T}
def n = 64/tw
def tab = if (tw==8) itab else itab_4_16
@ -140,19 +140,19 @@ fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
# i16 /w & i32 x+/w; 8 elts/iter; 64 bit table input, expanded to 128 or 256 via topper
def simd128{} = hasarch{'X86_64'} | hasarch{'AARCH64'}
def thresh{c==0, T==i16 & simd128{}} = 32
def thresh{c==0, T==i32 & simd128{}} = 16
fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c==0, T==i16 if simd128{}} = 32
def thresh{c==0, T==i32 if simd128{}} = 16
fn slash{c==0, T if simd128{} and i16<=T and T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def I = [16]i8
j := I**(if (T==i16) 0 else cast_i{i8,x})
def {top, inctop} = topper{T, I, 8, x}
@for_special_buffered{r,8} (w in *u8~~w over i to sum) {
ind := load{itab, w}
pc := popc_alt{w, ind, 8}
v := mzipLo{j + I~~make{[2]u64, ind, 0}, tupsel{0,top}}
v := mzipLo{j + I~~make{[2]u64, ind, 0}, select{top,0}}
def st{k, v:V} = store{*V~~r, k, v}
if (T==i16) st{0, v}
else each{st, iota{2}, mzip{v, tupsel{1,top}}}
else each{st, iota{2}, mzip{v, select{top,1}}}
r += pc
j += I**8
inctop{i, top}
@ -161,9 +161,9 @@ fn slash{c==0, T & simd128{} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64,
# i8 & i16 w/x; 128 bits/iter; [16]i8 shuffle
def shufb128{} = hasarch{'SSSE3'} | hasarch{'AARCH64'}
def thresh{c==1, T==i8 & shufb128{}} = 64
def thresh{c==1, T==i16 & shufb128{}} = 32
fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c==1, T==i8 if shufb128{}} = 64
def thresh{c==1, T==i16 if shufb128{}} = 32
fn slash{c==1, T if T<=i16 and shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def V = [16]i8
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
ind := load{itab, w}
@ -179,8 +179,8 @@ fn slash{c==1, T & T<=i16 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u6
# i32 w/x; 8 elts/iter into 2 steps; [16]i8 shuffle
i32tab:*u32 = maketab{4,8,2} # 16 elts, 64B
def thresh{c==1, T==i32 & shufb128{}} = 8
fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c==1, T==i32 if shufb128{}} = 8
fn slash{c==1, T==i32 if shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def V = [16]i8
expander := make{V, iota{16}>>2}
trail := make{V, tail{2,iota{16}}}
@ -202,9 +202,9 @@ fn slash{c==1, T==i32 & shufb128{}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) :
# i32 & i64 w/x & x+/w; 256 bits/step, 8 elts/iter; [8]i32 shuffle
i64tab:*u64 = maketab{4,16,1,{x}=>(1+x)*0x100 + x} # 16 elts, 128B
def thresh{c, T==i32 & hasarch{'AVX2'}} = 32
def thresh{c, T==i64 & hasarch{'AVX2'}} = 8
fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c, T==i32 if hasarch{'AVX2'}} = 32
def thresh{c, T==i64 if hasarch{'AVX2'}} = 8
fn slash{c, T if hasarch{'AVX2'} and T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def tw = width{T}
def V = [8]u32
expander := make{[32]u8, merge{...each{tup{., ... 3**128}, iota{8}>>lb{tw/32}}}}
@ -235,11 +235,11 @@ fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:
}
# everything; 512 bits/iter; AVX-512 compress
def thresh{c, T==i8 & hasarch{'AVX512VBMI2'}} = 256
def thresh{c, T==i16 & hasarch{'AVX512VBMI2'}} = 128
def thresh{c, T==i32 & hasarch{'AVX512F'}} = 64
def thresh{c, T==i64 & hasarch{'AVX512F'}} = 16
fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c, T==i8 if hasarch{'AVX512VBMI2'}} = 256
def thresh{c, T==i16 if hasarch{'AVX512VBMI2'}} = 128
def thresh{c, T==i32 if hasarch{'AVX512F'}} = 64
def thresh{c, T==i64 if hasarch{'AVX512F'}} = 16
fn slash{c, T if hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def f = fmtnat
def wt = width{T}
def vl = 512/wt
@ -283,7 +283,7 @@ def pext_popc{x:T, m:T} = {
# - z tells how many bits in the group are NOT used
# - x contains the bits, with z zeros above
def build{k==1} = tup{x&m, ~m}
def build{k & k > 1} = {
def build{k if k > 1} = {
def h = k>>1 # Increase size from h to k
{x,z} := build{h}
def low_s = lowbits{w,k} # Low bit in each new group
@ -326,12 +326,12 @@ def pext_popc{x:T, m:T} = {
pe := fold{|, x&s0, each{gr, g*slice{iota{k/g},1}}}
tup{pe, o>>(k-g)}
}
def build{k==32 & hasarch{'AVX2'} & isvec{T}} = {
def build{k==32 if hasarch{'AVX2'} and isvec{T}} = {
def S = re_el{ty_u{k}, T}
def c{T,vs} = each{{v}=>T~~v, vs}
c{T, multi_shift{...c{S, build{8}}, 8, k, {s}=>S**s}}
}
def build{k & ~isvec{T} & k > 8} = {
def build{k if not isvec{T} and k > 8} = {
multi_shift{...build{8}, 8, k, {s}=>s}
}
# Final result
@ -339,9 +339,9 @@ def pext_popc{x:T, m:T} = {
tup{pe, scal{w} - z}
}
def pext_width {& hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2
def thresh_bool{& hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32
def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = {
def pext_width {if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2
def thresh_bool{if hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32
def pext_popc{x0:V, m0:V if hasarch{'PCLMUL'} and V==[2]u64} = {
def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}}
m := m0
x := x0 & m
@ -359,9 +359,9 @@ def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = {
tup{x, @collect (j to 2) popc{extract{m0,j}}}
}
def pext_width {& fast_BMI2{}} = 1
def thresh_bool{& fast_BMI2{}} = 512
def pext_popc{x:T, m:T & fast_BMI2{} & T==u64} = tup{pext{x, m}, popc{m}}
def pext_width {if fast_BMI2{}} = 1
def thresh_bool{if fast_BMI2{}} = 512
def pext_popc{x:T, m:T if fast_BMI2{} and T==u64} = tup{pext{x, m}, popc{m}}
fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = {
cw:u64 = 0; # current word
@ -375,7 +375,7 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = {
}
ro = ro2%64
}
def extract{t, i & istup{t}} = tupsel{i,t}
def extract{t, i if istup{t}} = select{t,i}
def v = pext_width{}
if (v > 1) {
def V = [v]u64

View File

@ -7,24 +7,24 @@ include './vecfold'
def preserve_negative_zero = 0
# SSE2 versions avoid any 64-bit integer comparsions
def anySNaN{M, x:T & eltype{T}==u64} = {
def anySNaN{M, x:T if eltype{T}==u64} = {
homAny{inRangeLen{M{x}<<1, (0xFFE<<52)+2, (1<<52)-2}}
}
def anySNaN{M, x:T & T==[2]u64 & hasarch{'X86_64'} & ~hasarch{'SSE4.2'}} = {
def anySNaN{M, x:T if T==[2]u64 and hasarch{'X86_64'} and not hasarch{'SSE4.2'}} = {
topAny{M{andnot{unord{[2]f64~~x, [2]f64~~x}, [2]u64~~([4]u32**0xFFF8_0000 == ([4]u32~~x | [4]u32**0x8000_0000))}}}
}
def anyNonChar{M, x:T & isvec{T} & eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = {
def anyNonChar{M, x:T if isvec{T} and eltype{T}==u64} = homAny{M{~inRangeLen{x, cbqn_c32Tag{}<<48, 1<<48}}}
def anyNonChar{M, x:T if isvec{T} and hasarch{'X86_64'}} = {
def H = re_el{u32, T}
def ne = H~~x != H**cast_i{u32, cbqn_c32Tag{}<<16}
topAny{M{T~~ne}}
}
def cvtNarrow{T, x:X & width{T}==elwidth{X}} = cvt{T, x}
def cvtNarrow{T, x:X & width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X & elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X & elwidth{T}> elwidth{X}} = widen{T, x}
def cvtNarrow{T, x:X if width{T}==elwidth{X}} = cvt{T, x}
def cvtNarrow{T, x:X if width{T}< elwidth{X}} = narrow{T, x}
def cvtWiden{T, x:X if elwidth{T}==elwidth{X}} = cvt{eltype{T}, x}
def cvtWiden{T, x:X if elwidth{T}> elwidth{X}} = widen{T, x}
fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
assert{len>0}
@ -36,7 +36,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}}
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
def foldTotal{TE, x:T if hasarch{'AARCH64'}} = {
if (elwidth{T}==64) {
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
else vfold{max, narrow{TE, x}}
@ -92,21 +92,21 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
@muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) {
def int = {
def {int, wdn} = {
if (hasarch{'AARCH64'} and tuplen{is}==2) {
if (hasarch{'AARCH64'} and length{is}==2) {
def intp = narrowPair{...each{cvt{i64,.}, v0}}
def wdn = each{cvt{f64,.}, widen{intp}}
tup{intp, wdn}
} else {
def ints = each{{v} => cvtNarrow{ty_s{E}, v}, v0}
def wdn = each{{v} => cvtWiden{XV, v}, ints}
if (vcount{type{tupsel{0,ints}}} == bulk) {
if (vcount{type{select{ints,0}}} == bulk) {
def intp = {
if (tuplen{ints}==1) tupsel{0, ints}
if (length{ints}==1) select{ints, 0}
else pair{ints}
}
tup{intp, wdn}
} else if (tuplen{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) {
tup{tupsel{0, ints}, wdn}
} else if (length{ints}==1 and hasarch{'X86_64'} and ~hasarch{'AVX2'}) {
tup{select{ints, 0}, wdn}
} else assert{0}
}
}
@ -115,16 +115,16 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
def as = each{conv, v0}
def bs = each{conv, wdn}
def cond = {
if (tuplen{is}==1) anynePositive{...as, ...bs, M}
if (length{is}==1) anynePositive{...as, ...bs, M}
else ~homAll{tree_fold{&, each{==, as, bs}}}
}
if (cond) { # is any not an integer
if (B) case_B{tupsel{0, is}} # if B, need to give an even more special result
if (B) case_B{select{is, 0}} # if B, need to give an even more special result
else return{0xffff_ffff} # else, not integer => float
}
int
}
def acc = { if (tuplen{is}==2) r2; else r1 }
def acc = { if (length{is}==2) r2; else r1 }
acc|= M{getAcc{type{acc}, int}}
}

View File

@ -1,32 +1,32 @@
### SSSE3 ###
def sel{L, x:T, i:I & hasarch{'SSSE3'} & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:T, b:T, n & hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
def sel{L, x:T, i:I if hasarch{'SSSE3'} and lvec{L,16,8} and w128{T} and w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def vshl{a:T, b:T, n if hasarch{'SSSE3'}} = T~~emit{[16]u8, '_mm_alignr_epi8', v2i{b}, v2i{a}, n*(elwidth{T}/8)}
### SSE4.1 ###
def packs{a:T,b:T & hasarch{'SSE4.1'} & T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def packs{a:T,b:T if hasarch{'SSE4.1'} and T==[4]u32} = emit{[ 8]u16, '_mm_packus_epi32', a, b}
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
# conversion
def widen{T==[8]u16, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X & hasarch{'SSE4.1'} & X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X & hasarch{'SSE4.1'} & X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X & hasarch{'SSE4.1'} & w128i{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[8]u16, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi16', x}; def widen{T==[8]i16, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi16', x}
def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi32', x}
def widen{T==[4]u32, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi32', x}; def widen{T==[4]i32, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi32', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[16]u8} = emit{T, '_mm_cvtepu8_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[16]i8} = emit{T, '_mm_cvtepi8_epi64', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[8]u16} = emit{T, '_mm_cvtepu16_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[8]i16} = emit{T, '_mm_cvtepi16_epi64', x}
def widen{T==[2]u64, x:X if hasarch{'SSE4.1'} and X==[4]u32} = emit{T, '_mm_cvtepu32_epi64', x}; def widen{T==[2]i64, x:X if hasarch{'SSE4.1'} and X==[4]i32} = emit{T, '_mm_cvtepi32_epi64', x}
def widen{T==[2]f64, x:X if hasarch{'SSE4.1'} and w128i{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X & hasarch{'SSE4.1'} & w128i{X,32} & T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i8} = sel{[16]u8, [16]i8~~x, make{[16]i8, 0,4,8,12, 0,0,0,0, 0,0,0,0, 0,0,0,0}}
def narrow{T, x:X if hasarch{'SSE4.1'} and w128i{X,32} and T==i16} = sel{[16]u8, [8]i16~~x, make{[16]i8, 0,1,4,5, 8,9,12,13, 0,0,0,0, 0,0,0,0}}
# mask stuff
def andAllZero{x:T, y:T & hasarch{'SSE4.1'} & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
def andAllZero{x:T, y:T if hasarch{'SSE4.1'} and w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,32}} = T ~~ emit{[4]f32, '_mm_blendv_ps', v2f{f}, v2f{t}, v2f{m}}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
def topBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}}
# assumes all bits are the same in each mask item
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m}
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M} and elwidth{M}!=16} = topBlend{f, t, m}
def homBlend{f:T, t:T, m:M if hasarch{'SSE4.1'} and w128{T} and w128{M,16}} = topBlend{f, t, [16]i8~~m}

View File

@ -1,37 +1,37 @@
# compact casting for the annoying intrinsic type system
def v2i{x:T & w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x
def v2f{x:T & w128{T}} = [4]f32 ~~ x
def v2d{x:T & w128{T}} = [2]f64 ~~ x
def v2i{x:T if w128{T}} = if(isint{eltype{T}}) x else [16]u8 ~~ x
def v2f{x:T if w128{T}} = [4]f32 ~~ x
def v2d{x:T if w128{T}} = [2]f64 ~~ x
# load & store
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = load{ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def loadLow{ptr:P, w if w128{eltype{P}} and w==128} = load{ptr}
def storeLow{ptr:P, w, x:T & w128{T} & w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T & w128{T} & w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
def storeLow{ptr:P, w, x:T if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def storeLow{ptr:P, w, x:T if w128{T} and w==128} = store{*T~~ptr, 0, x}
# float comparison
def unord{a:T,b:T & T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T & T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
def unord{a:T,b:T if T==[4]f32} = [4]u32~~emit{[4]f32, '_mm_cmpunord_ps', a, b}
def unord{a:T,b:T if T==[2]f64} = [2]u64~~emit{[2]f64, '_mm_cmpunord_pd', a, b}
# shift
def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
def shl{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n}
def shr{S==[16]u8, x:T, n if w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n}
# integer arith
def mulh{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def mulh{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
def __mul{a:T,b:T & [4]i32==T} = {
def mulh{a:T,b:T if [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b}
def mulh{a:T,b:T if [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b}
def mul32{a:T,b:T if [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments
def __mul{a:T,b:T if [4]i32==T} = {
def mu{x, y} = [4]i32 ~~ mul32{[2]u64~~x, [2]u64~~y}
def sw{n, ...vs} = each{{c} => shuf{[4]i32, c, n}, vs}
lo:= mu{a, b}
@ -40,69 +40,69 @@ def __mul{a:T,b:T & [4]i32==T} = {
}
# float arith
def rsqrtE{a:[4]f32} = emit{[4]f32, '_mm_rsqrt_ps', a}
def rcpE{a:[4]f32} = emit{[4]f32, '_mm_rcp_ps', a}
def rsqrtE{a:([4]f32)} = emit{[4]f32, '_mm_rsqrt_ps', a}
def rcpE{a:([4]f32)} = emit{[4]f32, '_mm_rcp_ps', a}
# mask stuff
def andAllZero{x:T, y:T & w128i{T}} = homAll{(x & y) == T**0}
def andAllZero{x:T, y:T if w128i{T}} = homAll{(x & y) == T**0}
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T & w128{T}} = topMask{x}
def homMaskX{a:T & elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}}
def homMask{a:T, b:T & w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
def topMask{x:T if w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T if w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T if w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T if w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T if w128{T}} = topMask{x}
def homMaskX{a:T if elwidth{T}==16} = tup{2, homMask{re_el{u8,a}}}
def homMask{a:T, b:T if w128i{T,16}} = homMask{packs{ty_s{a},ty_s{b}}}
def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
def homAny{x:T if w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T if w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
def topAny{x:T & w128i{T}} = topMask{x} != 0
def topAll{x:T & w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
def topAny{x:T if w128i{T}} = topMask{x} != 0
def topAll{x:T if w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T if w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T if w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}
# bits of other things SSE2 has
def packs{a:T,b:T & T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T & T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T & T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T & w128i{T}} = packs{a,b}
def packs{a:T,b:T if T==[8]i16} = emit{[16]i8, '_mm_packs_epi16', a, b}
def packs{a:T,b:T if T==[4]i32} = emit{[ 8]i16, '_mm_packs_epi32', a, b}
def packs{a:T,b:T if T==[8]u16} = emit{[16]u8, '_mm_packus_epi16', a, b}
def packQ{a:T,b:T if w128i{T}} = packs{a,b}
def zipLo{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b}
def zipHi{a:T, b:T & w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b}
def zipLo{a:T, b:T & w128f{T}} = emit{T, merge{'_mm_unpacklo_p',if (elwidth{T}==32) 's' else 'd'}, a, b}
def zipHi{a:T, b:T & w128f{T}} = emit{T, merge{'_mm_unpackhi_p',if (elwidth{T}==32) 's' else 'd'}, a, b}
def zip{a:T, b:T & w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def zipLo{a:T, b:T if w128i{T}} = emit{T, merge{'_mm_unpacklo_epi',fmtnat{elwidth{T}}}, a, b}
def zipHi{a:T, b:T if w128i{T}} = emit{T, merge{'_mm_unpackhi_epi',fmtnat{elwidth{T}}}, a, b}
def zipLo{a:T, b:T if w128f{T}} = emit{T, merge{'_mm_unpacklo_p',if (elwidth{T}==32) 's' else 'd'}, a, b}
def zipHi{a:T, b:T if w128f{T}} = emit{T, merge{'_mm_unpackhi_p',if (elwidth{T}==32) 's' else 'd'}, a, b}
def zip{a:T, b:T if w128i{T}} = tup{zipLo{a,b}, zipHi{a,b}}
def unpackQ{a:T, b:T & w128{T}} = mzip{a, b}
def unpackQ{a:T, b:T if w128{T}} = mzip{a, b}
def shuf{L, x:T, n & w128{T} & lvec{L,4,32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n if w128{T} and lvec{L,4,32} and knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def shuf16Lo{x:T, n} = T~~emit{[8]i16, '_mm_shufflelo_epi16', x, n}
def shuf16Hi{x:T, n} = T~~emit{[8]i16, '_mm_shufflehi_epi16', x, n}
def homBlend{f:T, t:T, m:M & w128{T} & w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f &~ m))
def homMaskStoreF{p:P, m:M, v:T & w128i{M} & w128{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def homBlend{f:T, t:T, m:M if w128{T} and w128i{M,elwidth{T}}} = T ~~ ((M~~t & m) | (M~~f &~ m))
def homMaskStoreF{p:P, m:M, v:T if w128i{M} and w128{T,elwidth{M}} and eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
def widen{T, x:X & w128i{T} & w128i{X} & w128s{T}==w128s{X} & elwidth{T}>elwidth{X}} = {
def widen{T, x:X if w128i{T} and w128i{X} and w128s{T}==w128s{X} and elwidth{T}>elwidth{X}} = {
def s{v} = s{mzipLo{v, v}}
def s{v:V & V==T} = v
def s{v:V if V==T} = v
s{x} >> (elwidth{T} - elwidth{X})
}
def widen{T==[2]f64, x:X & w128s{X} & elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[2]f64, x:X & X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X & X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def widen{T==[2]f64, x:X if w128s{X} and elwidth{X}<32} = widen{T, widen{[4]i32, x}}
def widen{T==[2]f64, x:X if X==[4]i32} = emit{T, '_mm_cvtepi32_pd', x}
def widen{T==[2]f64, x:X if X==[4]f32} = emit{T, '_mm_cvtps_pd', x}
def narrow{T==i16, x:[4]i32} = packs{x,x}
def narrow{T==i8, x:[8]i16} = packs{x,x}
def narrow{T==u8, x:[8]u16} = packs{x,x}
def narrow{T==u16, x:[4]u32} = [8]u16~~shuf{[4]i32, shuf16Hi{shuf16Lo{x, 4b3320}, 4b3320}, 4b3320}
def narrow{T==i8, x:[4]i32} = narrow{T, narrow{i16, x}}
def narrow{T==u8, x:[4]u32} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{x}}}
def narrow{T==u8, x:[2]u64} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{f{x}}}}
def narrow{T==u16, x:[2]u64} = shuf16Lo{[8]u16~~shuf{[4]i32, x, 4b3320}, 4b3320}
def narrow{T==u32, x:[2]u64} = [4]u32~~shuf{[4]i32, x, 4b3320}
def narrow{T==i16, x:([4]i32)} = packs{x,x}
def narrow{T==i8, x:([8]i16)} = packs{x,x}
def narrow{T==u8, x:([8]u16)} = packs{x,x}
def narrow{T==u16, x:([4]u32)} = [8]u16~~shuf{[4]i32, shuf16Hi{shuf16Lo{x, 4b3320}, 4b3320}, 4b3320}
def narrow{T==i8, x:([4]i32)} = narrow{T, narrow{i16, x}}
def narrow{T==u8, x:([4]u32)} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{x}}}
def narrow{T==u8, x:([2]u64)} = { def f{v} = narrow{u8, [8]u16~~v}; f{f{f{x}}}}
def narrow{T==u16, x:([2]u64)} = shuf16Lo{[8]u16~~shuf{[4]i32, x, 4b3320}, 4b3320}
def narrow{T==u32, x:([2]u64)} = [4]u32~~shuf{[4]i32, x, 4b3320}
def narrow{T, x:X & w128f{X,64} & T<i32} = narrow{T, narrow{i32, x}}
def narrow{T==i32, x:[2]f64} = emit{[4]i32, '_mm_cvtpd_epi32', x}
def narrow{T, x:X if w128f{X,64} and T<i32} = narrow{T, narrow{i32, x}}
def narrow{T==i32, x:([2]f64)} = emit{[4]i32, '_mm_cvtpd_epi32', x}

View File

@ -7,9 +7,9 @@ include './bitops'
# Group l (power of 2) elements into paired groups of length o
# e.g. pairs{2, iota{8}} = {{0,1,4,5}, {2,3,6,7}}
def pairs{o, x} = {
def i = iota{tuplen{x}/2}
def i = iota{length{x}/2}
def g = 2*i - i%o
tupsel{tup{g, g+o}, x}
select{x, tup{g, g+o}}
}
def unpack_pass{o, x} = merge{...each{unpackQ, ...pairs{o, x}}}
def permute_pass{o, x} = {
@ -25,16 +25,16 @@ def unpack_to{f, l, x} = {
def shuf_pass{x} = each{{v} => shuf{[4]i64, v, 4b3120}, x}
# Square kernel where width is a full vector
def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x}
def transpose_square{VT, l, x if hasarch{'AVX2'}} = unpack_to{1, l/2, x}
def load2{a:T, b:T} = pair{load{a}, load{b}}
def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = {
def store2{a:T, b:T, v:T2 if w128i{eltype{T}} and w256{T2}} = {
each{{p:P, i} => store{p, 0, eltype{P}~~half{v,i}}, tup{a,b}, iota{2}}
}
def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
def load_k {VT, src, l, w if w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}
def store_k{VT, dst, x, l, h if w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x}
def load_k {VT, src, l, w if w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}}
def store_k{VT, dst, x, l, h if w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x}
# Transpose kernel of size kw,kh in size w,h array
def kernel{src:P, dst:P, kw, kh, w, h} = {

View File

@ -1,6 +1,6 @@
# Fold associative/commutative operation across a register
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
def vfold{F, x:T if w128{T} and hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
@ -9,4 +9,4 @@ def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} }
else extract{c, 0}
}
def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}
def vfold{F, x:T if w256{T} and hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}