move storeLow & loadLow to sse3 & avx; fix NEON __not
This commit is contained in:
parent
9fea5f1e13
commit
77edc734e8
@ -28,6 +28,13 @@ def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f6
|
||||
def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}}
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr}
|
||||
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==256} = store{*T~~ptr, 0, x}
|
||||
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w256i{T, 8}} = emit{T, '_mm256_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w256i{T, 16}} = emit{T, '_mm256_set1_epi16', promote{eltype{T},v}}
|
||||
|
||||
@ -56,7 +56,7 @@ def spreadBits{T==[32]u8, a:u32} = {
|
||||
e:= make{[32]u8, 1<<tail{3, idxs}}
|
||||
e == (d&e)
|
||||
}
|
||||
def spreadBits{T, a & vcount{T} <= elwidth{T} & w256u{T}} = {
|
||||
def spreadBits{T, a & vcount{T} <= elwidth{T} & width{T}==256 & quality{eltype{T}}=='u'} = {
|
||||
b:= make{T, 1<<iota{vcount{T}}}
|
||||
b == (b & T ~~ to_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
|
||||
}
|
||||
|
||||
@ -37,25 +37,6 @@ def maskAfter{n} = {
|
||||
|
||||
|
||||
# store low packed elements of x to P
|
||||
def storeLow{ptr:P, w, x:T & width{T}==w} = store{*T~~ptr, 0, x}
|
||||
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w==16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w==32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w==64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
|
||||
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==16} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==32} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = storeLow{ptr, w, half{x, 0}}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==128} = store{*[128/elwidth{T}](eltype{T})~~ptr, 0, half{x, 0}}
|
||||
|
||||
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = eltype{P} ~~ load{*[16]u8 ~~ ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
|
||||
|
||||
def loadLow{ptr:P, w & w256{eltype{P}} & w<256} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}}
|
||||
def loadLow{ptr:P, w & elwidth{P} == w} = load{*eltype{P} ~~ ptr}
|
||||
|
||||
def loadLowBatch{T, ptr:P, w, n & eltype{P}==eltype{T}} = loadLow{*T ~~ (ptr + n*(w/elwidth{P})), w}
|
||||
|
||||
|
||||
@ -48,7 +48,7 @@ def bblend{f:T, t:T, m:M & nvec{T} & nvecu{M} & width{T}==width{M} & elwidth{T}=
|
||||
|
||||
def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a}
|
||||
def addpw{a:T & nveci{T} & elwidth{T}<=32} = emit{ty_dbl{T}, ntyp{'vpaddl', T}, a}
|
||||
def __not{a:T & nvecu{T}} = emit{T, ntyp{'vmvn', T}, a}
|
||||
def __not{a:T & nvecu{T}} = T~~emit{to_el{u8,T}, ntyp{'vmvn', to_el{u8,T}}, a}
|
||||
def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a}
|
||||
def floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a}
|
||||
def ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a}
|
||||
|
||||
@ -26,6 +26,17 @@ def storea{a:T, n, v & w128f{eltype{T},64}} = emit{void, '_mm_store_pd', *f64 ~
|
||||
def store {a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_storeu_ps', *f32 ~~ (a+n), v}
|
||||
def storea{a:T, n, v & w128f{eltype{T},32}} = emit{void, '_mm_store_ps', *f32 ~~ (a+n), v}
|
||||
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 16} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 32} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w== 64} = eltype{P} ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
|
||||
def loadLow{ptr:P, w & w128{eltype{P}} & w==128} = load{ptr}
|
||||
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
|
||||
def storeLow{ptr:P, w, x:T & w128{T} & w==128} = store{*T~~ptr, 0, x}
|
||||
|
||||
|
||||
# broadcast
|
||||
def broadcast{T, v & w128i{T, 8}} = emit{T, '_mm_set1_epi8', promote{eltype{T},v}}
|
||||
def broadcast{T, v & w128i{T, 16}} = emit{T, '_mm_set1_epi16', promote{eltype{T},v}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user