Singeli copy from unaligned bitarr
This commit is contained in:
parent
63fa511c55
commit
c32285dba9
@ -72,8 +72,8 @@
|
|||||||
#define rand_popc64(X) POPC(X)
|
#define rand_popc64(X) POPC(X)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
||||||
u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||||
#if SINGELI_X86_64
|
#if SINGELI_X86_64
|
||||||
#define SINGELI_FILE slash
|
#define SINGELI_FILE slash
|
||||||
#include "../utils/includeSingeli.h"
|
#include "../utils/includeSingeli.h"
|
||||||
|
|||||||
@ -23,7 +23,7 @@ u8 elTypeWidthLogBits[] = {
|
|||||||
[el_B] = 6
|
[el_B] = 6
|
||||||
};
|
};
|
||||||
u8 arrTypeWidthLog[] = {
|
u8 arrTypeWidthLog[] = {
|
||||||
[t_bitarr]=99,
|
[t_bitarr]=0, // 0 for mut.c hack to allow restoring a bitarr offset from array pointer & arrTypeWidthLog-"corrected" element pointer
|
||||||
[t_i8arr ]=0, [t_i8slice ]=0, [t_c8arr ]=0, [t_c8slice ]=0,
|
[t_i8arr ]=0, [t_i8slice ]=0, [t_c8arr ]=0, [t_c8slice ]=0,
|
||||||
[t_i16arr]=1, [t_i16slice]=1, [t_c16arr]=1, [t_c16slice]=1,
|
[t_i16arr]=1, [t_i16slice]=1, [t_c16arr]=1, [t_c16slice]=1,
|
||||||
[t_i32arr]=2, [t_i32slice]=2, [t_c32arr]=2, [t_c32slice]=2,
|
[t_i32arr]=2, [t_i32slice]=2, [t_c32arr]=2, [t_c32slice]=2,
|
||||||
|
|||||||
@ -81,6 +81,18 @@ def truncBits{n, v & n<=8} = cast_i{u8, v}
|
|||||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||||
|
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||||
|
|
||||||
|
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
|
||||||
|
def loaduBitRaw{x:*u64, i} = {
|
||||||
|
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
|
||||||
|
}
|
||||||
|
def loaduBit{x:*u64, i, n} = {
|
||||||
|
assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))}
|
||||||
|
loaduBitRaw{x, i}
|
||||||
|
}
|
||||||
|
def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}}
|
||||||
|
|
||||||
|
|
||||||
def loadBatchBit{T, x:*u64, is & ktup{is}} = {
|
def loadBatchBit{T, x:*u64, is & ktup{is}} = {
|
||||||
# def len = tuplen{is}
|
# def len = tuplen{is}
|
||||||
|
|||||||
@ -30,3 +30,4 @@ def cbqn_elType{T & T==u8 } = 5
|
|||||||
def cbqn_elType{T & T==u16} = 6
|
def cbqn_elType{T & T==u16} = 6
|
||||||
def cbqn_elType{T & T==u32} = 7
|
def cbqn_elType{T & T==u32} = 7
|
||||||
|
|
||||||
|
def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'}
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
include './base'
|
include './base'
|
||||||
|
include './debug'
|
||||||
if (hasarch{'X86_64'}) {
|
if (hasarch{'X86_64'}) {
|
||||||
include './sse3'
|
include './sse3'
|
||||||
include './avx'
|
include './avx'
|
||||||
@ -10,14 +11,14 @@ include './mask'
|
|||||||
include './cbqnDefs'
|
include './cbqnDefs'
|
||||||
include './bitops'
|
include './bitops'
|
||||||
|
|
||||||
def copyFromBits{T, xp: *u64, rp: *eltype{T}, l:u64} = {
|
def copyFromBits{T, loadFn, rp, l:u64} = {
|
||||||
def bulk = vcount{T}
|
def bulk = vcount{T}
|
||||||
def TU = ty_u{T}
|
def TU = ty_u{T}
|
||||||
|
|
||||||
maskedLoop{bulk, l, {i, M} => {
|
maskedLoop{bulk, l, {i, M} => {
|
||||||
x:= loadBatchBit{TU, xp, i}
|
x:= loadFn{TU, i}
|
||||||
r:= x & TU ~~ T**1
|
r:= x & TU ~~ T**1
|
||||||
storeBatch{rp, i, T~~r, M}
|
storeBatch{*eltype{T}~~rp, i, T~~r, M}
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -50,7 +51,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
|||||||
}}
|
}}
|
||||||
} else if (X==u1) {
|
} else if (X==u1) {
|
||||||
# show{'X==u1', X, R}
|
# show{'X==u1', X, R}
|
||||||
copyFromBits{[bulk]R, *u64~~x, *R~~r, l}
|
copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l}
|
||||||
} else if (R==u1) {
|
} else if (R==u1) {
|
||||||
# show{'R==u1', X, R}
|
# show{'R==u1', X, R}
|
||||||
def XU = ty_u{XV}
|
def XU = ty_u{XV}
|
||||||
@ -74,6 +75,22 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||||
|
assert{l!=0}
|
||||||
|
x0:= (*u8~~xRaw) + cbqn_tyArrOffset{}
|
||||||
|
xs:= u64~~((*u8~~x) - x0)
|
||||||
|
# if ((xs&7)==0) {
|
||||||
|
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
|
||||||
|
# } else {
|
||||||
|
def vw = arch_defvw
|
||||||
|
def bulk = vw/width{R}
|
||||||
|
def RV = [bulk]R
|
||||||
|
rp:= *R~~r
|
||||||
|
|
||||||
|
copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l}
|
||||||
|
# }
|
||||||
|
}
|
||||||
|
|
||||||
# simd_copy_src_dst
|
# simd_copy_src_dst
|
||||||
# x→int & equal-width copies
|
# x→int & equal-width copies
|
||||||
'simd_copy_1_1' = copy{u1, u1}
|
'simd_copy_1_1' = copy{u1, u1}
|
||||||
@ -125,3 +142,9 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
|||||||
'simd_copy_B_c8' = copy{u64, u8}
|
'simd_copy_B_c8' = copy{u64, u8}
|
||||||
'simd_copy_B_c16' = copy{u64, u16}
|
'simd_copy_B_c16' = copy{u64, u16}
|
||||||
'simd_copy_B_c32' = copy{u64, u32}
|
'simd_copy_B_c32' = copy{u64, u32}
|
||||||
|
|
||||||
|
# unaligned bitarr widening
|
||||||
|
'simd_copy_1u_i8' = copy_ubit{i8}
|
||||||
|
'simd_copy_1u_i16' = copy_ubit{i16}
|
||||||
|
'simd_copy_1u_i32' = copy_ubit{i32}
|
||||||
|
'simd_copy_1u_f64' = copy_ubit{f64}
|
||||||
@ -220,6 +220,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
|||||||
}
|
}
|
||||||
|
|
||||||
#if SINGELI
|
#if SINGELI
|
||||||
|
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||||
#define SINGELI_FILE copy
|
#define SINGELI_FILE copy
|
||||||
#include "./includeSingeli.h"
|
#include "./includeSingeli.h"
|
||||||
typedef void (*copy_fn)(void*, void*, u64, void*);
|
typedef void (*copy_fn)(void*, void*, u64, void*);
|
||||||
@ -237,9 +238,8 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
|||||||
} \
|
} \
|
||||||
static void cpy##T##Arr_B(void* xp, void* rp, u64 ia, void* xRaw) { \
|
static void cpy##T##Arr_B(void* xp, void* rp, u64 ia, void* xRaw) { \
|
||||||
Arr* xa = (Arr*)xRaw; B* bxp = arrV_bptr(xa); \
|
Arr* xa = (Arr*)xRaw; B* bxp = arrV_bptr(xa); \
|
||||||
if (bxp!=NULL && sizeof(B)==sizeof(f64)) { \
|
if (bxp!=NULL && sizeof(B)==sizeof(f64)) H2T; \
|
||||||
H2T; \
|
else cpy##T##Arr_BF(xp, rp, ia, xa); \
|
||||||
} else cpy##T##Arr_BF(xp, rp, ia, xa); \
|
|
||||||
} \
|
} \
|
||||||
static copy_fn copy##T##Fns[] = __VA_ARGS__; \
|
static copy_fn copy##T##Fns[] = __VA_ARGS__; \
|
||||||
T##Arr* cpy##T##Arr(B x) { \
|
T##Arr* cpy##T##Arr(B x) { \
|
||||||
@ -263,24 +263,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
|||||||
Bit, {COPY_FN(1,1),COPY_FN(i8,1),COPY_FN(i16,1),COPY_FN(i32,1),COPY_FN(f64,1),badCopy, badCopy, badCopy, cpyBitArr_B, COPY_FN(f64,1)})
|
Bit, {COPY_FN(1,1),COPY_FN(i8,1),COPY_FN(i16,1),COPY_FN(i32,1),COPY_FN(f64,1),badCopy, badCopy, badCopy, cpyBitArr_B, COPY_FN(f64,1)})
|
||||||
|
|
||||||
|
|
||||||
static copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
|
copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1u_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
|
||||||
static copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
|
copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1u_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
|
||||||
static copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
|
copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1u_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
|
||||||
static copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
|
copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1u_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
|
||||||
static copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
|
copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
|
||||||
static copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
|
copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
|
||||||
static copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
|
copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
|
||||||
|
|
||||||
#define TCOPY_FN(T, N, NUM) static void m_copyG_##N(void* a, usz ms, B x, usz xs, usz l) { \
|
#define TCOPY_FN(T, N, NUM) static void m_copyG_##N(void* a, usz ms, B x, usz xs, usz l) { \
|
||||||
if (l==0) return; \
|
if (l==0) return; \
|
||||||
void* xp = tyany_ptr(x); \
|
void* xp = tyany_ptr(x); \
|
||||||
T* rp = ms + (T*)a; \
|
|
||||||
u8 xt = TY(x); \
|
u8 xt = TY(x); \
|
||||||
if (NUM && xt==t_bitarr) { \
|
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, ms + (T*)a, l, a(x)); \
|
||||||
for (usz i = 0; i < l; i++) rp[i] = bitp_get((u64*)xp, xs+i); \
|
|
||||||
} else { \
|
|
||||||
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, rp, l, a(x)); \
|
|
||||||
} \
|
|
||||||
}
|
}
|
||||||
TCOPY_FN(i8,i8, 1)
|
TCOPY_FN(i8,i8, 1)
|
||||||
TCOPY_FN(i16,i16, 1)
|
TCOPY_FN(i16,i16, 1)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user