Singeli copy from unaligned bitarr
This commit is contained in:
parent
63fa511c55
commit
c32285dba9
@ -72,8 +72,8 @@
|
||||
#define rand_popc64(X) POPC(X)
|
||||
#endif
|
||||
|
||||
void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
||||
u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
||||
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||
#if SINGELI_X86_64
|
||||
#define SINGELI_FILE slash
|
||||
#include "../utils/includeSingeli.h"
|
||||
|
||||
@ -23,7 +23,7 @@ u8 elTypeWidthLogBits[] = {
|
||||
[el_B] = 6
|
||||
};
|
||||
u8 arrTypeWidthLog[] = {
|
||||
[t_bitarr]=99,
|
||||
[t_bitarr]=0, // 0 for mut.c hack to allow restoring a bitarr offset from array pointer & arrTypeWidthLog-"corrected" element pointer
|
||||
[t_i8arr ]=0, [t_i8slice ]=0, [t_c8arr ]=0, [t_c8slice ]=0,
|
||||
[t_i16arr]=1, [t_i16slice]=1, [t_c16arr]=1, [t_c16slice]=1,
|
||||
[t_i32arr]=2, [t_i32slice]=2, [t_c32arr]=2, [t_c32slice]=2,
|
||||
|
||||
@ -81,6 +81,18 @@ def truncBits{n, v & n<=8} = cast_i{u8, v}
|
||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
|
||||
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
|
||||
def loaduBitRaw{x:*u64, i} = {
|
||||
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
|
||||
}
|
||||
def loaduBit{x:*u64, i, n} = {
|
||||
assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))}
|
||||
loaduBitRaw{x, i}
|
||||
}
|
||||
def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}}
|
||||
|
||||
|
||||
def loadBatchBit{T, x:*u64, is & ktup{is}} = {
|
||||
# def len = tuplen{is}
|
||||
|
||||
@ -30,3 +30,4 @@ def cbqn_elType{T & T==u8 } = 5
|
||||
def cbqn_elType{T & T==u16} = 6
|
||||
def cbqn_elType{T & T==u32} = 7
|
||||
|
||||
def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
include './base'
|
||||
include './debug'
|
||||
if (hasarch{'X86_64'}) {
|
||||
include './sse3'
|
||||
include './avx'
|
||||
@ -10,14 +11,14 @@ include './mask'
|
||||
include './cbqnDefs'
|
||||
include './bitops'
|
||||
|
||||
def copyFromBits{T, xp: *u64, rp: *eltype{T}, l:u64} = {
|
||||
def copyFromBits{T, loadFn, rp, l:u64} = {
|
||||
def bulk = vcount{T}
|
||||
def TU = ty_u{T}
|
||||
|
||||
maskedLoop{bulk, l, {i, M} => {
|
||||
x:= loadBatchBit{TU, xp, i}
|
||||
x:= loadFn{TU, i}
|
||||
r:= x & TU ~~ T**1
|
||||
storeBatch{rp, i, T~~r, M}
|
||||
storeBatch{*eltype{T}~~rp, i, T~~r, M}
|
||||
}}
|
||||
}
|
||||
|
||||
@ -50,7 +51,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
}}
|
||||
} else if (X==u1) {
|
||||
# show{'X==u1', X, R}
|
||||
copyFromBits{[bulk]R, *u64~~x, *R~~r, l}
|
||||
copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l}
|
||||
} else if (R==u1) {
|
||||
# show{'R==u1', X, R}
|
||||
def XU = ty_u{XV}
|
||||
@ -74,6 +75,22 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
assert{l!=0}
|
||||
x0:= (*u8~~xRaw) + cbqn_tyArrOffset{}
|
||||
xs:= u64~~((*u8~~x) - x0)
|
||||
# if ((xs&7)==0) {
|
||||
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
|
||||
# } else {
|
||||
def vw = arch_defvw
|
||||
def bulk = vw/width{R}
|
||||
def RV = [bulk]R
|
||||
rp:= *R~~r
|
||||
|
||||
copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l}
|
||||
# }
|
||||
}
|
||||
|
||||
# simd_copy_src_dst
|
||||
# x→int & equal-width copies
|
||||
'simd_copy_1_1' = copy{u1, u1}
|
||||
@ -125,3 +142,9 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
|
||||
'simd_copy_B_c8' = copy{u64, u8}
|
||||
'simd_copy_B_c16' = copy{u64, u16}
|
||||
'simd_copy_B_c32' = copy{u64, u32}
|
||||
|
||||
# unaligned bitarr widening
|
||||
'simd_copy_1u_i8' = copy_ubit{i8}
|
||||
'simd_copy_1u_i16' = copy_ubit{i16}
|
||||
'simd_copy_1u_i32' = copy_ubit{i32}
|
||||
'simd_copy_1u_f64' = copy_ubit{f64}
|
||||
@ -220,6 +220,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
||||
}
|
||||
|
||||
#if SINGELI
|
||||
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||
#define SINGELI_FILE copy
|
||||
#include "./includeSingeli.h"
|
||||
typedef void (*copy_fn)(void*, void*, u64, void*);
|
||||
@ -237,20 +238,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
||||
} \
|
||||
static void cpy##T##Arr_B(void* xp, void* rp, u64 ia, void* xRaw) { \
|
||||
Arr* xa = (Arr*)xRaw; B* bxp = arrV_bptr(xa); \
|
||||
if (bxp!=NULL && sizeof(B)==sizeof(f64)) { \
|
||||
H2T; \
|
||||
} else cpy##T##Arr_BF(xp, rp, ia, xa); \
|
||||
if (bxp!=NULL && sizeof(B)==sizeof(f64)) H2T; \
|
||||
else cpy##T##Arr_BF(xp, rp, ia, xa); \
|
||||
} \
|
||||
static copy_fn copy##T##Fns[] = __VA_ARGS__; \
|
||||
T##Arr* cpy##T##Arr(B x) { \
|
||||
usz ia = IA(x); \
|
||||
MAKE; arr_shCopy(r, x); \
|
||||
if (ia>0) { \
|
||||
T##Arr* cpy##T##Arr(B x) { \
|
||||
usz ia = IA(x); \
|
||||
MAKE; arr_shCopy(r, x); \
|
||||
if (ia>0) { \
|
||||
copy##T##Fns[TI(x,elType)](tyany_ptr(x), XRP, ia, a(x)); \
|
||||
} \
|
||||
if (TY) ptr_decT(a(x)); \
|
||||
else decG(x); \
|
||||
return (T##Arr*)r; \
|
||||
} \
|
||||
if (TY) ptr_decT(a(x)); \
|
||||
else decG(x); \
|
||||
return (T##Arr*)r; \
|
||||
}
|
||||
#define BIT_PUT(V) bitp_set((u64*)rp, i, o2bG(V))
|
||||
#define H2T_COPY(T) copy##T##Fns[el_MAX](bxp, rp, ia, xRaw)
|
||||
@ -263,24 +263,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
||||
Bit, {COPY_FN(1,1),COPY_FN(i8,1),COPY_FN(i16,1),COPY_FN(i32,1),COPY_FN(f64,1),badCopy, badCopy, badCopy, cpyBitArr_B, COPY_FN(f64,1)})
|
||||
|
||||
|
||||
static copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
|
||||
static copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
|
||||
static copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
|
||||
static copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
|
||||
static copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
|
||||
static copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
|
||||
static copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
|
||||
copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1u_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
|
||||
copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1u_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
|
||||
copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1u_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
|
||||
copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1u_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
|
||||
copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
|
||||
copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
|
||||
copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
|
||||
|
||||
#define TCOPY_FN(T, N, NUM) static void m_copyG_##N(void* a, usz ms, B x, usz xs, usz l) { \
|
||||
if (l==0) return; \
|
||||
void* xp = tyany_ptr(x); \
|
||||
T* rp = ms + (T*)a; \
|
||||
u8 xt = TY(x); \
|
||||
if (NUM && xt==t_bitarr) { \
|
||||
for (usz i = 0; i < l; i++) rp[i] = bitp_get((u64*)xp, xs+i); \
|
||||
} else { \
|
||||
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, rp, l, a(x)); \
|
||||
} \
|
||||
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, ms + (T*)a, l, a(x)); \
|
||||
}
|
||||
TCOPY_FN(i8,i8, 1)
|
||||
TCOPY_FN(i16,i16, 1)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user