Singeli copy from unaligned bitarr

This commit is contained in:
dzaima 2023-02-18 11:35:51 +02:00
parent 63fa511c55
commit c32285dba9
6 changed files with 62 additions and 31 deletions

View File

@ -72,8 +72,8 @@
#define rand_popc64(X) POPC(X)
#endif
void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#if SINGELI_X86_64
#define SINGELI_FILE slash
#include "../utils/includeSingeli.h"

View File

@ -23,7 +23,7 @@ u8 elTypeWidthLogBits[] = {
[el_B] = 6
};
u8 arrTypeWidthLog[] = {
[t_bitarr]=99,
[t_bitarr]=0, // 0 for mut.c hack to allow restoring a bitarr offset from array pointer & arrTypeWidthLog-"corrected" element pointer
[t_i8arr ]=0, [t_i8slice ]=0, [t_c8arr ]=0, [t_c8slice ]=0,
[t_i16arr]=1, [t_i16slice]=1, [t_c16arr]=1, [t_c16slice]=1,
[t_i32arr]=2, [t_i32slice]=2, [t_c32arr]=2, [t_c32slice]=2,

View File

@ -81,6 +81,18 @@ def truncBits{n, v & n<=8} = cast_i{u8, v}
def truncBits{n, v & n==16} = cast_i{u16, v}
def truncBits{n, v & n==32} = cast_i{u32, v}
def truncBits{n, v & n==64} = cast_i{u64, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
def loaduBitRaw{x:*u64, i} = {
loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7)
}
def loaduBit{x:*u64, i, n} = {
assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))}
loaduBitRaw{x, i}
}
def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}}
def loadBatchBit{T, x:*u64, is & ktup{is}} = {
# def len = tuplen{is}

View File

@ -30,3 +30,4 @@ def cbqn_elType{T & T==u8 } = 5
def cbqn_elType{T & T==u16} = 6
def cbqn_elType{T & T==u32} = 7
def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'}

View File

@ -1,4 +1,5 @@
include './base'
include './debug'
if (hasarch{'X86_64'}) {
include './sse3'
include './avx'
@ -10,14 +11,14 @@ include './mask'
include './cbqnDefs'
include './bitops'
def copyFromBits{T, xp: *u64, rp: *eltype{T}, l:u64} = {
def copyFromBits{T, loadFn, rp, l:u64} = {
def bulk = vcount{T}
def TU = ty_u{T}
maskedLoop{bulk, l, {i, M} => {
x:= loadBatchBit{TU, xp, i}
x:= loadFn{TU, i}
r:= x & TU ~~ T**1
storeBatch{rp, i, T~~r, M}
storeBatch{*eltype{T}~~rp, i, T~~r, M}
}}
}
@ -50,7 +51,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
}}
} else if (X==u1) {
# show{'X==u1', X, R}
copyFromBits{[bulk]R, *u64~~x, *R~~r, l}
copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l}
} else if (R==u1) {
# show{'R==u1', X, R}
def XU = ty_u{XV}
@ -74,6 +75,22 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
}
}
copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
assert{l!=0}
x0:= (*u8~~xRaw) + cbqn_tyArrOffset{}
xs:= u64~~((*u8~~x) - x0)
# if ((xs&7)==0) {
# copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw)
# } else {
def vw = arch_defvw
def bulk = vw/width{R}
def RV = [bulk]R
rp:= *R~~r
copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l}
# }
}
# simd_copy_src_dst
# x→int & equal-width copies
'simd_copy_1_1' = copy{u1, u1}
@ -125,3 +142,9 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
'simd_copy_B_c8' = copy{u64, u8}
'simd_copy_B_c16' = copy{u64, u16}
'simd_copy_B_c32' = copy{u64, u32}
# unaligned bitarr widening
'simd_copy_1u_i8' = copy_ubit{i8}
'simd_copy_1u_i16' = copy_ubit{i16}
'simd_copy_1u_i32' = copy_ubit{i32}
'simd_copy_1u_f64' = copy_ubit{f64}

View File

@ -220,6 +220,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
}
#if SINGELI
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#define SINGELI_FILE copy
#include "./includeSingeli.h"
typedef void (*copy_fn)(void*, void*, u64, void*);
@ -237,20 +238,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
} \
static void cpy##T##Arr_B(void* xp, void* rp, u64 ia, void* xRaw) { \
Arr* xa = (Arr*)xRaw; B* bxp = arrV_bptr(xa); \
if (bxp!=NULL && sizeof(B)==sizeof(f64)) { \
H2T; \
} else cpy##T##Arr_BF(xp, rp, ia, xa); \
if (bxp!=NULL && sizeof(B)==sizeof(f64)) H2T; \
else cpy##T##Arr_BF(xp, rp, ia, xa); \
} \
static copy_fn copy##T##Fns[] = __VA_ARGS__; \
T##Arr* cpy##T##Arr(B x) { \
usz ia = IA(x); \
MAKE; arr_shCopy(r, x); \
if (ia>0) { \
T##Arr* cpy##T##Arr(B x) { \
usz ia = IA(x); \
MAKE; arr_shCopy(r, x); \
if (ia>0) { \
copy##T##Fns[TI(x,elType)](tyany_ptr(x), XRP, ia, a(x)); \
} \
if (TY) ptr_decT(a(x)); \
else decG(x); \
return (T##Arr*)r; \
} \
if (TY) ptr_decT(a(x)); \
else decG(x); \
return (T##Arr*)r; \
}
#define BIT_PUT(V) bitp_set((u64*)rp, i, o2bG(V))
#define H2T_COPY(T) copy##T##Fns[el_MAX](bxp, rp, ia, xRaw)
@ -263,24 +263,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
Bit, {COPY_FN(1,1),COPY_FN(i8,1),COPY_FN(i16,1),COPY_FN(i32,1),COPY_FN(f64,1),badCopy, badCopy, badCopy, cpyBitArr_B, COPY_FN(f64,1)})
static copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
static copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
static copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
static copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
static copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
static copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
static copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1u_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8};
copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1u_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16};
copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1u_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32};
copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1u_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64};
copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8};
copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16};
copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32};
#define TCOPY_FN(T, N, NUM) static void m_copyG_##N(void* a, usz ms, B x, usz xs, usz l) { \
if (l==0) return; \
void* xp = tyany_ptr(x); \
T* rp = ms + (T*)a; \
u8 xt = TY(x); \
if (NUM && xt==t_bitarr) { \
for (usz i = 0; i < l; i++) rp[i] = bitp_get((u64*)xp, xs+i); \
} else { \
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, rp, l, a(x)); \
} \
tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, ms + (T*)a, l, a(x)); \
}
TCOPY_FN(i8,i8, 1)
TCOPY_FN(i16,i16, 1)