From c32285dba9eff1e576f65fc77a3b184e4a24bd69 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 18 Feb 2023 11:35:51 +0200 Subject: [PATCH] Singeli copy from unaligned bitarr --- src/builtins/slash.c | 4 +-- src/core/tyarr.c | 2 +- src/singeli/src/bitops.singeli | 12 +++++++++ src/singeli/src/cbqnDefs.singeli | 1 + src/singeli/src/copy.singeli | 31 ++++++++++++++++++++--- src/utils/mut.c | 43 ++++++++++++++------------------ 6 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7f789b65..3c09c4f3 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -72,8 +72,8 @@ #define rand_popc64(X) POPC(X) #endif - void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } - u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } + static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } + static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #if SINGELI_X86_64 #define SINGELI_FILE slash #include "../utils/includeSingeli.h" diff --git a/src/core/tyarr.c b/src/core/tyarr.c index 50d49f91..1344655f 100644 --- a/src/core/tyarr.c +++ b/src/core/tyarr.c @@ -23,7 +23,7 @@ u8 elTypeWidthLogBits[] = { [el_B] = 6 }; u8 arrTypeWidthLog[] = { - [t_bitarr]=99, + [t_bitarr]=0, // 0 for mut.c hack to allow restoring a bitarr offset from array pointer & arrTypeWidthLog-"corrected" element pointer [t_i8arr ]=0, [t_i8slice ]=0, [t_c8arr ]=0, [t_c8slice ]=0, [t_i16arr]=1, [t_i16slice]=1, [t_c16arr]=1, [t_c16slice]=1, [t_i32arr]=2, [t_i32slice]=2, [t_c32arr]=2, [t_c32slice]=2, diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 31d5fe0b..cfc02616 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -81,6 +81,18 @@ def truncBits{n, v & n<=8} = cast_i{u8, v} def truncBits{n, v & n==16} = cast_i{u16, v} def truncBits{n, v & n==32} = cast_i{u32, v} def truncBits{n, v & n==64} = cast_i{u64, v} +def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} + +# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it +def loaduBitRaw{x:*u64, i} = { + loadu{*u64~~((*u8~~x) + (i>>3))} >> (i&7) +} +def loaduBit{x:*u64, i, n} = { + assert{(n<58) | (((n==58) | (n==60)) & (i%n == 0))} + loaduBitRaw{x, i} +} +def loaduBitTrunc{x:*u64, i, n & knum{n}} = truncBits{n, loaduBit{x, i, n}} + def loadBatchBit{T, x:*u64, is & ktup{is}} = { # def len = tuplen{is} diff --git a/src/singeli/src/cbqnDefs.singeli b/src/singeli/src/cbqnDefs.singeli index 6c00cb9a..4a89a523 100644 --- a/src/singeli/src/cbqnDefs.singeli +++ b/src/singeli/src/cbqnDefs.singeli @@ -30,3 +30,4 @@ def cbqn_elType{T & T==u8 } = 5 def cbqn_elType{T & T==u16} = 6 def cbqn_elType{T & T==u32} = 7 +def cbqn_tyArrOffset{} = emit{u64, 'offsetof', 'TyArr', 'a'} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 5f995bae..cdccd4c3 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -1,4 +1,5 @@ include './base' +include './debug' if (hasarch{'X86_64'}) { include './sse3' include './avx' @@ -10,14 +11,14 @@ include './mask' include './cbqnDefs' include './bitops' -def copyFromBits{T, xp: *u64, rp: *eltype{T}, l:u64} = { +def copyFromBits{T, loadFn, rp, l:u64} = { def bulk = vcount{T} def TU = ty_u{T} maskedLoop{bulk, l, {i, M} => { - x:= loadBatchBit{TU, xp, i} + x:= loadFn{TU, i} r:= x & TU ~~ T**1 - storeBatch{rp, i, T~~r, M} + storeBatch{*eltype{T}~~rp, i, T~~r, M} }} } @@ -50,7 +51,7 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { }} } else if (X==u1) { # show{'X==u1', X, R} - copyFromBits{[bulk]R, *u64~~x, *R~~r, l} + copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l} } else if (R==u1) { # show{'R==u1', X, R} def XU = ty_u{XV} @@ -74,6 +75,22 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { } } +copy_ubit{R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { + assert{l!=0} + x0:= (*u8~~xRaw) + cbqn_tyArrOffset{} + xs:= u64~~((*u8~~x) - x0) + # if ((xs&7)==0) { + # copy{u1, R}(*void~~(x0 + (xs>>3)), r, l, xRaw) + # } else { + def vw = arch_defvw + def bulk = vw/width{R} + def RV = [bulk]R + rp:= *R~~r + + copyFromBits{RV, {T, i} => spreadBits{T, loaduBitTrunc{*u64~~x0, xs+i*vcount{T}, vcount{T}}}, r, l} + # } +} + # simd_copy_src_dst # x→int & equal-width copies 'simd_copy_1_1' = copy{u1, u1} @@ -125,3 +142,9 @@ copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { 'simd_copy_B_c8' = copy{u64, u8} 'simd_copy_B_c16' = copy{u64, u16} 'simd_copy_B_c32' = copy{u64, u32} + +# unaligned bitarr widening +'simd_copy_1u_i8' = copy_ubit{i8} +'simd_copy_1u_i16' = copy_ubit{i16} +'simd_copy_1u_i32' = copy_ubit{i32} +'simd_copy_1u_f64' = copy_ubit{f64} \ No newline at end of file diff --git a/src/utils/mut.c b/src/utils/mut.c index 19b00c21..b0e3bfaa 100644 --- a/src/utils/mut.c +++ b/src/utils/mut.c @@ -220,6 +220,7 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x } #if SINGELI + static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #define SINGELI_FILE copy #include "./includeSingeli.h" typedef void (*copy_fn)(void*, void*, u64, void*); @@ -237,20 +238,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x } \ static void cpy##T##Arr_B(void* xp, void* rp, u64 ia, void* xRaw) { \ Arr* xa = (Arr*)xRaw; B* bxp = arrV_bptr(xa); \ - if (bxp!=NULL && sizeof(B)==sizeof(f64)) { \ - H2T; \ - } else cpy##T##Arr_BF(xp, rp, ia, xa); \ + if (bxp!=NULL && sizeof(B)==sizeof(f64)) H2T; \ + else cpy##T##Arr_BF(xp, rp, ia, xa); \ } \ static copy_fn copy##T##Fns[] = __VA_ARGS__; \ - T##Arr* cpy##T##Arr(B x) { \ - usz ia = IA(x); \ - MAKE; arr_shCopy(r, x); \ - if (ia>0) { \ + T##Arr* cpy##T##Arr(B x) { \ + usz ia = IA(x); \ + MAKE; arr_shCopy(r, x); \ + if (ia>0) { \ copy##T##Fns[TI(x,elType)](tyany_ptr(x), XRP, ia, a(x)); \ - } \ - if (TY) ptr_decT(a(x)); \ - else decG(x); \ - return (T##Arr*)r; \ + } \ + if (TY) ptr_decT(a(x)); \ + else decG(x); \ + return (T##Arr*)r; \ } #define BIT_PUT(V) bitp_set((u64*)rp, i, o2bG(V)) #define H2T_COPY(T) copy##T##Fns[el_MAX](bxp, rp, ia, xRaw) @@ -263,24 +263,19 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x Bit, {COPY_FN(1,1),COPY_FN(i8,1),COPY_FN(i16,1),COPY_FN(i32,1),COPY_FN(f64,1),badCopy, badCopy, badCopy, cpyBitArr_B, COPY_FN(f64,1)}) - static copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8}; - static copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16}; - static copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32}; - static copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64}; - static copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8}; - static copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16}; - static copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32}; + copy_fn tcopy_i8Fns [] = {[t_bitarr]=simd_copy_1u_i8, [t_i8arr]=simd_copy_i8_i8 ,[t_i8slice]=simd_copy_i8_i8}; + copy_fn tcopy_i16Fns[] = {[t_bitarr]=simd_copy_1u_i16, [t_i8arr]=simd_copy_i8_i16,[t_i8slice]=simd_copy_i8_i16, [t_i16arr]=simd_copy_i16_i16,[t_i16slice]=simd_copy_i16_i16}; + copy_fn tcopy_i32Fns[] = {[t_bitarr]=simd_copy_1u_i32, [t_i8arr]=simd_copy_i8_i32,[t_i8slice]=simd_copy_i8_i32, [t_i16arr]=simd_copy_i16_i32,[t_i16slice]=simd_copy_i16_i32, [t_i32arr]=simd_copy_i32_i32,[t_i32slice]=simd_copy_i32_i32}; + copy_fn tcopy_f64Fns[] = {[t_bitarr]=simd_copy_1u_f64, [t_i8arr]=simd_copy_i8_f64,[t_i8slice]=simd_copy_i8_f64, [t_i16arr]=simd_copy_i16_f64,[t_i16slice]=simd_copy_i16_f64, [t_i32arr]=simd_copy_i32_f64,[t_i32slice]=simd_copy_i32_f64, [t_f64arr]=simd_copy_f64_f64,[t_f64slice]=simd_copy_f64_f64}; + copy_fn tcopy_c8Fns [] = {[t_c8arr]=simd_copy_c8_c8 ,[t_c8slice]=simd_copy_c8_c8}; + copy_fn tcopy_c16Fns[] = {[t_c8arr]=simd_copy_c8_c16,[t_c8slice]=simd_copy_c8_c16, [t_c16arr]=simd_copy_c16_c16,[t_c16slice]=simd_copy_c16_c16}; + copy_fn tcopy_c32Fns[] = {[t_c8arr]=simd_copy_c8_c32,[t_c8slice]=simd_copy_c8_c32, [t_c16arr]=simd_copy_c16_c32,[t_c16slice]=simd_copy_c16_c32, [t_c32arr]=simd_copy_c32_c32,[t_c32slice]=simd_copy_c32_c32}; #define TCOPY_FN(T, N, NUM) static void m_copyG_##N(void* a, usz ms, B x, usz xs, usz l) { \ if (l==0) return; \ void* xp = tyany_ptr(x); \ - T* rp = ms + (T*)a; \ u8 xt = TY(x); \ - if (NUM && xt==t_bitarr) { \ - for (usz i = 0; i < l; i++) rp[i] = bitp_get((u64*)xp, xs+i); \ - } else { \ - tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, rp, l, a(x)); \ - } \ + tcopy_##N##Fns[xt]((xs << arrTypeWidthLog(xt)) + (u8*)xp, ms + (T*)a, l, a(x)); \ } TCOPY_FN(i8,i8, 1) TCOPY_FN(i16,i16, 1)