•file.MapBytes

This commit is contained in:
dzaima 2022-04-09 19:45:02 +03:00
parent 0ec0658ada
commit acc8e35c2f
14 changed files with 146 additions and 48 deletions

1
.gitignore vendored
View File

@ -8,4 +8,5 @@ perf.*
/asm_* /asm_*
/src/singeli/gen /src/singeli/gen
/Singeli /Singeli
/local/
CBQNHeapDump CBQNHeapDump

View File

@ -14,7 +14,7 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo
| `•name` | | | `•name` | |
| `•wdpath` | | | `•wdpath` | |
| `•Exit` | | | `•Exit` | |
| `•file` | Fields: `path`, `At`, `List`, `Bytes`, `Chars`, `Lines`, `Type`, `Name` | | `•file` | Fields: `path`, `At`, `List`, `Bytes`, `Chars`, `Lines`, `Type`, `Name`; has extensions |
| `•FChars` | | | `•FChars` | |
| `•FBytes` | | | `•FBytes` | |
| `•FLines` | | | `•FLines` | |
@ -22,7 +22,7 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo
| `•Show` | | | `•Show` | |
| `•Repr` | | | `•Repr` | |
| `•Fmt` | | | `•Fmt` | |
| `•term` | Fields: `Flush`, `RawMode`, `CharB`, `CharN` | | `•term` | Fields: `Flush`, `RawMode`, `CharB`, `CharN`; has extensions |
| `•SH` | Left argument can be `{stdin⇐"input"}` | | `•SH` | Left argument can be `{stdin⇐"input"}` |
| `•Type` | | | `•Type` | |
| `•Glyph` | | | `•Glyph` | |
@ -36,11 +36,15 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo
| `•rand` | seeds with system time (can be hard-coded by setting the C macro `RANDSEED`), same algorithm as `•MakeRand` | | `•rand` | seeds with system time (can be hard-coded by setting the C macro `RANDSEED`), same algorithm as `•MakeRand` |
| `•bit` | Fields: `_cast`; casting an sNaN bit pattern to a float is undefined behavior | | `•bit` | Fields: `_cast`; casting an sNaN bit pattern to a float is undefined behavior |
# CBQN-specific system functions # CBQN-specific system functions and extensions
## `•file.MapBytes`
`mmap`s file at path `𝕩` as an 8-bit signed integer array; Use `•bit._cast` to interpret as other types, and `↓`/`↑` to select only a part of the file.
## `•term` ## `•term`
Besides the previously mentioned supported things, `•term.OutRaw` and `•term.ErrRaw` output the given bytes directly to the specific stream, without any trailing newline. May be removed once a proper interface for stream I/O has been made. `•term.OutRaw` and `•term.ErrRaw` output the given bytes directly to the specific stream, without any trailing newline. May be removed once a proper interface for stream I/O has been made.
## `•_while_` ## `•_while_`

View File

@ -10,23 +10,6 @@ B md2BI_ucw(Md2* t, B o, B f, B g, B w, B x) { return ((BMd2*)t)->ucw(t, o, f, g
B val_c1(Md2D* d, B x) { return c1(d->f, x); } B val_c1(Md2D* d, B x) { return c1(d->f, x); }
B val_c2(Md2D* d, B w, B x) { return c2(d->g, w,x); } B val_c2(Md2D* d, B w, B x) { return c2(d->g, w,x); }
typedef struct CustomObj {
struct Value;
V2v visit;
V2v freeO;
} CustomObj;
void* customObj(u64 size, V2v visit, V2v freeO) {
CustomObj* r = mm_alloc(size, t_customObj);
r->visit = visit;
r->freeO = freeO;
return r;
}
void customObj_visit(Value* v) { ((CustomObj*)v)->visit(v); }
void customObj_freeO(Value* v) { ((CustomObj*)v)->freeO(v); }
void customObj_freeF(Value* v) { ((CustomObj*)v)->freeO(v); mm_free(v); }
#if CATCH_ERRORS #if CATCH_ERRORS
extern B lastErrMsg; // sysfn.c extern B lastErrMsg; // sysfn.c
@ -308,8 +291,5 @@ void md2_init() {
TIi(t_md2BI,m2_im) = md2BI_im; TIi(t_md2BI,m2_im) = md2BI_im;
TIi(t_md2BI,m2_iw) = md2BI_iw; TIi(t_md2BI,m2_iw) = md2BI_iw;
TIi(t_md2BI,m2_ix) = md2BI_ix; TIi(t_md2BI,m2_ix) = md2BI_ix;
TIi(t_customObj,freeO) = customObj_freeO;
TIi(t_customObj,freeF) = customObj_freeF;
TIi(t_customObj,visit) = customObj_visit;
c(BMd2,bi_before)->uc1 = before_uc1; c(BMd2,bi_before)->uc1 = before_uc1;
} }

View File

@ -672,6 +672,7 @@ static void sys_gcFn() {
static NFnDesc* fTypeDesc; static NFnDesc* fTypeDesc;
static NFnDesc* fExistsDesc; static NFnDesc* fExistsDesc;
static NFnDesc* fListDesc; static NFnDesc* fListDesc;
static NFnDesc* fMapBytesDesc;
B list_c1(B d, B x) { B list_c1(B d, B x) {
return path_list(path_rel(nfn_objU(d), x)); return path_list(path_rel(nfn_objU(d), x));
} }
@ -691,6 +692,10 @@ B fName_c1(B t, B x) {
return path_name(x); return path_name(x);
} }
B mapBytes_c1(B d, B x) {
return mmap_file(path_rel(nfn_objU(d), x));
}
B unixTime_c1(B t, B x) { B unixTime_c1(B t, B x) {
dec(x); dec(x);
return m_f64(time(NULL)); return m_f64(time(NULL));
@ -874,7 +879,7 @@ B sh_c2(B t, B w, B x) {
return m_hVec3(m_i32(WEXITSTATUS(status)), s_out, s_err); return m_hVec3(m_i32(WEXITSTATUS(status)), s_out, s_err);
} }
#else #else
B sh_c2(B t, B w, B x) { thrM("•SH: CBQN was built without <spawn.h>"); } B sh_c2(B t, B w, B x) { thrM("•SH: CBQN was compiled without <spawn.h>"); }
#endif #endif
B sh_c1(B t, B x) { return sh_c2(t, bi_N, x); } B sh_c1(B t, B x) { return sh_c2(t, bi_N, x); }
@ -1078,7 +1083,7 @@ B sys_c1(B t, B x) {
if(!fileNS.u) { if(!fileNS.u) {
REQ_PATH; REQ_PATH;
#define F(X) m_nfn(X##Desc, inc(path)) #define F(X) m_nfn(X##Desc, inc(path))
fileNS = m_nns(file_nsGen, q_N(path)? m_c32(0) : inc(path), F(fileAt), F(fList), F(fBytes), F(fChars), F(fLines), F(fType), F(fExists), inc(bi_fName)); fileNS = m_nns(file_nsGen, q_N(path)? m_c32(0) : inc(path), F(fileAt), F(fList), F(fBytes), F(fChars), F(fLines), F(fType), F(fExists), inc(bi_fName), F(fMapBytes));
#undef F #undef F
} }
cr = inc(fileNS); cr = inc(fileNS);
@ -1147,11 +1152,12 @@ void sysfn_init() {
fBytesDesc = registerNFn(m_str8l("(file).Bytes"), fbytes_c1, fbytes_c2); fBytesDesc = registerNFn(m_str8l("(file).Bytes"), fbytes_c1, fbytes_c2);
fListDesc = registerNFn(m_str8l("(file).List"), list_c1, c2_bad); fListDesc = registerNFn(m_str8l("(file).List"), list_c1, c2_bad);
fTypeDesc = registerNFn(m_str8l("(file).Type"), ftype_c1, c2_bad); fTypeDesc = registerNFn(m_str8l("(file).Type"), ftype_c1, c2_bad);
fMapBytesDesc = registerNFn(m_str8l("(file).MapBytes"), mapBytes_c1, c2_bad);
fExistsDesc = registerNFn(m_str8l("(file).Exists"), fexists_c1, c2_bad); fExistsDesc = registerNFn(m_str8l("(file).Exists"), fexists_c1, c2_bad);
importDesc = registerNFn(m_str32(U"•Import"), import_c1, import_c2); importDesc = registerNFn(m_str32(U"•Import"), import_c1, import_c2);
reBQNDesc = registerNFn(m_str8l("(REPL)"), repl_c1, repl_c2); reBQNDesc = registerNFn(m_str8l("(REPL)"), repl_c1, repl_c2);
} }
void sysfnPost_init() { void sysfnPost_init() {
file_nsGen = m_nnsDesc("path","at","list","bytes","chars","lines","type","exists","name"); file_nsGen = m_nnsDesc("path","at","list","bytes","chars","lines","type","exists","name","mmap");
c(BMd1,bi_bitcast)->im = bitcast_im; c(BMd1,bi_bitcast)->im = bitcast_im;
} }

View File

@ -861,7 +861,7 @@ void g_pst(void) { vm_pstLive(); }
printf("bad array tag/type: type=%d, obj=%p\n", v(x)->type, (void*)x.u); printf("bad array tag/type: type=%d, obj=%p\n", v(x)->type, (void*)x.u);
PRINT_ID(v(x)); PRINT_ID(v(x));
print(x); print(x);
err("\nk"); err("\n");
} }
return x; return x;
} }

View File

@ -10,6 +10,13 @@ void vm_pstLive(void);
#define MAP_NORESERVE 0 // apparently needed for freebsd or something #define MAP_NORESERVE 0 // apparently needed for freebsd or something
#endif #endif
typedef struct CustomObj {
struct Value;
V2v visit;
V2v freeO;
} CustomObj;
void* customObj(u64 size, V2v visit, V2v freeO);
// shape mess // shape mess
typedef struct ShArr { typedef struct ShArr {

View File

@ -19,7 +19,7 @@ static void TP(,arr_init)() {
TIi(T_ARR,get) = TP(,arr_get); TIi(T_SLICE,get) = TP(,slice_get); TIi(T_ARR,get) = TP(,arr_get); TIi(T_SLICE,get) = TP(,slice_get);
TIi(T_ARR,getU) = TP(,arr_get); TIi(T_SLICE,getU) = TP(,slice_get); TIi(T_ARR,getU) = TP(,arr_get); TIi(T_SLICE,getU) = TP(,slice_get);
TIi(T_ARR,slice) = TP(,arr_slice); TIi(T_SLICE,slice) = TP(,slice_slice); TIi(T_ARR,slice) = TP(,arr_slice); TIi(T_SLICE,slice) = TP(,slice_slice);
TIi(T_ARR,freeO) = tyarr_freeO; TIi(T_SLICE,freeO) = slice_freeO; TIi(T_ARR,freeO) = tyarr_freeO; TIi(T_SLICE,freeO) = slice_freeO; // typed array slice frees calling the generic function is relied on by mmap & •bit._cast
TIi(T_ARR,freeF) = tyarr_freeF; TIi(T_SLICE,freeF) = slice_freeF; TIi(T_ARR,freeF) = tyarr_freeF; TIi(T_SLICE,freeF) = slice_freeF;
TIi(T_ARR,visit) = noop_visit; TIi(T_SLICE,visit) = slice_visit; TIi(T_ARR,visit) = noop_visit; TIi(T_SLICE,visit) = slice_visit;
TIi(T_ARR,print) = farr_print; TIi(T_SLICE,print) = farr_print; TIi(T_ARR,print) = farr_print; TIi(T_SLICE,print) = farr_print;

View File

@ -214,7 +214,7 @@ typedef union B {
\ \
/*32*/ F(comp) F(block) F(body) F(scope) F(scopeExt) F(blBlocks) \ /*32*/ F(comp) F(block) F(body) F(scope) F(scopeExt) F(blBlocks) \
/*38*/ F(ns) F(nsDesc) F(fldAlias) F(vfyObj) F(hashmap) F(temp) F(nfn) F(nfnDesc) \ /*38*/ F(ns) F(nsDesc) F(fldAlias) F(vfyObj) F(hashmap) F(temp) F(nfn) F(nfnDesc) \
/*46*/ F(freed) F(harrPartial) F(customObj) \ /*46*/ F(freed) F(harrPartial) F(customObj) F(mmapH) \
\ \
/*49*/ IF_WRAP(F(funWrap) F(md1Wrap) F(md2Wrap)) /*49*/ IF_WRAP(F(funWrap) F(md1Wrap) F(md2Wrap))

View File

@ -5,7 +5,7 @@
#include "ns.h" #include "ns.h"
#include "builtins.h" #include "builtins.h"
#define FOR_INIT(F) F(base) F(harr) F(mutF) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(load) F(sysfnPost) F(dervPost) #define FOR_INIT(F) F(base) F(harr) F(mutF) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(load) F(sysfnPost) F(dervPost) F(mmap)
#define F(X) void X##_init(void); #define F(X) void X##_init(void);
FOR_INIT(F) FOR_INIT(F)
#undef F #undef F
@ -558,6 +558,16 @@ static B funBI_imInit(B t, B x) {
} }
void* customObj(u64 size, V2v visit, V2v freeO) {
CustomObj* r = mm_alloc(size, t_customObj);
r->visit = visit;
r->freeO = freeO;
return r;
}
void customObj_visit(Value* v) { ((CustomObj*)v)->visit(v); }
void customObj_freeO(Value* v) { ((CustomObj*)v)->freeO(v); }
void customObj_freeF(Value* v) { ((CustomObj*)v)->freeO(v); mm_free(v); }
static NOINLINE B m_bfn(BB2B c1, BBB2B c2, u8 id) { static NOINLINE B m_bfn(BB2B c1, BBB2B c2, u8 id) {
BFn* f = mm_alloc(sizeof(BFn), t_funBI); BFn* f = mm_alloc(sizeof(BFn), t_funBI);
@ -639,6 +649,9 @@ void base_init() { // very first init function
TIi(t_funBI,freeF) = TIi(t_md1BI,freeF) = TIi(t_md2BI,freeF) = builtin_free; TIi(t_funBI,freeF) = TIi(t_md1BI,freeF) = TIi(t_md2BI,freeF) = builtin_free;
TIi(t_funBI,visit) = funBI_visit; TIi(t_funBI,visit) = funBI_visit;
TIi(t_hashmap,visit) = noop_visit; TIi(t_hashmap,visit) = noop_visit;
TIi(t_customObj,freeO) = customObj_freeO;
TIi(t_customObj,freeF) = customObj_freeF;
TIi(t_customObj,visit) = customObj_visit;
assert((MD1_TAG>>1) == (MD2_TAG>>1)); // just to be sure it isn't changed incorrectly, `isMd` depends on this assert((MD1_TAG>>1) == (MD2_TAG>>1)); // just to be sure it isn't changed incorrectly, `isMd` depends on this

View File

@ -3,13 +3,6 @@
#include "nfns.h" #include "nfns.h"
#include "utils/mut.h" #include "utils/mut.h"
struct NFnDesc {
struct Value;
u32 id;
B name;
BB2B c1;
BBB2B c2;
};
static u32 nfn_curr; static u32 nfn_curr;
static B nfn_list; static B nfn_list;

View File

@ -9,10 +9,20 @@ typedef struct NFn { // native function
i32 data; i32 data;
B obj; B obj;
} NFn; } NFn;
struct NFnDesc {
struct Value;
u32 id;
B name;
BB2B c1;
BBB2B c2;
};
NFnDesc* registerNFn(B name, BB2B c1, BBB2B c2); // should be called a constant number of times; consumes name NFnDesc* registerNFn(B name, BB2B c1, BBB2B c2); // should be called a constant number of times; consumes name
B m_nfn(NFnDesc* desc, B obj); // consumes obj B m_nfn(NFnDesc* desc, B obj); // consumes obj
B nfn_name(B x); // doesn't consume B nfn_name(B x); // doesn't consume
static void nfn_lateInit(NFn* fn, NFnDesc* desc) {
fn->id = desc->id;
}
static B nfn_objU(B t) { static B nfn_objU(B t) {
assert(isVal(t) && v(t)->type == t_nfn); assert(isVal(t) && v(t)->type == t_nfn);
return c(NFn,t)->obj; return c(NFn,t)->obj;

14
src/utils/cstr.h Normal file
View File

@ -0,0 +1,14 @@
#pragma once
#include "talloc.h"
#include "utf.h"
static char* toCStr(B x) { // doesn't consume
u64 len = utf8lenB(x);
TALLOC(char, p, len+1);
toUTF8(x, p);
p[len] = 0;
return p;
}
static void freeCStr(char* p) {
TFREE(p);
}

View File

@ -3,17 +3,8 @@
#include "file.h" #include "file.h"
#include "mut.h" #include "mut.h"
#include "talloc.h" #include "talloc.h"
#include "cstr.h"
char* toCStr(B path) {
u64 plen = utf8lenB(path);
TALLOC(char, p, plen+1);
toUTF8(path, p);
p[plen] = 0;
return p;
}
void freeCStr(char* p) {
TFREE(p);
}
FILE* file_open(B path, char* desc, char* mode) { // doesn't consume FILE* file_open(B path, char* desc, char* mode) { // doesn't consume
char* p = toCStr(path); char* p = toCStr(path);
@ -222,6 +213,83 @@ B path_list(B path) {
return res; return res;
} }
#if __has_include(<sys/mman.h>) && __has_include(<fcntl.h>) && __has_include(<errno.h>) && __has_include(<unistd.h>) && !WASM
#include <sys/mman.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
typedef struct MmapHolder {
struct Arr;
int fd;
u64 size;
u8* a;
} MmapHolder;
void mmapH_visit(Value* v) { }
DEF_FREE(mmapH) {
MmapHolder* p = (MmapHolder*)x;
if (munmap(p->a, p->size)) thrF("Failed to unmap: %S", strerror(errno));
if (close(p->fd)) thrF("Failed to close file: %S", strerror(errno));
}
B info_c1(B,B);
static Arr* mmapH_slice(B x, usz s, usz ia) {
TySlice* r = m_arr(sizeof(TySlice), t_i8slice, ia);
r->a = c(MmapHolder,x)->a + s;
r->p = a(x);
return (Arr*)r;
}
B mmap_file(B path) {
char* p = toCStr(path);
dec(path);
int fd = open(p, 0);
freeCStr(p);
u64 len = lseek(fd, 0, SEEK_END);
if (fd==-1) thrF("Failed to open file: %S", strerror(errno));
u8* data = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); // TODO count in heap usage
if (data==MAP_FAILED) {
close(fd);
thrM("failed to mmap file");
}
MmapHolder* holder = m_arr(sizeof(MmapHolder), t_mmapH, len);
holder->fd = fd;
holder->a = data;
holder->size = len;
arr_shVec((Arr*)holder);
Arr* r = mmapH_slice(taga(holder), 0, len);
arr_shVec(r);
return taga(r);
}
B mmapH_get(Arr* a, usz pos) { thrM("Reading mmapH directly"); }
void mmap_init() {
TIi(t_mmapH,get) = mmapH_get;
TIi(t_mmapH,getU) = mmapH_get;
TIi(t_mmapH,slice) = mmapH_slice;
TIi(t_mmapH,freeO) = mmapH_freeO;
TIi(t_mmapH,freeF) = mmapH_freeF;
TIi(t_mmapH,visit) = noop_visit;
TIi(t_mmapH,print) = farr_print;
TIi(t_mmapH,isArr) = true;
TIi(t_mmapH,arrD1) = true;
TIi(t_mmapH,elType) = el_i8;
// use default canStore
}
#else
B mmap_file(B path) {
thrM("CBQN was compiled without mmap");
}
void mmap_init() { }
#endif
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>

View File

@ -13,6 +13,8 @@ B path_lines(B path); // consumes
I8Arr* stream_bytes(FILE* f); I8Arr* stream_bytes(FILE* f);
B mmap_file(B path); // consumes
void path_wChars(B path, B x); // consumes path void path_wChars(B path, B x); // consumes path
void path_wBytes(B path, B x); // consumes path void path_wBytes(B path, B x); // consumes path
void file_wBytes(FILE* file, B name, B x); // consumes x void file_wBytes(FILE* file, B name, B x); // consumes x