diff --git a/.gitignore b/.gitignore index 8ae4f7f3..a0334f5f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ perf.* /asm_* /src/singeli/gen /Singeli +/local/ CBQNHeapDump diff --git a/docs/system.md b/docs/system.md index 78ceaf4b..71c600fb 100644 --- a/docs/system.md +++ b/docs/system.md @@ -14,7 +14,7 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo | `•name` | | | `•wdpath` | | | `•Exit` | | -| `•file` | Fields: `path`, `At`, `List`, `Bytes`, `Chars`, `Lines`, `Type`, `Name` | +| `•file` | Fields: `path`, `At`, `List`, `Bytes`, `Chars`, `Lines`, `Type`, `Name`; has extensions | | `•FChars` | | | `•FBytes` | | | `•FLines` | | @@ -22,7 +22,7 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo | `•Show` | | | `•Repr` | | | `•Fmt` | | -| `•term` | Fields: `Flush`, `RawMode`, `CharB`, `CharN` | +| `•term` | Fields: `Flush`, `RawMode`, `CharB`, `CharN`; has extensions | | `•SH` | Left argument can be `{stdin⇐"input"}` | | `•Type` | | | `•Glyph` | | @@ -36,11 +36,15 @@ See [the BQN specification](https://mlochbaum.github.io/BQN/spec/system.html) fo | `•rand` | seeds with system time (can be hard-coded by setting the C macro `RANDSEED`), same algorithm as `•MakeRand` | | `•bit` | Fields: `_cast`; casting an sNaN bit pattern to a float is undefined behavior | -# CBQN-specific system functions +# CBQN-specific system functions and extensions + +## `•file.MapBytes` + +`mmap`s file at path `𝕩` as an 8-bit signed integer array; Use `•bit._cast` to interpret as other types, and `↓`/`↑` to select only a part of the file. ## `•term` -Besides the previously mentioned supported things, `•term.OutRaw` and `•term.ErrRaw` output the given bytes directly to the specific stream, without any trailing newline. May be removed once a proper interface for stream I/O has been made. +`•term.OutRaw` and `•term.ErrRaw` output the given bytes directly to the specific stream, without any trailing newline. May be removed once a proper interface for stream I/O has been made. ## `•_while_` diff --git a/src/builtins/md2.c b/src/builtins/md2.c index 00555f3c..0d4211b3 100644 --- a/src/builtins/md2.c +++ b/src/builtins/md2.c @@ -10,23 +10,6 @@ B md2BI_ucw(Md2* t, B o, B f, B g, B w, B x) { return ((BMd2*)t)->ucw(t, o, f, g B val_c1(Md2D* d, B x) { return c1(d->f, x); } B val_c2(Md2D* d, B w, B x) { return c2(d->g, w,x); } - -typedef struct CustomObj { - struct Value; - V2v visit; - V2v freeO; -} CustomObj; -void* customObj(u64 size, V2v visit, V2v freeO) { - CustomObj* r = mm_alloc(size, t_customObj); - r->visit = visit; - r->freeO = freeO; - return r; -} -void customObj_visit(Value* v) { ((CustomObj*)v)->visit(v); } -void customObj_freeO(Value* v) { ((CustomObj*)v)->freeO(v); } -void customObj_freeF(Value* v) { ((CustomObj*)v)->freeO(v); mm_free(v); } - - #if CATCH_ERRORS extern B lastErrMsg; // sysfn.c @@ -308,8 +291,5 @@ void md2_init() { TIi(t_md2BI,m2_im) = md2BI_im; TIi(t_md2BI,m2_iw) = md2BI_iw; TIi(t_md2BI,m2_ix) = md2BI_ix; - TIi(t_customObj,freeO) = customObj_freeO; - TIi(t_customObj,freeF) = customObj_freeF; - TIi(t_customObj,visit) = customObj_visit; c(BMd2,bi_before)->uc1 = before_uc1; } diff --git a/src/builtins/sysfn.c b/src/builtins/sysfn.c index 0d2076c5..c2228071 100644 --- a/src/builtins/sysfn.c +++ b/src/builtins/sysfn.c @@ -672,6 +672,7 @@ static void sys_gcFn() { static NFnDesc* fTypeDesc; static NFnDesc* fExistsDesc; static NFnDesc* fListDesc; +static NFnDesc* fMapBytesDesc; B list_c1(B d, B x) { return path_list(path_rel(nfn_objU(d), x)); } @@ -691,6 +692,10 @@ B fName_c1(B t, B x) { return path_name(x); } +B mapBytes_c1(B d, B x) { + return mmap_file(path_rel(nfn_objU(d), x)); +} + B unixTime_c1(B t, B x) { dec(x); return m_f64(time(NULL)); @@ -874,7 +879,7 @@ B sh_c2(B t, B w, B x) { return m_hVec3(m_i32(WEXITSTATUS(status)), s_out, s_err); } #else -B sh_c2(B t, B w, B x) { thrM("•SH: CBQN was built without "); } +B sh_c2(B t, B w, B x) { thrM("•SH: CBQN was compiled without "); } #endif B sh_c1(B t, B x) { return sh_c2(t, bi_N, x); } @@ -1078,7 +1083,7 @@ B sys_c1(B t, B x) { if(!fileNS.u) { REQ_PATH; #define F(X) m_nfn(X##Desc, inc(path)) - fileNS = m_nns(file_nsGen, q_N(path)? m_c32(0) : inc(path), F(fileAt), F(fList), F(fBytes), F(fChars), F(fLines), F(fType), F(fExists), inc(bi_fName)); + fileNS = m_nns(file_nsGen, q_N(path)? m_c32(0) : inc(path), F(fileAt), F(fList), F(fBytes), F(fChars), F(fLines), F(fType), F(fExists), inc(bi_fName), F(fMapBytes)); #undef F } cr = inc(fileNS); @@ -1147,11 +1152,12 @@ void sysfn_init() { fBytesDesc = registerNFn(m_str8l("(file).Bytes"), fbytes_c1, fbytes_c2); fListDesc = registerNFn(m_str8l("(file).List"), list_c1, c2_bad); fTypeDesc = registerNFn(m_str8l("(file).Type"), ftype_c1, c2_bad); + fMapBytesDesc = registerNFn(m_str8l("(file).MapBytes"), mapBytes_c1, c2_bad); fExistsDesc = registerNFn(m_str8l("(file).Exists"), fexists_c1, c2_bad); importDesc = registerNFn(m_str32(U"•Import"), import_c1, import_c2); reBQNDesc = registerNFn(m_str8l("(REPL)"), repl_c1, repl_c2); } void sysfnPost_init() { - file_nsGen = m_nnsDesc("path","at","list","bytes","chars","lines","type","exists","name"); + file_nsGen = m_nnsDesc("path","at","list","bytes","chars","lines","type","exists","name","mmap"); c(BMd1,bi_bitcast)->im = bitcast_im; } diff --git a/src/core/stuff.c b/src/core/stuff.c index cc20f474..bce86fe0 100644 --- a/src/core/stuff.c +++ b/src/core/stuff.c @@ -861,7 +861,7 @@ void g_pst(void) { vm_pstLive(); } printf("bad array tag/type: type=%d, obj=%p\n", v(x)->type, (void*)x.u); PRINT_ID(v(x)); print(x); - err("\nk"); + err("\n"); } return x; } diff --git a/src/core/stuff.h b/src/core/stuff.h index 63cc431a..a55cfa27 100644 --- a/src/core/stuff.h +++ b/src/core/stuff.h @@ -10,6 +10,13 @@ void vm_pstLive(void); #define MAP_NORESERVE 0 // apparently needed for freebsd or something #endif +typedef struct CustomObj { + struct Value; + V2v visit; + V2v freeO; +} CustomObj; +void* customObj(u64 size, V2v visit, V2v freeO); + // shape mess typedef struct ShArr { diff --git a/src/core/tyarrTemplate.c b/src/core/tyarrTemplate.c index ea256560..81f99521 100644 --- a/src/core/tyarrTemplate.c +++ b/src/core/tyarrTemplate.c @@ -19,7 +19,7 @@ static void TP(,arr_init)() { TIi(T_ARR,get) = TP(,arr_get); TIi(T_SLICE,get) = TP(,slice_get); TIi(T_ARR,getU) = TP(,arr_get); TIi(T_SLICE,getU) = TP(,slice_get); TIi(T_ARR,slice) = TP(,arr_slice); TIi(T_SLICE,slice) = TP(,slice_slice); - TIi(T_ARR,freeO) = tyarr_freeO; TIi(T_SLICE,freeO) = slice_freeO; + TIi(T_ARR,freeO) = tyarr_freeO; TIi(T_SLICE,freeO) = slice_freeO; // typed array slice frees calling the generic function is relied on by mmap & •bit._cast TIi(T_ARR,freeF) = tyarr_freeF; TIi(T_SLICE,freeF) = slice_freeF; TIi(T_ARR,visit) = noop_visit; TIi(T_SLICE,visit) = slice_visit; TIi(T_ARR,print) = farr_print; TIi(T_SLICE,print) = farr_print; diff --git a/src/h.h b/src/h.h index 38796379..425a6827 100644 --- a/src/h.h +++ b/src/h.h @@ -214,7 +214,7 @@ typedef union B { \ /*32*/ F(comp) F(block) F(body) F(scope) F(scopeExt) F(blBlocks) \ /*38*/ F(ns) F(nsDesc) F(fldAlias) F(vfyObj) F(hashmap) F(temp) F(nfn) F(nfnDesc) \ - /*46*/ F(freed) F(harrPartial) F(customObj) \ + /*46*/ F(freed) F(harrPartial) F(customObj) F(mmapH) \ \ /*49*/ IF_WRAP(F(funWrap) F(md1Wrap) F(md2Wrap)) diff --git a/src/load.c b/src/load.c index 5c7149b3..736c3eef 100644 --- a/src/load.c +++ b/src/load.c @@ -5,7 +5,7 @@ #include "ns.h" #include "builtins.h" -#define FOR_INIT(F) F(base) F(harr) F(mutF) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(load) F(sysfnPost) F(dervPost) +#define FOR_INIT(F) F(base) F(harr) F(mutF) F(fillarr) F(tyarr) F(hash) F(sfns) F(fns) F(arith) F(md1) F(md2) F(derv) F(comp) F(rtWrap) F(ns) F(nfn) F(sysfn) F(inverse) F(load) F(sysfnPost) F(dervPost) F(mmap) #define F(X) void X##_init(void); FOR_INIT(F) #undef F @@ -558,6 +558,16 @@ static B funBI_imInit(B t, B x) { } +void* customObj(u64 size, V2v visit, V2v freeO) { + CustomObj* r = mm_alloc(size, t_customObj); + r->visit = visit; + r->freeO = freeO; + return r; +} +void customObj_visit(Value* v) { ((CustomObj*)v)->visit(v); } +void customObj_freeO(Value* v) { ((CustomObj*)v)->freeO(v); } +void customObj_freeF(Value* v) { ((CustomObj*)v)->freeO(v); mm_free(v); } + static NOINLINE B m_bfn(BB2B c1, BBB2B c2, u8 id) { BFn* f = mm_alloc(sizeof(BFn), t_funBI); @@ -639,6 +649,9 @@ void base_init() { // very first init function TIi(t_funBI,freeF) = TIi(t_md1BI,freeF) = TIi(t_md2BI,freeF) = builtin_free; TIi(t_funBI,visit) = funBI_visit; TIi(t_hashmap,visit) = noop_visit; + TIi(t_customObj,freeO) = customObj_freeO; + TIi(t_customObj,freeF) = customObj_freeF; + TIi(t_customObj,visit) = customObj_visit; assert((MD1_TAG>>1) == (MD2_TAG>>1)); // just to be sure it isn't changed incorrectly, `isMd` depends on this diff --git a/src/nfns.c b/src/nfns.c index 43faa9e5..de2d23db 100644 --- a/src/nfns.c +++ b/src/nfns.c @@ -3,13 +3,6 @@ #include "nfns.h" #include "utils/mut.h" -struct NFnDesc { - struct Value; - u32 id; - B name; - BB2B c1; - BBB2B c2; -}; static u32 nfn_curr; static B nfn_list; diff --git a/src/nfns.h b/src/nfns.h index 7ff59e56..d1ee6597 100644 --- a/src/nfns.h +++ b/src/nfns.h @@ -9,10 +9,20 @@ typedef struct NFn { // native function i32 data; B obj; } NFn; +struct NFnDesc { + struct Value; + u32 id; + B name; + BB2B c1; + BBB2B c2; +}; NFnDesc* registerNFn(B name, BB2B c1, BBB2B c2); // should be called a constant number of times; consumes name B m_nfn(NFnDesc* desc, B obj); // consumes obj B nfn_name(B x); // doesn't consume +static void nfn_lateInit(NFn* fn, NFnDesc* desc) { + fn->id = desc->id; +} static B nfn_objU(B t) { assert(isVal(t) && v(t)->type == t_nfn); return c(NFn,t)->obj; diff --git a/src/utils/cstr.h b/src/utils/cstr.h new file mode 100644 index 00000000..e219e56f --- /dev/null +++ b/src/utils/cstr.h @@ -0,0 +1,14 @@ +#pragma once +#include "talloc.h" +#include "utf.h" + +static char* toCStr(B x) { // doesn't consume + u64 len = utf8lenB(x); + TALLOC(char, p, len+1); + toUTF8(x, p); + p[len] = 0; + return p; +} +static void freeCStr(char* p) { + TFREE(p); +} diff --git a/src/utils/file.c b/src/utils/file.c index f256119e..dc0c916b 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -3,17 +3,8 @@ #include "file.h" #include "mut.h" #include "talloc.h" +#include "cstr.h" -char* toCStr(B path) { - u64 plen = utf8lenB(path); - TALLOC(char, p, plen+1); - toUTF8(path, p); - p[plen] = 0; - return p; -} -void freeCStr(char* p) { - TFREE(p); -} FILE* file_open(B path, char* desc, char* mode) { // doesn't consume char* p = toCStr(path); @@ -222,6 +213,83 @@ B path_list(B path) { return res; } +#if __has_include() && __has_include() && __has_include() && __has_include() && !WASM + +#include +#include +#include +#include + +typedef struct MmapHolder { + struct Arr; + int fd; + u64 size; + u8* a; +} MmapHolder; + +void mmapH_visit(Value* v) { } +DEF_FREE(mmapH) { + MmapHolder* p = (MmapHolder*)x; + if (munmap(p->a, p->size)) thrF("Failed to unmap: %S", strerror(errno)); + if (close(p->fd)) thrF("Failed to close file: %S", strerror(errno)); +} + +B info_c1(B,B); +static Arr* mmapH_slice(B x, usz s, usz ia) { + TySlice* r = m_arr(sizeof(TySlice), t_i8slice, ia); + r->a = c(MmapHolder,x)->a + s; + r->p = a(x); + return (Arr*)r; +} + +B mmap_file(B path) { + char* p = toCStr(path); + dec(path); + int fd = open(p, 0); + freeCStr(p); + u64 len = lseek(fd, 0, SEEK_END); + if (fd==-1) thrF("Failed to open file: %S", strerror(errno)); + + u8* data = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); // TODO count in heap usage + if (data==MAP_FAILED) { + close(fd); + thrM("failed to mmap file"); + } + + MmapHolder* holder = m_arr(sizeof(MmapHolder), t_mmapH, len); + holder->fd = fd; + holder->a = data; + holder->size = len; + arr_shVec((Arr*)holder); + + Arr* r = mmapH_slice(taga(holder), 0, len); + arr_shVec(r); + return taga(r); +} + +B mmapH_get(Arr* a, usz pos) { thrM("Reading mmapH directly"); } + +void mmap_init() { + TIi(t_mmapH,get) = mmapH_get; + TIi(t_mmapH,getU) = mmapH_get; + TIi(t_mmapH,slice) = mmapH_slice; + TIi(t_mmapH,freeO) = mmapH_freeO; + TIi(t_mmapH,freeF) = mmapH_freeF; + TIi(t_mmapH,visit) = noop_visit; + TIi(t_mmapH,print) = farr_print; + TIi(t_mmapH,isArr) = true; + TIi(t_mmapH,arrD1) = true; + TIi(t_mmapH,elType) = el_i8; + // use default canStore +} +#else +B mmap_file(B path) { + thrM("CBQN was compiled without mmap"); +} +void mmap_init() { } +#endif + + #include #include diff --git a/src/utils/file.h b/src/utils/file.h index 2489e859..95cb963e 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -13,6 +13,8 @@ B path_lines(B path); // consumes I8Arr* stream_bytes(FILE* f); +B mmap_file(B path); // consumes + void path_wChars(B path, B x); // consumes path void path_wBytes(B path, B x); // consumes path void file_wBytes(FILE* file, B name, B x); // consumes x