From cd5eb4dfe9557327a91bcf29de83d43e0da12f8b Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 12 May 2021 16:26:34 +0300 Subject: [PATCH] =?UTF-8?q?=E2=80=A2FChars,=20=E2=80=A2FBytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/h.h | 15 ++++++++------- src/main.c | 8 ++++++-- src/sysfn.c | 43 ++++++++++++++++++++++++++++++++++++++++++- src/utf.c | 43 +++++++++++++++++++++++++++++++++---------- src/vm.c | 4 ++-- 5 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/h.h b/src/h.h index f754c7ac..f3c26633 100644 --- a/src/h.h +++ b/src/h.h @@ -67,14 +67,14 @@ enum Type { /* 8*/ t_fork, t_atop, /*10*/ t_md1D, t_md2D, t_md2H, - /*13*/ t_harr , t_i32arr , t_fillarr , t_c32arr , t_f64arr , - /*18*/ t_hslice, t_i32slice, t_fillslice, t_c32slice, t_f64slice, + /*13*/ t_harr , t_i8arr , t_i32arr , t_fillarr , t_c32arr , t_f64arr , + /*19*/ t_hslice, t_i8slice, t_i32slice, t_fillslice, t_c32slice, t_f64slice, - /*22*/ t_comp, t_block, t_body, t_scope, - /*26*/ t_ns, t_nsDesc, - /*27*/ t_freed, t_harrPartial, + /*25*/ t_comp, t_block, t_body, t_scope, + /*29*/ t_ns, t_nsDesc, + /*31*/ t_freed, t_harrPartial, #ifdef RT_PERF - /*29*/ t_funPerf, t_md1Perf, t_md2Perf, + /*33*/ t_funPerf, t_md1Perf, t_md2Perf, #endif t_COUNT }; @@ -110,7 +110,8 @@ char* format_type(u8 u) { /*derv.c*/ F(fork,"(fork)") F(atop,"(atop)") F(md1d,"(derived 1-modifier)") F(md2d,"(derived 2-modifier)") \ /*sort.c*/ F(gradeUp,"⍋") \ /*sysfn.c*/ F(type,"•Type") F(decp,"•Decompose") F(primInd,"•PrimInd") F(glyph,"•Glyph") F(fill,"•FillFn") \ - /*sysfn.c*/ F(grLen,"•GroupLen") F(grOrd,"•groupOrd") F(asrt,"!") F(sys,"•getsys") F(bqn,"•BQN") F(cmp,"•Cmp") F(internal,"•Internal") F(show,"•Show") F(out,"•Out") + /*sysfn.c*/ F(grLen,"•GroupLen") F(grOrd,"•groupOrd") F(asrt,"!") F(sys,"•getsys") F(bqn,"•BQN") F(cmp,"•Cmp") F(internal,"•Internal") F(show,"•Show") F(out,"•Out") \ + /*sysfn.c*/ F(fchars,"•FChars") F(fbytes,"•FBytes") enum PrimFns { #define F(N,X) pf_##N, diff --git a/src/main.c b/src/main.c index ccf865d8..d9abc965 100644 --- a/src/main.c +++ b/src/main.c @@ -75,10 +75,14 @@ int main() { c_src = NULL; } if (c_src) { + B srcB = fromUTF8(c_src, c_len); #ifdef COMP_COMP_TIME - for (i32 i = 0; i < 100; i++) { dec(bqn_exec(fromUTF8(c_src, c_len))); gc_maybeGC(); } rtPerf_print(); exit(0); + gc_add(srcB); + for (i32 i = 0; i < 100; i++) { dec(bqn_exec(inc(srcB))); gc_maybeGC(); } + rtPerf_print(); + exit(0); #endif - bqn_setComp(bqn_exec(fromUTF8(c_src, c_len))); + bqn_setComp(bqn_exec(srcB)); } else { printf("couldn't read c.bqn\n"); exit(1); diff --git a/src/sysfn.c b/src/sysfn.c index 1068cf64..62878e45 100644 --- a/src/sysfn.c +++ b/src/sysfn.c @@ -163,7 +163,46 @@ B cmp_c2(B t, B w, B x) { return r; } -#define F(A,M,D) M(type) M(decp) M(primInd) M(glyph) A(fill) A(grLen) D(grOrd) A(asrt) M(out) M(show) M(sys) M(bqn) D(cmp) D(internal) +typedef struct TmpFile { // to be turned into a proper I8Arr + struct Arr; + i8 a[]; +} TmpFile; + +TmpFile* readFile(B path) { // consumes + u64 plen = utf8lenB(path); + char p[plen+1]; + toUTF8(path, p); + p[plen] = 0; + FILE* f = fopen(p, "r"); + if (f==NULL) thrM("Couldn't read file"); + fseek(f, 0, SEEK_END); + u64 len = ftell(f); + fseek(f, 0, SEEK_SET); + TmpFile* src = mm_allocN(fsizeof(TmpFile,a,u8,len), t_i8arr); + arr_shVec(tag(src,ARR_TAG), len); + fread((char*)src->a, 1, len, f); + fclose(f); + dec(path); + return src; +} + +B fchars_c1(B t, B x) { + TmpFile* c = readFile(x); + B r = fromUTF8((char*)c->a, c->ia); + ptr_dec(c); + return r; +} +B fbytes_c1(B t, B x) { + TmpFile* f = readFile(x); usz ia = f->ia; u8* p = (u8*)f->a; + u32* rp; B r = m_c32arrv(&rp, ia); + for (i64 i = 0; i < ia; i++) rp[i] = p[i]; + ptr_dec(f); + return r; +} + + + +#define F(A,M,D) M(type) M(decp) M(primInd) M(glyph) A(fill) A(grLen) D(grOrd) A(asrt) M(out) M(show) M(sys) M(bqn) D(cmp) D(internal) M(fchars) M(fbytes) BI_FNS0(F); static inline void sysfn_init() { BI_FNS1(F) } #undef F @@ -185,6 +224,8 @@ B sys_c1(B t, B x) { else if (eqStr(c, U"bqn")) r.a[i] = inc(bi_bqn); else if (eqStr(c, U"cmp")) r.a[i] = inc(bi_cmp); else if (eqStr(c, U"timed")) r.a[i] = inc(bi_timed); + else if (eqStr(c, U"fchars")) r.a[i] = inc(bi_fchars); + else if (eqStr(c, U"fbytes")) r.a[i] = inc(bi_fbytes); else { dec(x); thrM("Unknown system function"); } } return harr_fcd(r, x); diff --git a/src/utf.c b/src/utf.c index c1ea7977..9c1580a1 100644 --- a/src/utf.c +++ b/src/utf.c @@ -1,4 +1,4 @@ -i8 utf8_len(u8 ch) { +i8 utf8lenb(u8 ch) { if (ch<128) return 1; if ((ch>>5)== 0b110) return 2; if ((ch>>4)== 0b1110) return 3; @@ -6,7 +6,7 @@ i8 utf8_len(u8 ch) { return -1; } u32 utf8_p(u8* p) { - i32 len = utf8_len(*p); + i32 len = utf8lenb(*p); switch (len) { default: UD; case -1: return (u32)-1; case 1: return *p; @@ -19,7 +19,7 @@ B fromUTF8(char* s, i64 len) { u64 sz = 0; i64 j = 0; while (true) { - i8 l = utf8_len((u8)s[j]); + i8 l = utf8lenb((u8)s[j]); if (l==-1) thrM("Invalid UTF-8"); if (j>=len) { if (j!=len) thrM("Invalid UTF-8"); @@ -31,7 +31,7 @@ B fromUTF8(char* s, i64 len) { u32* rp; B r = m_c32arrv(&rp, sz); u64 p = 0; // TODO verify - for (i64 i = 0; i < len; i+= utf8_len((u8)s[i])) rp[p++] = utf8_p((u8*)s+i); // may read after end, eh + for (i64 i = 0; i < len; i+= utf8lenb((u8)s[i])) rp[p++] = utf8_p((u8*)s+i); // may read after end, eh return r; } @@ -41,14 +41,37 @@ B fromUTF8l(char* s) { void printUTF8(u32 c) { if (c<128) printf("%c", c); - else if (c<=0x07FF) printf("%c%c" , 0xC0| c>>6 , 0x80| (c &0x3F) ); - else if (c<=0xFFFF) printf("%c%c%c" , 0xE0| c>>12, 0x80| (c>>6 &0x3F), 0x80| (c &0x3F) ); - else printf("%c%c%c%c", 0xF0| c>>18, 0x80| (c>>12 &0x3F), 0x80| (c>>6 &0x3F), 0x80| (c&0x3F)); + else if (c<=0x07FF) printf("%c%c" , 0xC0| c>>6 , 0x80|(c &0x3F) ); + else if (c<=0xFFFF) printf("%c%c%c" , 0xE0| c>>12, 0x80|(c>>6 &0x3F), 0x80|(c &0x3F) ); + else printf("%c%c%c%c", 0xF0| c>>18, 0x80|(c>>12&0x3F), 0x80|(c>>6&0x3F), 0x80|(c&0x3F)); } u64 snprintUTF8(char* p, u64 l, u32 c) { if (c<128) return snprintf(p, l, "%c", c); - else if (c<=0x07FF) return snprintf(p, l, "%c%c" , 0xC0| c>>6 , 0x80| (c &0x3F) ); - else if (c<=0xFFFF) return snprintf(p, l, "%c%c%c" , 0xE0| c>>12, 0x80| (c>>6 &0x3F), 0x80| (c &0x3F) ); - else return snprintf(p, l, "%c%c%c%c", 0xF0| c>>18, 0x80| (c>>12 &0x3F), 0x80| (c>>6 &0x3F), 0x80| (c&0x3F)); + else if (c<=0x07FF) return snprintf(p, l, "%c%c" , 0xC0|c>>6 , 0x80|(c &0x3F) ); + else if (c<=0xFFFF) return snprintf(p, l, "%c%c%c" , 0xE0|c>>12, 0x80|(c>>6 &0x3F), 0x80|(c &0x3F) ); + else return snprintf(p, l, "%c%c%c%c", 0xF0|c>>18, 0x80|(c>>12&0x3F), 0x80|(c>>6&0x3F), 0x80|(c&0x3F)); } + +u64 utf8lenB(B x) { // doesn't consume; may error as it verifies whether is all chars + assert(isArr(x)); + BS2B xgetU = TI(x).getU; + usz ia = a(x)->ia; + u64 res = 0; + for (usz i = 0; i < ia; i++) { + u32 c = o2c(xgetU(x,i)); + res+= c<128? 1 : c<0x07FF? 2 : c<0xFFFF? 3 : 4; + } + return res; +} +void toUTF8(B x, char* p) { // doesn't consume; doesn't verify anything; p must have utf8_lenB(x) bytes + BS2B xgetU = TI(x).getU; + usz ia = a(x)->ia; + for (usz i = 0; i < ia; i++) { + u32 c = o2cu(xgetU(x,i)); + if (c<128) { *p++ = c; } + else if (c<=0x07FF) { *p++ = 0xC0|c>>6 ; *p++ = 0x80|(c &0x3F); } + else if (c<=0xFFFF) { *p++ = 0xE0|c>>12; *p++ = 0x80|(c>>6 &0x3F);*p++ = 0x80|(c &0x3F); } + else { *p++ = 0xF0|c>>18; *p++ = 0x80|(c>>12&0x3F);*p++ = 0x80|(c>>6&0x3F); *p++ = 0x80|(c&0x3F); } + } +} \ No newline at end of file diff --git a/src/vm.c b/src/vm.c index 3806fe94..c692c57c 100644 --- a/src/vm.c +++ b/src/vm.c @@ -653,7 +653,7 @@ void popCatch() { #endif } -NOINLINE void thr(B msg) { +NOINLINE NORETURN void thr(B msg) { if (cf>cfStart) { catchMessage = msg; cf--; @@ -688,7 +688,7 @@ NOINLINE void thr(B msg) { #endif } -NOINLINE void thrM(char* s) { +NOINLINE NORETURN void thrM(char* s) { thr(fromUTF8(s, strlen(s))); }