From 5f9aeafec6f3315687d2446aa8f19fb00e29f20f Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 26 Jan 2023 19:19:21 +0200 Subject: [PATCH] use switch for tokenization --- src/opt/comp.c | 118 +++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/src/opt/comp.c b/src/opt/comp.c index ebd9e1dc..b2ec206c 100644 --- a/src/opt/comp.c +++ b/src/opt/comp.c @@ -70,62 +70,74 @@ NOINLINE B nc_tokenize(B prims, B sysvs, u32* chars, usz len, bool* hasBlock) { u32 c = chars[i++]; if (i==pi) thrF("Native compiler: Tokenizer stuck on \\u%xi / %i", c, c); B val; - if (c==U'∞') val = nc_literal(m_f64(1.0/0)); - else if (c==U'Ο€') val = nc_literal(m_f64(3.141592653589793)); - else if (c=='@') val = nc_literal(m_c32(0)); - else if (c==' ') continue; - else if (nc_num(c) | (c==U'Β―')) { - bool neg = c==U'Β―'; - if (!neg) i--; - else if (!nc_num(chars[i])) thrM("Native compiler: Standalone negative sign"); - f64 num = 0; - while (nc_num(chars[i])) num = num*10 + (chars[i++]-'0'); - if (neg) num = -num; - val = nc_literal(m_f64(num)); - } else if (c=='"') { - usz i0 = i; - while ('"'!=chars[i]) { i++; if (i>=len) thrM("Native compiler: Unclosed string literal"); } - usz ia = i-i0; - u32* vp; val = nc_literal(m_c32arrv(&vp, ia)); - memcpy(vp, chars+i0, ia*sizeof(u32)); - i++; - } else if (c=='#') { // comments - while (iβ‰ =≀β‰₯β‰‘β‰’βŠ£βŠ’β₯ŠβˆΎβ‰β‹ˆβ†‘β†“β†•Β«Β»βŒ½β‰/β‹β’βŠβŠ‘βŠβŠ’βˆŠβ·βŠ”!Λ™ΛœΛ˜Β¨βŒœβΌΒ΄Λ`βˆ˜β—‹βŠΈβŸœβŒΎβŠ˜β—ΆβŽ‰βš‡βŸβŽŠ"; - usz j = 0; - while(primRepr[j]) { - if (primRepr[j]==c) { val = nc_literal(IGet(prims, j)); goto add; } - j++; + case '"': { // string literal + usz i0 = i; + while ('"'!=chars[i]) { i++; if (i>=len) thrM("Native compiler: Unclosed string literal"); } + if (chars[i+1]=='"') thrM("Native compiler: No support for escaped quote characters in strings"); + usz ia = i-i0; + u32* vp; val = nc_literal(m_c32arrv(&vp, ia)); + memcpy(vp, chars+i0, ia*sizeof(u32)); + i++; + break; } - // syntax - u32* basicChars = U"()βŸ¨βŸ©β†β†©{}"; - while (*basicChars) { - if (c==*basicChars) { val = m_c32(c); goto add; } - basicChars++; + case '#': { // comments + while (iβ‰ =≀β‰₯β‰‘β‰’βŠ£βŠ’β₯ŠβˆΎβ‰β‹ˆβ†‘β†“β†•Β«Β»βŒ½β‰/β‹β’βŠβŠ‘βŠβŠ’βˆŠβ·βŠ”!Λ™ΛœΛ˜Β¨βŒœβΌΒ΄Λ`βˆ˜β—‹βŠΈβŸœβŒΎβŠ˜β—ΆβŽ‰βš‡βŸβŽŠ"; + usz j = 0; + while(primRepr[j]) { + if (primRepr[j]==c) { val = nc_literal(IGet(prims, j)); goto add; } + j++; + } + thrF("Native compiler: Can't tokenize \\u%xi / %i", c, c); } - thrF("Native compiler: Can't tokenize \\u%xi / %i", c, c); } add: nc_add(&r, val);