Implement some of the JSON5 enhancements to string and numeric literals.

This is an incremental check-in of work in progress.

FossilOrigin-Name: 9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82

diff --git a/manifest b/manifest
index aefd7ea1211f5b032c047cd2cc76035b85a3086e..4c3f3616dd8df14c777c1d70e2a7ac2d9a626ced 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Partial\simplementation\sof\sJSON5\snumeric\sliteral\sextensions.\s\sUse\sa\sswitch()\nstatement\sin\sthe\sparser\sfor\sbetter\sperformance.
-D 2023-04-26T17:30:28.870
+C Implement\ssome\sof\sthe\sJSON5\senhancements\sto\sstring\sand\snumeric\sliterals.\nThis\sis\san\sincremental\scheck-in\sof\swork\sin\sprogress.
+D 2023-04-26T20:26:14.388
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4
-F src/json.c a346dbe63628f8cb9c6bf6bbccb8fc81a49730db2311ec9b483d7fc3582da223
+F src/json.c f82b86599b220a2c60731eba883bbea92ee4d6e61aaa56740179e8aa5e6ec596
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136
 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d
@@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3
-R 0e63126100fc0a5fcfa419eb4dbeae91
+P 78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485
+R e7aefca3f1ce2ae26f999bdfb414a2b2
 U drh
-Z 7c56c83a7327e89fcacb5797373de3a9
+Z b316d61c8d04f67716fa9ffd06cd770b
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 76ab21e90fcd389d07575d4a41cf920f8daa9c72..2e71d5072eb3b3e844f3a3aa887e4681448d633e 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485
\ No newline at end of file
+9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82
\ No newline at end of file

diff --git a/src/json.c b/src/json.c
index 63feff94882ad4d722fe422f9331a131f6d7ae56..3e2d6646d83a72e1ac92b375be4e2513dc7213a9 100644 (file)
--- a/src/json.c
+++ b/src/json.c
@@ -104,6 +104,7 @@ static const char * const jsonType[] = {
 #define JNODE_PATCH   0x10         /* Patch with JsonNode.u.pPatch */
 #define JNODE_APPEND  0x20         /* More ARRAY/OBJECT entries at u.iAppend */
 #define JNODE_LABEL   0x40         /* Is a label of an object */
+#define JNODE_JSON5   0x80         /* Node contains JSON5 enhancements */
 
 
 /* A single node of parsed JSON
@@ -295,6 +296,72 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){
   assert( p->nUsed<p->nAlloc );
 }
 
+/*
+** The zIn[0..N] string is a JSON5 string literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){
+  jsonAppendChar(p, '"');
+  jsonAppendRaw(p, &zIn[1], N-2); /* TODO: translate JSON5 escapes */
+  jsonAppendChar(p, '"');
+}
+
+/*
+** The zIn[0..N] string is a JSON5 integer literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){
+  if( zIn[0]=='+' ){
+    zIn++;
+    N--;
+  }else if( zIn[0]=='-' ){
+    jsonAppendChar(p, '-');
+    zIn++;
+    N--;
+  }
+  while( zIn[0]=='0' && N>1 ){
+    zIn++;
+    N--;
+  } 
+  jsonAppendRaw(p, zIn, N);
+}
+
+/*
+** The zIn[0..N] string is a JSON5 real literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){
+  int i;
+  if( zIn[0]=='+' ){
+    zIn++;
+    N--;
+  }else if( zIn[0]=='-' ){
+    jsonAppendChar(p, '-');
+    zIn++;
+    N--;
+  }
+  if( zIn[0]=='.' ){
+    jsonAppendChar(p, '0');
+  }
+  for(i=0; i<N; i++){
+    if( zIn[i]=='.' && (i+1==N || !sqlite3Isdigit(zIn[i+1])) ){
+      jsonAppendRaw(p, zIn, i);
+      zIn += i;
+      N -= i;
+      jsonAppendChar(p, '0');
+      break;
+    }
+  }
+  if( N>0 ){
+    jsonAppendRaw(p, zIn, N);
+  }
+}
+
+
+
 /*
 ** Append a function parameter value to the JSON string under 
 ** construction.
@@ -425,17 +492,32 @@ static void jsonRenderNode(
       break;
     }
     case JSON_STRING: {
+      assert( pNode->eU==1 );
       if( pNode->jnFlags & JNODE_RAW ){
-        assert( pNode->eU==1 );
         jsonAppendString(pOut, pNode->u.zJContent, pNode->n);
-        break;
+      }else if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
       }
-      /* no break */ deliberate_fall_through
+      break;
+    }
+    case JSON_REAL: {
+      assert( pNode->eU==1 );
+      if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      }
+      break;
     }
-    case JSON_REAL:
     case JSON_INT: {
       assert( pNode->eU==1 );
-      jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      }
       break;
     }
     case JSON_ARRAY: {
@@ -615,9 +697,10 @@ static void jsonReturn(
         const char *z;
         char *zOut;
         u32 j;
+        u32 nOut = n;
         assert( pNode->eU==1 );
         z = pNode->u.zJContent;
-        zOut = sqlite3_malloc( n+1 );
+        zOut = sqlite3_malloc( nOut+1 );
         if( zOut==0 ){
           sqlite3_result_error_nomem(pCtx);
           break;
@@ -669,6 +752,15 @@ static void jsonReturn(
                 c = '\r';
               }else if( c=='t' ){
                 c = '\t';
+              }else if( c=='v' ){
+                c = '\v';
+              }else if( c=='\'' ){
+                c = '\'';
+              }else if( c=='0' ){
+                c = 0;
+              }else if( c=='x' ){
+                c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]);
+                i += 2;
               }
               zOut[j++] = c;
             }
@@ -749,13 +841,18 @@ static int jsonParseAddNode(
   return pParse->nNode++;
 }
 
+/*
+** Return true if z[] begins with 2 (or more) hexadecimal digits
+*/
+static int jsonIs2Hex(const char *z){
+  return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]);
+}
+
 /*
 ** Return true if z[] begins with 4 (or more) hexadecimal digits
 */
 static int jsonIs4Hex(const char *z){
-  int i;
-  for(i=0; i<4; i++) if( !sqlite3Isxdigit(z[i]) ) return 0;
-  return 1;
+  return jsonIs2Hex(z) && jsonIs2Hex(&z[2]);
 }
 
 /*
@@ -978,9 +1075,17 @@ json_parse_restart:
     pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1;
     return j+1;
   }
-  case '"': {
+  case '\'': {
+    u8 jnFlags;
+    char cDelim;
+    pParse->has5 = 1;
+    jnFlags = JNODE_JSON5;
+    goto parse_string;
+  case '"':
     /* Parse string */
-    u8 jnFlags = 0;
+    jnFlags = 0;
+  parse_string:
+    cDelim = z[i];
     j = i+1;
     for(;;){
       c = z[j];
@@ -992,12 +1097,16 @@ json_parse_restart:
         c = z[++j];
         if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
            || c=='n' || c=='r' || c=='t'
-           || (c=='u' && jsonIs4Hex(z+j+1)) ){
-          jnFlags = JNODE_ESCAPE;
+           || (c=='u' && jsonIs4Hex(&z[j+1])) ){
+          jnFlags |= JNODE_ESCAPE;
+        }else if( c=='\'' || c=='0' || c=='v'
+           || (c=='x' && jsonIs2Hex(&z[j+1])) ){
+          jnFlags |= (JNODE_ESCAPE|JNODE_JSON5);
+          pParse->has5 = 1;
         }else{
           return -1;
         }
-      }else if( c=='"' ){
+      }else if( c==cDelim ){
         break;
       }
       j++;
@@ -1027,9 +1136,11 @@ json_parse_restart:
     }
     return -1;
   }
-  case '+':
+  case '+': {
+    u8 seenDP, seenE, jnFlags;
     pParse->has5 = 1;
-    /* fall through */
+    jnFlags = JNODE_JSON5;
+    goto parse_number;
   case '-':
   case '0':
   case '1':
@@ -1040,10 +1151,12 @@ json_parse_restart:
   case '6':
   case '7':
   case '8':
-  case '9': {
+  case '9':
     /* Parse number */
-    u8 seenDP = 0;
-    u8 seenE = 0;
+    jnFlags = 0;
+  parse_number:
+    seenDP = 0;
+    seenE = 0;
     assert( '-' < '0' );
     assert( '+' < '0' );
     assert( '.' < '0' );
@@ -1053,10 +1166,14 @@ json_parse_restart:
       if( c=='0' ){
         if( sqlite3Isdigit(z[i+1]) ){
           pParse->has5 = 1;
+          jnFlags = JNODE_JSON5;
         }
       }else{
         if( !sqlite3Isdigit(z[i+1]) ) return -1;
-        if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ) pParse->has5 = 1;
+        if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ){
+          pParse->has5 = 1;
+          jnFlags = JNODE_JSON5;
+        }
       }
     }
     for(j=i+1;; j++){
@@ -1103,6 +1220,9 @@ json_parse_restart:
     if( z[j-1]<'0' ) return -1;
     jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT,
                         j - i, &z[i]);
+    if( jnFlags && !pParse->oom ){
+      pParse->aNode[pParse->nNode-1].jnFlags = jnFlags;
+    }
     return j;
   }
   case '}': {