lib-json: Implement JSON value input stream

diff --git a/src/lib-json/Makefile.am b/src/lib-json/Makefile.am
index e170e8c8595d62a573b3be504fb908f8aff6a422..24934ae3c7ba0ea0f0912cae82c1fff62bf8436b 100644 (file)
--- a/src/lib-json/Makefile.am
+++ b/src/lib-json/Makefile.am
@@ -8,19 +8,22 @@ libjson_la_SOURCES = \
        json-syntax.c \
        json-types.c \
        json-parser.new.c \
-       json-generator.c
+       json-generator.c \
+       json-istream.c
 libjson_la_LIBADD = -lm
 
 headers = \
        json-syntax.h \
        json-types.h \
        json-parser.new.h \
-       json-generator.h
+       json-generator.h \
+       json-istream.h
 
 test_programs = \
        test-json-parser \
        test-json-generator \
-       test-json-io
+       test-json-io \
+       test-json-istream
 
 noinst_PROGRAMS = $(test_programs)
 
@@ -57,6 +60,13 @@ test_json_io_LDADD = \
 test_json_io_DEPENDENCIES = \
        $(test_deps)
 
+test_json_istream_SOURCE = \
+       test-json-istream.c
+test_json_istream_LDADD = \
+       $(test_libs)
+test_json_istream_DEPENDENCIES = \
+       $(test_deps)
+
 pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
 

diff --git a/src/lib-json/json-istream.c b/src/lib-json/json-istream.c
new file mode 100644 (file)
index 0000000..17ca3e9
--- /dev/null
+++ b/src/lib-json/json-istream.c
@@ -0,0 +1,584 @@
+/* Copyright (c) 2017-2023 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "istream-seekable.h"
+
+#include "json-istream.h"
+
+struct json_istream {
+       int refcount;
+
+       struct istream *input;
+       enum json_istream_type type;
+       struct json_parser *parser;
+
+       struct json_node node;
+       unsigned int node_level;
+       unsigned int read_node_level;
+       unsigned int skip_nodes;
+
+       char *error;
+
+       bool opened:1;
+       bool closed:1;
+       bool node_parsed:1;    /* Parsed a node */
+       bool member_parsed:1;  /* Parsed an object member name */
+       bool read_member:1;    /* Read only the object member name */
+       bool end_of_list:1;    /* Encountered the end of current array/object */
+       bool end_of_input:1;   /* Encountered end of input */
+       bool skip_to_end:1;    /* Skip to the end of the JSON text */
+};
+
+/*
+ * Parser callbacks
+ */
+
+static void
+json_istream_parse_list_open(void *context, void *list_context,
+                            const char *name, bool object,
+                            void **list_context_r);
+static void
+json_istream_parse_list_close(void *context, void *list_context, bool object);
+static void
+json_istream_parse_object_member(void *context,
+                                void *parent_context ATTR_UNUSED,
+                                const char *name);
+static void
+json_istream_parse_value(void *context, void *list_context, const char *name,
+                        enum json_type type,
+                        const struct json_value *value);
+
+static const struct json_parser_callbacks parser_callbacks = {
+       .parse_list_open = json_istream_parse_list_open,
+       .parse_list_close = json_istream_parse_list_close,
+       .parse_object_member = json_istream_parse_object_member,
+       .parse_value = json_istream_parse_value
+};
+
+/*
+ * Object
+ */
+
+struct json_istream *
+json_istream_create(struct istream *input, enum json_istream_type type,
+                   const struct json_limits *limits,
+                   enum json_parser_flags parser_flags)
+{
+       struct json_istream *stream;
+
+       stream = i_new(struct json_istream, 1);
+       stream->refcount = 1;
+       stream->type = type;
+
+       stream->input = input; /* Parser holds reference */
+       stream->parser = json_parser_init(input, limits, parser_flags,
+                                         &parser_callbacks, (void *)stream);
+
+       return stream;
+}
+
+void json_istream_ref(struct json_istream *stream)
+{
+       i_assert(stream->refcount > 0);
+       stream->refcount++;
+}
+
+void json_istream_unref(struct json_istream **_stream)
+{
+       struct json_istream *stream = *_stream;
+
+       if (stream == NULL)
+               return;
+       *_stream = NULL;
+
+       i_assert(stream->refcount > 0);
+       if (--stream->refcount > 0)
+               return;
+
+       json_parser_deinit(&stream->parser);
+       i_free(stream->error);
+       i_free(stream);
+}
+
+void json_istream_destroy(struct json_istream **_stream)
+{
+       struct json_istream *stream = *_stream;
+
+       if (stream == NULL)
+               return;
+
+       json_istream_close(stream);
+       json_istream_unref(_stream);
+}
+
+void json_istream_close(struct json_istream *stream)
+{
+       stream->closed = TRUE;
+}
+
+bool json_istream_is_closed(struct json_istream *stream)
+{
+       return stream->closed;
+}
+
+unsigned int json_istream_get_node_level(struct json_istream *stream)
+{
+       return stream->read_node_level;
+}
+
+bool json_istream_is_at_end(struct json_istream *stream)
+{
+       i_assert(!stream->end_of_input || stream->input->eof);
+       return stream->end_of_input;
+}
+
+bool json_istream_failed(struct json_istream *stream)
+{
+       return (stream->error != NULL || stream->closed);
+}
+
+static void
+json_istream_set_error(struct json_istream *stream, const char *error)
+{
+       if (stream->error != NULL)
+               return;
+       stream->error = i_strdup(error);
+       json_istream_close(stream);
+}
+
+const char *json_istream_get_error(struct json_istream *stream)
+{
+       if (stream->error == NULL) {
+               return (stream->closed ? "<closed>" :
+                       (stream->end_of_input ? "END-OF-INPUT" : "<no error>"));
+       }
+       return stream->error;
+}
+
+int json_istream_finish(struct json_istream **_stream, const char **error_r)
+{
+       struct json_istream *stream = *_stream;
+       const char *error = NULL;
+       int ret;
+
+       i_assert(stream != NULL);
+       stream->skip_to_end = TRUE;
+       ret = json_istream_read(stream, NULL);
+       if (ret == 0)
+               return 0;
+
+       ret = 1;
+       if (stream->error != NULL) {
+               error = t_strdup(stream->error);
+               ret = -1;
+       } else if (stream->closed) {
+               error = "Stream is closed";
+               ret = -1;
+       } else if (!stream->end_of_input) {
+               error = "Spurious data at end of JSON value";
+               ret = -1;
+       }
+       if (ret < 0 && stream->error == NULL && stream->refcount > 1)
+               stream->error = i_strdup(error);
+       json_istream_unref(_stream);
+
+       if (error_r != NULL)
+               *error_r = error;
+       return ret;
+}
+
+void json_istream_get_location(struct json_istream *stream,
+                              struct json_parser_location *loc_r)
+{
+       json_parser_get_location(stream->parser, loc_r);
+}
+
+/*
+ * Parser callbacks
+ */
+
+static inline bool json_istream_parse_skip(struct json_istream *stream)
+{
+       if (stream->skip_to_end)
+               return TRUE;
+       if (stream->skip_nodes > 0) {
+               if (stream->skip_nodes < UINT_MAX)
+                       stream->skip_nodes--;
+               return TRUE;
+       }
+       return FALSE;
+}
+
+static void
+json_istream_parse_list_open(void *context, void *parent_context ATTR_UNUSED,
+                            const char *name, bool object,
+                            void **list_context_r ATTR_UNUSED)
+{
+       struct json_istream *stream = context;
+       unsigned int node_level = stream->node_level;
+
+       i_assert(!stream->node_parsed);
+       i_assert(stream->node_level >= stream->read_node_level);
+
+       if (!stream->opened) {
+               stream->opened = TRUE;
+               switch (stream->type) {
+               case JSON_ISTREAM_TYPE_NORMAL:
+                       break;
+               case JSON_ISTREAM_TYPE_ARRAY:
+                       if (object) {
+                               i_assert(stream->error == NULL);
+                               json_istream_set_error(
+                                       stream, "Root is not an array");
+                               json_parser_interrupt(stream->parser);
+                       }
+                       return;
+               case JSON_ISTREAM_TYPE_OBJECT:
+                       if (!object) {
+                               i_assert(stream->error == NULL);
+                               json_istream_set_error(
+                                       stream, "Root is not an object");
+                               json_parser_interrupt(stream->parser);
+                       }
+                       return;
+               }
+       }
+
+       stream->node_level++;
+
+       if (node_level == stream->read_node_level) {
+               i_zero(&stream->node);
+               stream->node.name = name;
+               stream->node.type = (object ?
+                       JSON_TYPE_OBJECT : JSON_TYPE_ARRAY);
+               stream->node.value.content_type =
+                       JSON_CONTENT_TYPE_LIST;
+               if (!json_istream_parse_skip(stream)) {
+                       stream->node_parsed = TRUE;
+                       json_parser_interrupt(stream->parser);
+               }
+       }
+}
+
+static void
+json_istream_parse_list_close(void *context, void *list_context ATTR_UNUSED,
+                             bool object)
+{
+       struct json_istream *stream = context;
+
+       i_assert(!stream->node_parsed);
+
+       if (stream->node_level == 0) {
+               /* Already at lowest level; eat the root node */
+               i_assert(stream->opened);
+               stream->opened = FALSE;
+               switch (stream->type) {
+               case JSON_ISTREAM_TYPE_ARRAY:
+               case JSON_ISTREAM_TYPE_OBJECT:
+                       return;
+               default:
+                       i_unreached();
+               }
+       }
+
+       stream->node_level--;
+       if (stream->node_level == 0) {
+               /* Moved to lowest level */
+               switch (stream->type) {
+               case JSON_ISTREAM_TYPE_NORMAL:
+                       stream->opened = FALSE;
+                       break;
+               case JSON_ISTREAM_TYPE_ARRAY:
+               case JSON_ISTREAM_TYPE_OBJECT:
+                       break;
+               }
+       }
+
+       if (stream->node_level < stream->read_node_level) {
+               stream->end_of_list = TRUE;
+               if (!json_istream_parse_skip(stream)) {
+                       i_zero(&stream->node);
+                       stream->node.type = (object ?
+                               JSON_TYPE_OBJECT : JSON_TYPE_ARRAY);
+                       stream->node_parsed = TRUE;
+                       stream->member_parsed = TRUE;
+                       json_parser_interrupt(stream->parser);
+               }
+       }
+}
+
+static void
+json_istream_parse_object_member(void *context,
+                                void *parent_context ATTR_UNUSED,
+                                const char *name)
+{
+       struct json_istream *stream = context;
+
+       i_assert(!stream->node_parsed && !stream->member_parsed);
+
+       if (!stream->read_member)
+               return;
+       if (stream->skip_to_end || stream->skip_nodes > 0)
+               return;
+
+       i_assert(stream->node_level >= stream->read_node_level);
+
+       if (stream->node_level != stream->read_node_level)
+               return;
+
+       i_zero(&stream->node);
+       stream->node.name = name;
+       stream->member_parsed = TRUE;
+       json_parser_interrupt(stream->parser);
+}
+
+static void
+json_istream_parse_value(void *context, void *parent_context ATTR_UNUSED,
+                        const char *name, enum json_type type,
+                        const struct json_value *value)
+{
+       struct json_istream *stream = context;
+
+       i_assert(!stream->node_parsed);
+       i_assert(stream->node_level >= stream->read_node_level);
+
+       if (!stream->opened) {
+               switch (stream->type) {
+               case JSON_ISTREAM_TYPE_NORMAL:
+                       break;
+               case JSON_ISTREAM_TYPE_ARRAY:
+                       i_assert(stream->error == NULL);
+                       json_istream_set_error(
+                               stream, "Root is not an array");
+                       json_parser_interrupt(stream->parser);
+                       return;
+               case JSON_ISTREAM_TYPE_OBJECT:
+                       i_assert(stream->error == NULL);
+                       json_istream_set_error(
+                               stream, "Root is not an object");
+                       json_parser_interrupt(stream->parser);
+                       return;
+               }
+       }
+
+       if (stream->node_level != stream->read_node_level)
+               return;
+       if (json_istream_parse_skip(stream))
+               return;
+       i_zero(&stream->node);
+       stream->node.name = name;
+       stream->node.type = type;
+       stream->node.value = *value;
+       stream->node_parsed = TRUE;
+       json_parser_interrupt(stream->parser);
+}
+
+/*
+ *
+ */
+
+int json_istream_read(struct json_istream *stream, struct json_node *node_r)
+{
+       const char *error;
+       int ret;
+
+       if (stream->closed)
+               return -1;
+
+       if (!stream->node_parsed) {
+               if (stream->end_of_input) {
+                       if (node_r != NULL)
+                               *node_r = stream->node;
+                       return -1;
+               }
+               if (stream->end_of_list) {
+                       if (node_r != NULL)
+                               *node_r = stream->node;
+                       return 1;
+               }
+               ret = json_parse_more(stream->parser, &error);
+               if (ret < 0) {
+                       json_istream_set_error(stream, error);
+                       return ret;
+               }
+               if (stream->error != NULL)
+                       return -1;
+               if (ret == 0 && !stream->node_parsed) {
+                       if (node_r != NULL)
+                               i_zero(node_r);
+                       return 0;
+               }
+               if (ret > 0) {
+                       stream->end_of_input = TRUE;
+                       if (!stream->node_parsed) {
+                               if (node_r != NULL)
+                                       *node_r = stream->node;
+                               return -1;
+                       }
+               }
+       }
+
+       if (node_r != NULL)
+               *node_r = stream->node;
+       return 1;
+}
+
+int json_istream_read_next(struct json_istream *stream,
+                          struct json_node *node_r)
+{
+       int ret;
+
+       ret = json_istream_read(stream, node_r);
+       if (ret <= 0)
+               return ret;
+       json_istream_skip(stream);
+       return 1;
+}
+
+static void json_istream_next_node(struct json_istream *stream)
+{
+       if (stream->skip_nodes == 0 &&
+           stream->member_parsed && !stream->node_parsed)
+               stream->skip_nodes = 1;
+       stream->node_parsed = FALSE;
+       stream->member_parsed = FALSE;
+}
+
+void json_istream_skip(struct json_istream *stream)
+{
+       json_istream_next_node(stream);
+}
+
+void json_istream_ignore(struct json_istream *stream, unsigned int count)
+{
+       bool parsed = (stream->member_parsed || stream->node_parsed);
+
+       if (count == 0)
+               return;
+       json_istream_skip(stream);
+       if (count == UINT_MAX)
+               stream->skip_nodes = UINT_MAX;
+       else {
+               if (parsed)
+                       count--;
+               if (stream->skip_nodes >= (UINT_MAX - count))
+                       stream->skip_nodes = UINT_MAX;
+               else
+                       stream->skip_nodes += count;
+       }
+}
+
+int json_istream_read_object_member(struct json_istream *stream,
+                                   const char **name_r)
+{
+       const char *error;
+       int ret;
+
+       if (stream->closed)
+               return -1;
+
+       if (!stream->node_parsed && !stream->member_parsed) {
+               if (stream->end_of_input) {
+                       *name_r = NULL;
+                       return -1;
+               }
+               if (stream->end_of_list) {
+                       *name_r = NULL;
+                       return 1;
+               }
+               stream->read_member = TRUE;
+               ret = json_parse_more(stream->parser, &error);
+               stream->read_member = FALSE;
+               if (ret < 0) {
+                       json_istream_set_error(stream, error);
+                       return ret;
+               }
+               if (stream->error != NULL)
+                       return -1;
+               if (stream->node_parsed)
+                       stream->node_parsed = FALSE;
+               if (ret == 0 && !stream->member_parsed) {
+                       *name_r = NULL;
+                       return 0;
+               }
+               if (ret > 0) {
+                       stream->end_of_input = TRUE;
+                       i_assert(!stream->member_parsed);
+                       *name_r = NULL;
+                       return -1;
+               }
+       }
+
+       if (stream->end_of_list) {
+               *name_r = NULL;
+               return 1;
+       }
+
+       *name_r = stream->node.name;
+       return 1;
+}
+
+int json_istream_descend(struct json_istream *stream,
+                        struct json_node *node_r)
+{
+       struct json_node node;
+       int ret;
+
+       ret = json_istream_read(stream, &node);
+       if (ret <= 0)
+               return ret;
+
+       json_istream_skip(stream);
+       if (json_node_is_object(&node) || json_node_is_array(&node))
+               stream->read_node_level++;
+       if (node_r != NULL)
+               *node_r = node;
+       return ret;
+}
+
+static void json_istream_ascend_common(struct json_istream *stream)
+{
+       stream->skip_nodes = 0;
+       stream->node_parsed = FALSE;
+       stream->member_parsed = FALSE;
+       stream->end_of_list = FALSE;
+}
+
+void json_istream_ascend(struct json_istream *stream)
+{
+       i_assert(stream->read_node_level > 0);
+       json_istream_ascend_common(stream);
+       stream->read_node_level--;
+}
+
+void json_istream_ascend_to(struct json_istream *stream,
+                            unsigned int node_level)
+{
+       i_assert(stream->read_node_level >= node_level);
+       if (node_level == stream->read_node_level) {
+               json_istream_skip(stream);
+               return;
+       }
+       json_istream_ascend_common(stream);
+       stream->read_node_level = node_level;
+}
+
+int json_istream_walk(struct json_istream *stream, struct json_node *node_r)
+{
+       struct json_node node;
+       int ret;
+
+       ret = json_istream_descend(stream, &node);
+       if (ret <= 0)
+               return ret;
+       if (json_node_is_end(&node)) {
+               i_assert(stream->end_of_list);
+               i_assert(stream->read_node_level > 0);
+               json_istream_ascend_common(stream);
+               stream->read_node_level--;
+       }
+       if (node_r != NULL)
+               *node_r = node;
+       return 1;
+}

diff --git a/src/lib-json/json-istream.h b/src/lib-json/json-istream.h
new file mode 100644 (file)
index 0000000..5d85079
--- /dev/null
+++ b/src/lib-json/json-istream.h
@@ -0,0 +1,146 @@
+#ifndef JSON_ISTREAM_H
+#define JSON_ISTREAM_H
+
+#include "json-parser.new.h"
+
+// FIXME: don't bother recording values if we're only validating/skipping
+
+struct json_istream;
+
+enum json_istream_type {
+       /* Normal JSON stream: there is one node at root; need to descend first
+          for reading the members of a top-level array or object. */
+       JSON_ISTREAM_TYPE_NORMAL = 0,
+       /* Stream starts inside a JSON array. If the top-level node is not an
+          array, the stream returns an error at first read. */
+       JSON_ISTREAM_TYPE_ARRAY,
+       /* Stream starts inside a JSON object. If the top-level node is not an
+          object, the stream returns an error at first read. */
+       JSON_ISTREAM_TYPE_OBJECT,
+};
+
+struct json_istream *
+json_istream_create(struct istream *input, enum json_istream_type type,
+                   const struct json_limits *limits,
+                   enum json_parser_flags parser_flags);
+void json_istream_ref(struct json_istream *stream);
+void json_istream_unref(struct json_istream **_stream);
+void json_istream_destroy(struct json_istream **_stream);
+
+void json_istream_close(struct json_istream *stream);
+bool json_istream_is_closed(struct json_istream *stream) ATTR_PURE;
+
+static inline struct json_istream *
+json_istream_create_array(struct istream *input,
+                         const struct json_limits *limits,
+                         enum json_parser_flags parser_flags)
+{
+       return json_istream_create(input, JSON_ISTREAM_TYPE_ARRAY,
+                                  limits, parser_flags);
+}
+
+static inline struct json_istream *
+json_istream_create_object(struct istream *input,
+                          const struct json_limits *limits,
+                          enum json_parser_flags parser_flags)
+{
+       return json_istream_create(input, JSON_ISTREAM_TYPE_OBJECT,
+                                  limits, parser_flags);
+}
+
+/* Get the current node's level in the JSON syntax hierarchy. */
+unsigned int json_istream_get_node_level(struct json_istream *stream);
+
+/* Returns TRUE if the JSON text is parsed to the end. Whether this means
+   that the input stream is also at EOF depends on the parser flags. */
+bool json_istream_is_at_end(struct json_istream *stream);
+
+/* Returns true when the stream has reached an error condition. */
+bool json_istream_failed(struct json_istream *stream);
+/* Returns error string for the last error. It also returns "EOF" in case there
+   is no error, but eof is set. Otherwise it returns "<no error>". */
+const char *json_istream_get_error(struct json_istream *stream);
+/* Get the current parse location */
+void json_istream_get_location(struct json_istream *stream,
+                              struct json_parser_location *loc_r);
+
+/* Finish reading input from the JSON stream. If any unread data remains in the
+   remainder of the JSON input, an error will occur. This function returns -1
+   upon error, 0 when more input is needed to finish and 1 when finishing the
+   input was successful. The error_r parameter will be set when the return value
+   is -1 and will return any (pre-existing or final) error in the stream. The
+   provided stream is dereferenced implicitly when the return value is not 0. */
+int json_istream_finish(struct json_istream **_stream,
+                       const char **error_r);
+
+/* Read a JSON node from the stream. If one is already read, this function
+   returns that node immediately. Returns 1 on success, 0 if more data is
+   needed from the input stream, or -1 upon error or EOF. The last node in
+   an object/array reads as JSON_TYPE_OBJECT/JSON_TYPE_ARRAY with
+   JSON_CONTENT_TYPE_NONE. Use json_node_is_end() or the more specific
+   json_node_is_object_end()/json_node_is_array_end() to check for these.
+ */
+int json_istream_read(struct json_istream *stream,
+                     struct json_node *node_r);
+/* Read the next JSON node from the stream. This is equivalent to calling
+   json_istream_read() and subsequently json_istream_skip() upon success.
+   Returns 1 on success, 0 if more data is needed from the input stream, or
+   -1 upon error or EOF. Note that this doesn't descend into array/object nodes,
+   these are skipped as a unit. */
+int json_istream_read_next(struct json_istream *stream,
+                          struct json_node *node_r);
+/* Skip the JSON node that was (partially) parsed before. Calling
+   json_istream_read() will then read the next JSON node from the input stream.
+ */
+void json_istream_skip(struct json_istream *stream);
+/* Ignore a number of JSON nodes at the current node level. If a node was
+   (partially) parsed before using json_istream_read() or
+   json_istream_read_object_member() it is skipped and counted as one ignored
+   node. The count may exceed the actual number of nodes left on this level
+   without consequence. If count == UINT_MAX, the rest of the current node level
+   is ignored no matter how many nodes are left.
+ */
+void json_istream_ignore(struct json_istream *stream, unsigned int count);
+
+/* Read a JSON object member name from the stream. If an object member name
+   or a complete node is already read, this function returns the name
+   immediately. This function can be used to peek what the next object member
+   is. Returns 1 on success, 0 if more data is needed from the input
+   stream, or -1 upon error or EOF. The last member in an object has
+   name_r=NULL. */
+int json_istream_read_object_member(struct json_istream *stream,
+                                   const char **name_r);
+
+/* Equivalent to json_istream_read_next(), but descends into array and object
+   nodes. Subsequent calls to json_istream_read() will be performed on the
+   child nodes of the array/object. This way, the hierarchy of the JSON document
+   can be followed downwards.
+ */
+int json_istream_descend(struct json_istream *stream,
+                        struct json_node *node_r);
+/* Skips to the end of the current array/object and ascends to the parent
+   JSON node. Any JSON nodes left at the end of the list will be skipped.
+   The next node read will be the first one that follows the array/object
+   the stream ascended from.
+ */
+void json_istream_ascend(struct json_istream *stream);
+
+/* Skips to the end of the current array/object structure and thereby ascends to
+   the indicated JSON node level. Any JSON nodes left at the end of the
+   structure will be skipped. The next node read will be the first one that
+   follows the JSON structure from which the stream ascended.
+ */
+void json_istream_ascend_to(struct json_istream *stream,
+                           unsigned int node_level);
+
+/* Equivalent to json_istream_descend(), but ascends implicitly after
+   encountering the end of array/object. The end of each is reported once
+   (json_node_is_end(node) == TRUE), after which it ascends implicitly.
+   So, the subsequent call will return the next node after the array/object.
+   This way, the hierarchy of the JSON document can be followed depth-first.
+   Returns 1 on success, 0 if more data is needed from the input stream,
+   or -1 upon error or EOF. */
+int json_istream_walk(struct json_istream *stream,
+                     struct json_node *node_r);
+
+#endif

diff --git a/src/lib-json/json-parser.new.c b/src/lib-json/json-parser.new.c
index d983da93f4c3e9b3a0cb7c82902bdb45ca15dc2c..06fffd74c8b55dbcf76d0658c7774b076bd9a877 100644 (file)
--- a/src/lib-json/json-parser.new.c
+++ b/src/lib-json/json-parser.new.c
@@ -483,7 +483,10 @@ static int str_float_to_intmax(const char *str, intmax_t *num_r)
        } else {
                if (un > (uintmax_t)INTMAX_MAX + 1)
                        return -1;
-               *num_r = -(intmax_t)un;
+               if (un == (uintmax_t)INTMAX_MAX + 1)
+                       *num_r = -(intmax_t)(un - 1) - 1;
+               else
+                       *num_r = -(intmax_t)un;
        }
        return 0;
 }

diff --git a/src/lib-json/test-json-istream.c b/src/lib-json/test-json-istream.c
new file mode 100644 (file)
index 0000000..185d9e8
--- /dev/null
+++ b/src/lib-json/test-json-istream.c
@@ -0,0 +1,2983 @@
+/* Copyright (c) 2017-2023 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "istream-failure-at.h"
+#include "test-common.h"
+
+#include "json-istream.h"
+
+#include <unistd.h>
+
+static bool debug = FALSE;
+
+static void
+test_json_read_success_text(struct json_istream **_jinput, const char *text)
+{
+       const char *error;
+       int ret;
+
+       ret = json_istream_finish(_jinput, &error);
+       test_out_reason_quiet(text, ret > 0, error);
+
+       /* Don't leak in case of test failure with ret == 0 */
+       json_istream_destroy(_jinput);
+}
+
+static void
+test_json_read_failure_text(struct json_istream **_jinput, const char *text)
+{
+       const char *error;
+       int ret;
+
+       ret = json_istream_finish(_jinput, &error);
+       test_out_reason_quiet(text, ret < 0, error);
+
+       /* Don't leak in case of test failure with ret == 0 */
+       json_istream_destroy(_jinput);
+}
+
+static void test_json_read_success(struct json_istream **_jinput)
+{
+       test_json_read_success_text(_jinput, "read success");
+}
+
+static void test_json_read_failure(struct json_istream **_jinput)
+{
+       test_json_read_failure_text(_jinput, "read failure");
+}
+
+/*
+ * Test: read number values
+ */
+
+static void test_json_istream_read_number(void)
+{
+       static const struct {
+               const char *text;
+               intmax_t number;
+       } number_tests[] = {
+               {
+                       .text = "1234",
+                       .number = 1234,
+               },
+               {
+                       .text = "1234e3",
+                       .number = 1234000,
+               },
+               {
+                       .text = "1234e-3",
+                       .number = 1,
+               },
+               {
+                       .text = "1234e003",
+                       .number = 1234000,
+               },
+               {
+                       .text = "1234e-003",
+                       .number = 1,
+               },
+               {
+                       .text = "123.4",
+                       .number = 123,
+               },
+               {
+                       .text = "12.34",
+                       .number = 12,
+               },
+               {
+                       .text = "1.234",
+                       .number = 1,
+               },
+               {
+                       .text = "0.1234",
+                       .number = 0,
+               },
+               {
+                       .text = "987.6e01",
+                       .number = 9876,
+               },
+               {
+                       .text = "98.76e02",
+                       .number = 9876,
+               },
+               {
+                       .text = "9.876e03",
+                       .number = 9876,
+               },
+               {
+                       .text = "0.9876e04",
+                       .number = 9876,
+               },
+               {
+                       .text = "0.9876e03",
+                       .number = 987,
+               },
+               {
+                       .text = "0.9876e02",
+                       .number = 98,
+               },
+               {
+                       .text = "0.9876e01",
+                       .number = 9,
+               },
+               {
+                       .text = "",
+                       .number = INTMAX_MAX,
+               },
+               {
+                       .text = "",
+                       .number = INTMAX_MIN,
+               },
+               {
+                       .text = "e00",
+                       .number = INTMAX_MAX,
+               },
+               {
+                       .text = "e00",
+                       .number = INTMAX_MIN,
+               },
+               {
+                       .text = "00e-02",
+                       .number = INTMAX_MAX,
+               },
+               {
+                       .text = "00e-02",
+                       .number = INTMAX_MIN,
+               },
+               {
+                       .text = "0000000000e-10",
+                       .number = INTMAX_MAX,
+               },
+               {
+                       .text = "0000000000e-10",
+                       .number = INTMAX_MIN,
+               },
+               {
+                       .text = ".999999999",
+                       .number = INTMAX_MAX,
+               },
+               {
+                       .text = ".999999999",
+                       .number = INTMAX_MIN,
+               },
+               {
+                       .text = "123e-10000000",
+                       .number = 0,
+               },
+               {
+                       .text = "2342e-2",
+                       .number = 23,
+               },
+               {
+                       .text = "23423232e-6",
+                       .number = 23,
+               },
+               {
+                       .text = "234232327654e-10",
+                       .number = 23,
+               },
+               {
+                       .text = "2342323276546785432098e-20",
+                       .number = 23,
+               },
+               {
+                       .text = "23423232765467854320984567894323e-30",
+                       .number = 23,
+               },
+               {
+                       .text = "23423232765467854320984567894323"
+                               "000000000000000000000000000000e-60",
+                       .number = 23,
+               },
+               {
+                       .text = "0.23423232765467854320984567894323e2",
+                       .number = 23,
+               },
+               {
+                       .text = "0.0000234232327654678543209845678943e6",
+                       .number = 23,
+               },
+               {
+                       .text = "0.0000000023234232327654678543209845e10",
+                       .number = 23,
+               },
+               {
+                       .text = "0.0000000000000000002323423232765467e20",
+                       .number = 23,
+               },
+               {
+                       .text = "0.0000000000000000000000000000232342e30",
+                       .number = 23,
+               },
+               {
+                       .text = "0.000000000000000000000000000000"
+                               "000000000000000000000000000023e60",
+                       .number = 23,
+               },
+       };
+       static const unsigned int number_tests_count =
+               N_ELEMENTS(number_tests);
+       unsigned int i;
+
+       for (i = 0; i < number_tests_count; i++) {
+               const char *text;
+               unsigned int text_len;
+               struct json_node jnode;
+               struct istream *input;
+               struct json_istream *jinput;
+               const char *str_val;
+               intmax_t num_val = 0;
+               int ret = 0;
+
+               i_zero(&jnode);
+
+               if (number_tests[i].number == INTMAX_MAX) {
+                       text = t_strdup_printf("%"PRIdMAX"%s", INTMAX_MAX,
+                                              number_tests[i].text);
+               } else if (number_tests[i].number == INTMAX_MIN) {
+                       text = t_strdup_printf("%"PRIdMAX"%s", INTMAX_MIN,
+                                              number_tests[i].text);
+               } else {
+                       text = number_tests[i].text;
+               }
+               text_len = strlen(text);
+
+               test_begin(t_strdup_printf("json istream read number[%u]", i));
+
+               /* As string */
+               input = i_stream_create_from_data(text, text_len);
+               jinput = json_istream_create(
+                       input, 0, NULL, JSON_PARSER_FLAG_NUMBERS_AS_STRING);
+
+               ret = json_istream_read(jinput, &jnode);
+               test_assert(ret != 0);
+               test_assert(json_node_is_number(&jnode));
+               if (json_node_is_number(&jnode)) {
+                       str_val = json_node_get_str(&jnode);
+                       test_assert_strcmp(str_val, text);
+               }
+               json_istream_skip(jinput);
+               ret = json_istream_read(jinput, &jnode);
+               test_assert(ret != 0);
+               test_json_read_success_text(&jinput, "read str success");
+
+               json_istream_unref(&jinput);
+               i_stream_unref(&input);
+
+               /* As number */
+               input = i_stream_create_from_data(text, text_len);
+               jinput = json_istream_create(input, 0, NULL, 0);
+
+               ret = json_istream_read(jinput, &jnode);
+               test_assert(ret != 0);
+               test_assert(json_node_is_number(&jnode));
+               if (json_node_is_number(&jnode)) {
+                       test_assert(json_node_get_intmax(
+                                       &jnode, &num_val) == 0);
+                       test_assert(num_val == number_tests[i].number);
+               }
+               json_istream_skip(jinput);
+               ret = json_istream_read(jinput, &jnode);
+               test_assert(ret != 0);
+               test_json_read_success_text(&jinput, "read int success");
+
+               json_istream_unref(&jinput);
+               i_stream_unref(&input);
+
+               test_end();
+       }
+}
+
+/*
+ * Test: read string values
+ */
+
+static void test_json_istream_read_string(void)
+{
+       static const unsigned char data1[] = { 0x00 };
+       static const unsigned char data2[] =
+               { 'a', 'a', 'a', 0x00, 'b', 'b', 'b' };
+       static const struct {
+               const char *text;
+               const char *string;
+               const unsigned char *data;
+               size_t size;
+       } string_tests[] = {
+               {
+                       .text = "\"bla\"",
+                       .string = "bla",
+               },
+               {
+                       .text = "\"\\\"\\\\\\/\\r\\n\\t\"",
+                       .string = "\"\\/\r\n\t",
+               },
+               {
+                       .text = "\"\\b\\f\"",
+                       .string = "\x08\x0c",
+               },
+               {
+                       .text = "\"\0\"",
+               },
+               {
+                       .text = "\"\\\0\"",
+               },
+               {
+                       .text = "\"\\u0000\"",
+                       .data = data1,
+                       .size = sizeof(data1),
+               },
+               {
+                       .text = "\"aaa\\u0000bbb\"",
+                       .data = data2,
+                       .size = sizeof(data2),
+               },
+               {
+                       .text = "\"\\uD83D\\uDD4A\"",
+                       .string = "\xF0\x9F\x95\x8A",
+               },
+               {
+                       .text = "\"fly \\uD83D\\uDD4A fly\"",
+                       .string = "fly \xF0\x9F\x95\x8A fly",
+               },
+               {
+                       .text = "\"\\uD83D\\uDD4A\\uD83D\\uDD4A\"",
+                       .string = "\xF0\x9F\x95\x8A\xF0\x9F\x95\x8A",
+               },
+               {
+                       .text = "\"\\uD83D\\uDD4A\\uD83D\\uDD4A\"",
+                       .string = "\xF0\x9F\x95\x8A\xF0\x9F\x95\x8A",
+               },
+       };
+       static const unsigned int string_tests_count =
+               N_ELEMENTS(string_tests);
+       unsigned int i;
+
+       for (i = 0; i < string_tests_count; i++) {
+               const char *text;
+               unsigned int text_len;
+               struct json_node jnode;
+               struct istream *input;
+               struct json_istream *jinput;
+               const char *str_val;
+               const unsigned char *data_val;
+               size_t size_val;
+               int ret = 0;
+
+               i_zero(&jnode);
+
+               text = string_tests[i].text;
+               text_len = strlen(text);
+
+               test_begin(t_strdup_printf("json istream read string[%u]", i));
+
+               /* As C string */
+               input = i_stream_create_from_data(text, text_len);
+               jinput = json_istream_create(input, 0, NULL, 0);
+
+               ret = json_istream_read(jinput, &jnode);
+               if (string_tests[i].string == NULL) {
+                       test_assert(ret <  0);
+                       test_json_read_failure(&jinput);
+               } else {
+                       test_assert(ret != 0);
+                       test_assert(json_node_is_string(&jnode));
+                       if (json_node_is_string(&jnode)) {
+                               str_val = json_node_get_str(&jnode);
+                               test_assert_strcmp(str_val,
+                                                  string_tests[i].string);
+                       }
+                       json_istream_skip(jinput);
+                       ret = json_istream_read(jinput, &jnode);
+                       test_assert(ret != 0);
+                       test_json_read_success_text(
+                               &jinput, "read cstr success");
+               }
+
+               json_istream_unref(&jinput);
+               i_stream_unref(&input);
+
+               /* As DATA */
+               input = i_stream_create_from_data(text, text_len);
+               jinput = json_istream_create(
+                       input, 0, NULL, JSON_PARSER_FLAG_STRINGS_ALLOW_NUL);
+
+               ret = json_istream_read(jinput, &jnode);
+               if (string_tests[i].string != NULL) {
+                       test_assert(ret != 0);
+                       test_assert(json_node_is_string(&jnode));
+                       if (json_node_is_string(&jnode)) {
+                               str_val = json_node_get_str(&jnode);
+                               test_assert_strcmp(str_val,
+                                                  string_tests[i].string);
+                       }
+                       json_istream_skip(jinput);
+                       ret = json_istream_read(jinput, &jnode);
+                       test_assert(ret != 0);
+                       test_json_read_success_text(
+                               &jinput, "read str success");
+               } else if (string_tests[i].data != NULL) {
+                       test_assert(ret != 0);
+                       test_assert(json_node_is_string(&jnode));
+                       if (json_node_is_string(&jnode)) {
+                               data_val =
+                                       json_node_get_data(&jnode, &size_val);
+                               test_assert_ucmp(size_val, ==,
+                                                string_tests[i].size);
+                               test_assert(
+                                       memcmp(data_val, string_tests[i].data,
+                                              I_MIN(size_val,
+                                                    string_tests[i].size)) == 0);
+                       }
+                       json_istream_skip(jinput);
+                       ret = json_istream_read(jinput, &jnode);
+                       test_assert(ret != 0);
+                       test_json_read_success_text(
+                               &jinput, "read data success");
+
+               } else {
+                       test_assert(ret <  0);
+                       test_json_read_failure(&jinput);
+               }
+
+               json_istream_unref(&jinput);
+               i_stream_unref(&input);
+
+               test_end();
+       }
+}
+
+/*
+ * Test: read buffer
+ */
+
+static void test_json_istream_read_buffer(void)
+{
+       struct istream *input;
+       struct json_istream *jinput;
+       const char *text;
+       struct json_node jnode;
+       unsigned int text_len;
+       intmax_t num_val = 0;
+       int ret = 0;
+
+       i_zero(&jnode);
+
+       /* number */
+       text = "2234234";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - number");
+
+       ret = json_istream_read(jinput, &jnode);
+       test_assert(ret != 0);
+       test_assert(json_node_is_number(&jnode));
+       test_assert(json_node_get_intmax(&jnode, &num_val) == 0);
+       test_assert(num_val == 2234234);
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       test_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* string */
+       text = "\"text\"";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - string");
+
+       ret = json_istream_read(jinput, &jnode);
+       test_assert(ret != 0);
+       test_assert(json_node_is_string(&jnode));
+       test_assert(null_strcmp(json_node_get_str(&jnode), "text") == 0);
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       test_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array */
+       text = "[\"text\"]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - array");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_array(&jnode));
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object */
+       text = "{\"text\": 1}";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - object");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array descend */
+       text = "[\"text\"]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - array descend");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_array(&jnode));
+
+       ret = json_istream_descend(jinput, &jnode);
+       test_assert(ret > 0);
+       test_assert(json_node_is_array(&jnode));
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_string(&jnode));
+       test_assert(null_strcmp(json_node_get_str(&jnode),
+                               "text") == 0);
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object descend */
+       text = "{\"member\": 14234234}";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read buffer - object descend");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+
+       ret = json_istream_descend(jinput, &jnode);
+       test_assert(ret > 0);
+       test_assert(json_node_is_object(&jnode));
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_number(&jnode));
+       test_assert(json_node_get_intmax(&jnode, &num_val) == 0);
+       test_assert(num_val == 14234234);
+       test_assert(null_strcmp(jnode.name, "member") == 0);
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* type=array */
+       text = "[\"text\",1,[false,[true],null],[1],{\"a\":1}]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create_array(input, NULL, 0);
+
+       test_begin("json istream read buffer - type=array");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_string(&jnode));
+       test_assert(null_strcmp(json_node_get_str(&jnode), "text") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_number(&jnode));
+       test_assert(json_node_get_intmax(&jnode, &num_val) == 0);
+       test_assert(num_val == 1);
+       json_istream_skip(jinput);
+
+       ret = json_istream_descend(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_array(&jnode));
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_false(&jnode));
+       json_istream_skip(jinput);
+
+       ret = json_istream_descend(jinput, &jnode);
+       test_assert(ret > 0);
+       test_assert(json_node_is_array(&jnode));
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_true(&jnode));
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_null(&jnode));
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_array(&jnode));
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret < 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* type=object */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create_object(input, NULL, 0);
+
+       test_begin("json istream read buffer - type=object");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_string(&jnode));
+       test_assert(null_strcmp(json_node_get_str(&jnode), "text") == 0);
+       test_assert(null_strcmp(jnode.name, "a") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_number(&jnode));
+       test_assert(json_node_get_intmax(&jnode, &num_val) == 0);
+       test_assert(num_val == 1);
+       test_assert(null_strcmp(jnode.name, "b") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_descend(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+       test_assert(null_strcmp(jnode.name, "c") == 0);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_false(&jnode));
+       test_assert(null_strcmp(jnode.name, "d") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_descend(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+       test_assert(null_strcmp(jnode.name, "e") == 0);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_true(&jnode));
+       test_assert(null_strcmp(jnode.name, "f") == 0);
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_null(&jnode));
+       test_assert(null_strcmp(jnode.name, "g") == 0);
+       json_istream_ascend(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_array(&jnode));
+       test_assert(null_strcmp(jnode.name, "h") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret != 0);
+       test_assert(json_node_is_object(&jnode));
+       test_assert(null_strcmp(jnode.name, "i") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret < 0);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+}
+
+/*
+ * Test: read trickle
+ */
+
+static void test_json_istream_read_trickle(void)
+{
+       struct istream *input;
+       struct json_istream *jinput;
+       const char *text;
+       struct json_node jnode;
+       unsigned int pos, text_len, state;
+       intmax_t num_val = 0;
+       int ret = 0;
+
+       /* number */
+       text = "2234234";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - number");
+
+       pos = 0;
+       state = 0;
+       while (ret >= 0 && state <= 1) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               ret = json_istream_read(jinput, &jnode);
+               if (ret == 0)
+                       continue;
+               if (ret < 0)
+                       break;
+               test_assert(json_node_is_number(&jnode));
+               test_assert(json_node_get_intmax(&jnode, &num_val) == 0);
+               test_assert(num_val == 2234234);
+               state++;
+               json_istream_skip(jinput);
+       }
+       test_assert(state == 1);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* string */
+       text = "\"text\"";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - string");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 1) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               ret = json_istream_read(jinput, &jnode);
+               if (ret == 0)
+                       continue;
+               if (ret < 0)
+                       break;
+               test_assert(json_node_is_string(&jnode));
+               test_assert(null_strcmp(json_node_get_str(&jnode),
+                                       "text") == 0);
+               state++;
+               json_istream_skip(jinput);
+       }
+       test_assert(state == 1);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array */
+       text = "[\"text\"]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 1) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               ret = json_istream_read(jinput, &jnode);
+               if (ret == 0)
+                       continue;
+               if (ret < 0)
+                       break;
+               test_assert(json_node_is_array(&jnode));
+               state++;
+               json_istream_skip(jinput);
+       }
+       test_assert(state == 1);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object */
+       text = "{\"text\": 1}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - object");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 1) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               ret = json_istream_read(jinput, &jnode);
+               if (ret == 0)
+                       continue;
+               if (ret < 0)
+                       break;
+               test_assert(json_node_is_object(&jnode));
+               state++;
+               json_istream_skip(jinput);
+       }
+       test_assert(state == 1);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array descend */
+       text = "[\"text\"]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array descend");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 3) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 3);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object descend */
+       text = "{\"member\": 14234234}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - object descend");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 3) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 14234234);
+                       test_assert(null_strcmp(jnode.name, "member") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 3);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array descend one */
+       text = "[\"text\",1,false,true,null,[1],{\"a\":1}]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array descend one");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 9) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 9);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object descend one */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":false,"
+               "\"d\":true,\"e\":null,\"f\":[1],\"g\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array descend one");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 9) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       test_assert(null_strcmp(jnode.name, "b") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       test_assert(null_strcmp(jnode.name, "c") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       test_assert(null_strcmp(jnode.name, "d") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       test_assert(null_strcmp(jnode.name, "e") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       test_assert(null_strcmp(jnode.name, "f") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "g") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 9);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array descend deep */
+       text = "[\"text\",1,[false,[true],null],[1],{\"a\":1}]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array descend deep");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 11) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 10:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 11:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 11);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object descend deep */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array descend deep");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 11) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       test_assert(null_strcmp(jnode.name, "b") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "c") == 0);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       test_assert(null_strcmp(jnode.name, "d") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "e") == 0);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       test_assert(null_strcmp(jnode.name, "f") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       test_assert(null_strcmp(jnode.name, "g") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       test_assert(null_strcmp(jnode.name, "h") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 10:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "i") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 11:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 11);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* array ascend ignore */
+       text = "[\"text\",1,false,true,null,[1,true,false],{\"a\":[1,2,3]}]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - array ascend ignore");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 6) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 6);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* object ascend ignore */
+       text = "{\"a\":\"text\",\"b\":[\"bbb\",1],\"c\":false,"
+               "\"d\":true,\"e\":null,\"f\":[1,2,3,4],"
+               "\"g\":{\"a\":[1,false,null]}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream read trickle - object ascend ignore");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 6) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       test_assert(null_strcmp(jnode.name, "b") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       test_assert(null_strcmp(jnode.name, "c") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       test_assert(null_strcmp(jnode.name, "d") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 6);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* type=array */
+       text = "[\"text\",1,[false,[true],null],[1],{\"a\":1}]";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create_array(input, NULL, 0);
+
+       test_begin("json istream read trickle - type=array");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 9) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_descend(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_array(&jnode));
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 9);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* type=object */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create_object(input, NULL, 0);
+
+       test_begin("json istream read trickle - type=object");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 9) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       test_assert(null_strcmp(jnode.name, "b") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "c") == 0);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       test_assert(null_strcmp(jnode.name, "d") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_descend(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "e") == 0);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       test_assert(null_strcmp(jnode.name, "f") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       test_assert(null_strcmp(jnode.name, "g") == 0);
+                       json_istream_ascend(jinput);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       test_assert(null_strcmp(jnode.name, "h") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "i") == 0);
+                       json_istream_skip(jinput);
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 9);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* json istream walk - object descend deep */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream walk trickle - object descend deep");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 18) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_walk(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       test_assert(null_strcmp(jnode.name, "b") == 0);
+                       state++;
+                       break;
+               case 4:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "c") == 0);
+                       state++;
+                       break;
+               case 5:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_false(&jnode));
+                       test_assert(null_strcmp(jnode.name, "d") == 0);
+                       state++;
+                       break;
+               case 6:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "e") == 0);
+                       state++;
+                       break;
+               case 7:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_true(&jnode));
+                       test_assert(null_strcmp(jnode.name, "f") == 0);
+                       state++;
+                       break;
+               case 8:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object_end(&jnode));
+                       state++;
+                       break;
+               case 9:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_null(&jnode));
+                       test_assert(null_strcmp(jnode.name, "g") == 0);
+                       state++;
+                       break;
+               case 10:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object_end(&jnode));
+                       state++;
+                       break;
+               case 11:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array(&jnode));
+                       test_assert(null_strcmp(jnode.name, "h") == 0);
+                       state++;
+                       break;
+               case 12:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       state++;
+                       break;
+               case 13:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_array_end(&jnode));
+                       state++;
+                       break;
+               case 14:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       test_assert(null_strcmp(jnode.name, "i") == 0);
+                       state++;
+                       break;
+               case 15:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_number(&jnode));
+                       test_assert(json_node_get_intmax(
+                               &jnode, &num_val) == 0);
+                       test_assert(num_val == 1);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       state++;
+                       break;
+               case 16:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object_end(&jnode));
+                       state++;
+                       break;
+               case 17:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object_end(&jnode));
+                       state++;
+                       break;
+               case 18:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_assert(state == 18);
+       test_istream_set_size(input, text_len);
+       test_json_read_success(&jinput);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+}
+
+/*
+ * Test: finish
+ */
+
+static void test_json_istream_finish(void)
+{
+       struct istream *input;
+       struct json_istream *jinput;
+       const char *text, *error;
+       struct json_node jnode;
+       unsigned int pos, text_len, state;
+       int ret = 0;
+
+       /* json istream finish buffer */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create_object(input, NULL, 0);
+
+       test_begin("json istream finish buffer");
+
+       ret = json_istream_read(jinput, &jnode);
+       i_assert(ret >= 0);
+       test_assert(json_node_is_string(&jnode));
+       test_assert(null_strcmp(json_node_get_str(&jnode), "text") == 0);
+       test_assert(null_strcmp(jnode.name, "a") == 0);
+       json_istream_skip(jinput);
+
+       ret = json_istream_finish(&jinput, &error);
+       test_out_reason_quiet("read success", ret > 0,
+                             (ret == 0 ? "ret == 0" : error));
+
+       test_end();
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       /* json istream finish trickle */
+       text = "{\"a\":\"text\",\"b\":1,\"c\":{\"d\":false,"
+               "\"e\":{\"f\":true},\"g\":null},\"h\":[1],\"i\":{\"a\":1}}";
+       text_len = strlen(text);
+
+       input = test_istream_create_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream finish trickle");
+
+       ret = 0; pos = 0; state = 0;
+       while (ret >= 0 && state <= 3) {
+               if (pos <= text_len)
+                       pos++;
+               test_istream_set_size(input, pos);
+               switch (state) {
+               case 0:
+                       ret = json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 1:
+                       ret = json_istream_walk(jinput, &jnode);
+                       test_assert(ret > 0);
+                       test_assert(json_node_is_object(&jnode));
+                       state++;
+                       break;
+               case 2:
+                       ret = json_istream_walk(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       test_assert(json_node_is_string(&jnode));
+                       test_assert(null_strcmp(json_node_get_str(&jnode),
+                                               "text") == 0);
+                       test_assert(null_strcmp(jnode.name, "a") == 0);
+                       state++;
+                       break;
+               case 3:
+                       ret = json_istream_finish(&jinput, &error);
+                       if (ret == 0)
+                               continue;
+                       if (ret < 0)
+                               break;
+                       state++;
+                       break;
+               }
+       }
+       test_out_reason_quiet("read success", ret > 0,
+                             (error != NULL || jinput == NULL ? error :
+                              json_istream_get_error(jinput)));
+       test_assert(state == 4);
+
+       test_end();
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+}
+
+/*
+ * Test: tokens
+ */
+
+static const char test_json_tokens_input[] =
+       "{\n"
+       "\t\"key\"\t:\t\t\"string\","
+       " \"key2\"  :  1234,  \n"
+       "\"key3\":true,"
+       "\"key4\":false,"
+       "\"skip1\": \"jsifjaisfjiasji\","
+       "\"skip2\": { \"x\":{ \"y\":123}, \"z\":[5,[6],{\"k\":0},3]},"
+       "\"key5\":null,"
+       "\"key6\": {},"
+       "\"key7\": {"
+       "  \"sub1\":\"value\""
+       "},"
+       "\"key8\": {"
+       "  \"sub2\":-12.456,\n"
+       "  \"sub3\":12.456e9,\n"
+       "  \"sub4\":0.456e-789"
+       "},"
+       "\"key9\": \"foo\\\\\\\"\\b\\f\\n\\r\\t\\u0001\\u10ff\","
+       "\"key11\": [],"
+       "\"key12\": [ \"foo\" , 5.24,[true],{\"aobj\":[]}],"
+       "\"key13\": \"\\ud801\\udc37\","
+       "\"key14\": \"\xd8\xb3\xd9\x84\xd8\xa7\xd9\x85\","
+       "\"key15\": \"\\u10000\""
+       "}\n";
+
+static struct json_node test_json_tokens_output[] = {
+       {
+               .name = "key", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "string" } },
+       }, {
+               .name = "key2", .type = JSON_TYPE_NUMBER,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "1234" } },
+       }, {
+               .name = "key3", .type = JSON_TYPE_TRUE,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key4", .type = JSON_TYPE_FALSE,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "skip1", .type = JSON_TYPE_NONE,
+       }, {
+               .name = "skip2", .type = JSON_TYPE_NONE,
+       }, {
+               .name = "key5", .type = JSON_TYPE_NULL,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key6", .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key7", .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .name = "sub1", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "value" } },
+       }, {
+               .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key8", .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .name = "sub2", .type = JSON_TYPE_NUMBER,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "-12.456" } },
+       }, {
+               .name = "sub3", .type = JSON_TYPE_NUMBER,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "12.456e9" } },
+       }, {
+               .name = "sub4", .type = JSON_TYPE_NUMBER,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "0.456e-789" } },
+       }, {
+               .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key9", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = {
+                               .str = "foo\\\"\b\f\n\r\t\001\xe1\x83\xbf" } },
+       }, {
+               .name = "key11", .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key12", .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = {
+                               .str = "foo" } },
+       }, {
+               .type = JSON_TYPE_NUMBER,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = {
+                               .str = "5.24" } },
+       }, {
+               .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .type = JSON_TYPE_TRUE,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .name = "aobj", .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_LIST },
+       }, {
+               .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .type = JSON_TYPE_OBJECT,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .type = JSON_TYPE_ARRAY,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_NONE },
+       }, {
+               .name = "key13", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "\xf0\x90\x90\xb7" } },
+       }, {
+               .name = "key14", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = {
+                               .str = "\xd8\xb3\xd9\x84\xd8\xa7\xd9\x85" } },
+       }, {
+               .name = "key15", .type = JSON_TYPE_STRING,
+               .value = {
+                       .content_type = JSON_CONTENT_TYPE_STRING,
+                       .content = { .str = "\xe1\x80\x80""0" } },
+       }
+};
+
+static void test_json_istream_tokens(bool full_size)
+{
+       struct json_istream *jinput;
+       struct istream *input;
+       struct json_node jnode;
+       const char *value;
+       unsigned int i, pos, json_input_len = strlen(test_json_tokens_input);
+       unsigned int ntokens = N_ELEMENTS(test_json_tokens_output);
+       int ret = 0;
+
+       input = test_istream_create_data(test_json_tokens_input,
+                                        json_input_len);
+       test_istream_set_allow_eof(input, FALSE);
+       jinput = json_istream_create_object(
+               input, NULL, JSON_PARSER_FLAG_NUMBERS_AS_STRING);
+
+       i = full_size ? json_input_len : 0;
+       for (pos = 0; i <= json_input_len; i++) {
+               test_istream_set_size(input, i);
+
+               for (;;) {
+                       const struct json_node *test_output =
+                               &test_json_tokens_output[pos];
+
+                       value = NULL;
+                       if (pos < ntokens &&
+                           test_output->type == JSON_TYPE_NONE) {
+                               json_istream_ignore(jinput, 1);
+                               pos++;
+                               continue;
+                       } else {
+                               ret = json_istream_walk(jinput, &jnode);
+                               if (ret > 0 &&
+                                   test_output->value.content_type ==
+                                       JSON_CONTENT_TYPE_STRING)
+                                       value = jnode.value.content.str;
+                       }
+                       if (ret <= 0)
+                               break;
+
+                       i_assert(pos < ntokens);
+                       test_assert_idx(test_output->type == jnode.type, pos);
+                       test_assert_idx(test_output->value.content_type ==
+                                       jnode.value.content_type, pos);
+                       test_assert_idx(
+                               null_strcmp(test_output->name,
+                                           jnode.name) == 0, pos);
+                       test_assert_idx(
+                               test_output->value.content_type !=
+                                       JSON_CONTENT_TYPE_STRING ||
+                               null_strcmp(test_output->value.content.str,
+                                           value) == 0, pos);
+
+                       pos++;
+               }
+               test_assert_idx(ret == 0, pos);
+       }
+       test_assert(pos == N_ELEMENTS(test_json_tokens_output));
+       test_istream_set_allow_eof(input, TRUE);
+       ret = json_istream_read_next(jinput, &jnode);
+       test_assert(ret < 0);
+       test_json_read_success(&jinput);
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+}
+
+static void test_json_istream_tokens_buffer(void)
+{
+       test_begin("json istream tokens (buffer)");
+       test_json_istream_tokens(TRUE);
+       test_end();
+}
+
+static void test_json_istream_tokens_trickle(void)
+{
+       test_begin("json istream tokens (trickle)");
+       test_json_istream_tokens(FALSE);
+       test_end();
+}
+
+/*
+ * Test: skip array
+ */
+
+static void test_json_istream_skip_array(void)
+{
+       static const char *test_input =
+               "[ 1, {\"foo\": 1 }, 2, \"bar\", 3, "
+               "1.234, 4, [], 5, [[]], 6, true ]";
+       struct istream *input;
+       struct json_istream *jinput;
+       struct json_node jnode;
+       intmax_t num;
+       int ret, i;
+
+       test_begin("json istream skip array");
+
+       input = test_istream_create_data(test_input, strlen(test_input));
+       jinput = json_istream_create_array(input, NULL, 0);
+       for (i = 1; i <= 6; i++) {
+               ret = json_istream_read_next(jinput, &jnode);
+               test_assert(ret > 0 && jnode.type == JSON_TYPE_NUMBER &&
+                           json_node_get_intmax(&jnode, &num) == 0 &&
+                           num == i);
+               json_istream_ignore(jinput, 1);
+       }
+
+       ret = json_istream_read_next(jinput, &jnode);
+       test_assert(ret < 0);
+       test_assert(json_istream_is_at_end(jinput));
+       test_json_read_success(&jinput);
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       test_end();
+}
+
+/*
+ * Test: skip object fields
+ */
+
+static void test_json_istream_skip_object_fields(void)
+{
+       static const char *test_input =
+               "{\"access_token\":\"9a2dea3c-f8be-4271-b9c8-5b37da4f2f7e\","
+                "\"grant_type\":\"authorization_code\","
+                "\"openid\":\"\","
+                "\"scope\":[\"openid\",\"profile\",\"email\"],"
+                "\"profile\":\"\","
+                "\"realm\":\"/employees\","
+                "\"token_type\":\"Bearer\","
+                "\"expires_in\":2377,"
+                "\"client_id\":\"mosaic\","
+                "\"email\":\"\","
+                "\"extensions\":"
+                "{\"algorithm\":\"cuttlefish\","
+                 "\"tentacles\":8"
+                "}"
+               "}";
+       static const char *const keys[] = {
+               "access_token", "grant_type", "openid", "scope", "profile",
+               "realm", "token_type", "expires_in", "client_id", "email",
+               "extensions"
+       };
+       static const unsigned int keys_count = N_ELEMENTS(keys);
+       struct istream *input;
+       struct json_istream *jinput;
+       struct json_node jnode;
+       const char *key;
+       unsigned int i;
+       size_t pos;
+       int ret;
+
+       test_begin("json istream skip object fields (by key)");
+
+       input = test_istream_create_data(test_input, strlen(test_input));
+       jinput = json_istream_create_object(input, NULL, 0);
+       for (i = 0; i < keys_count; i++) {
+               ret =  json_istream_read_object_member(jinput, &key);
+               if (ret < 0)
+                       break;
+               test_assert(ret > 0);
+               test_assert(strcmp(key, keys[i]) == 0);
+               json_istream_skip(jinput);
+       }
+       ret =  json_istream_read_object_member(jinput, &key);
+       test_assert(ret < 0);
+       test_assert(json_istream_is_at_end(jinput));
+       test_json_read_success(&jinput);
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       i = 0;
+       input = test_istream_create_data(test_input, strlen(test_input));
+       jinput = json_istream_create_object(input, NULL, 0);
+       for (pos = 0; pos <= strlen(test_input); pos +=2) {
+               test_istream_set_size(input, pos);
+               ret =  json_istream_read_object_member(jinput, &key);
+               if (ret == 0)
+                       continue;
+               if (ret < 0)
+                       break;
+               i_assert(i < keys_count);
+               test_assert(strcmp(key, keys[i]) == 0);
+               json_istream_skip(jinput);
+               i++;
+       }
+       ret =  json_istream_read_object_member(jinput, &key);
+       test_assert(ret < 0);
+       test_assert(json_istream_is_at_end(jinput));
+       test_json_read_success(&jinput);
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       test_end();
+
+       test_begin("json istream skip object fields (by value type)");
+
+       input = test_istream_create_data(test_input, strlen(test_input));
+       jinput = json_istream_create_object(input, NULL, 0);
+       for (i = 0; i < keys_count; i++) {
+               ret =  json_istream_read_object_member(jinput, &key);
+               if (ret < 0)
+                       break;
+               test_assert(ret > 0);
+               test_assert(strcmp(key, keys[i]) == 0);
+               ret =  json_istream_read(jinput, &jnode);
+               test_assert(ret > 0);
+               test_assert(strcmp(jnode.name, keys[i]) == 0);
+               json_istream_skip(jinput);
+       }
+       ret =  json_istream_read_object_member(jinput, &key);
+       test_assert(ret < 0);
+       test_assert(json_istream_is_at_end(jinput));
+       test_json_read_success(&jinput);
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       i = 0;
+       input = test_istream_create_data(test_input, strlen(test_input));
+       jinput = json_istream_create_object(input, NULL, 0);
+       for (;;) {
+               for (pos = 0; pos <= strlen(test_input); pos +=2) {
+                       test_istream_set_size(input, pos);
+                       ret =  json_istream_read_object_member(jinput, &key);
+                       if (ret == 0)
+                               continue;
+                       if (ret > 0) {
+                               i_assert(i < keys_count);
+                               test_assert(strcmp(key, keys[i]) == 0);
+                       }
+                       break;
+               }
+               if (ret < 0)
+                       break;
+               for (pos = 0; pos <= strlen(test_input); pos +=2) {
+                       test_istream_set_size(input, pos);
+                       ret =  json_istream_read(jinput, &jnode);
+                       if (ret == 0)
+                               continue;
+                       if (ret > 0) {
+                               i_assert(i < keys_count);
+                               test_assert(strcmp(jnode.name, keys[i]) == 0);
+                               i++;
+                       }
+                       break;
+               }
+               if (ret < 0)
+                       break;
+               json_istream_skip(jinput);
+       }
+       ret =  json_istream_read_object_member(jinput, &key);
+       test_assert(ret < 0);
+       test_assert(json_istream_is_at_end(jinput));
+       test_json_read_success(&jinput);
+
+       json_istream_destroy(&jinput);
+       i_stream_unref(&input);
+
+       test_end();
+}
+
+/*
+ * Test: error
+ */
+
+static void test_json_istream_error(void)
+{
+       struct istream *input, *err_input;
+       struct json_istream *jinput;
+       const char *text, *error;
+       struct json_node jnode;
+       unsigned int text_len;
+       int ret = 0;
+
+       /* stream error */
+       text = "[\"array\"]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       err_input = i_stream_create_failure_at(input, 5, EIO, "IO Error");
+       i_stream_unref(&input);
+       jinput = json_istream_create(err_input, 0, NULL, 0);
+       i_stream_unref(&err_input);
+
+       test_begin("json istream error - stream error");
+
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_descend(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_read(jinput, &jnode);
+       ret = json_istream_read(jinput, &jnode);
+       error = json_istream_get_error(jinput);
+       test_out_reason("read failure", (ret < 0 && error != NULL), error);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+
+       /* parse error */
+       text = "[\"unclosed array\"";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream error - parse error");
+
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_descend(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_string(&jnode));
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       error = json_istream_get_error(jinput);
+       test_out_reason("read failure", (ret < 0 && error != NULL), error);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* spurious data at end of input */
+       text = "[\"data\"][\"junk\"]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create(input, 0, NULL, 0);
+
+       test_begin("json istream error - spurious data at end of input");
+
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_descend(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array(&jnode));
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_string(&jnode));
+       json_istream_skip(jinput);
+       ret = json_istream_read(jinput, &jnode);
+       test_out_reason_quiet("read success", ret > 0,
+                             json_istream_get_error(jinput));
+       test_assert(json_node_is_array_end(&jnode));
+       json_istream_ascend(jinput);
+       ret = json_istream_finish(&jinput, &error);
+       test_out_reason("finish failure", (ret < 0 && error != NULL), error);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* root not array */
+       text = "\"string\"";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create_array(input, NULL, 0);
+
+       test_begin("json istream error - root not array");
+
+       ret = json_istream_read(jinput, &jnode);
+       error = json_istream_get_error(jinput);
+       test_out_reason("read failure", (ret < 0 && error != NULL), error);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+
+       /* root not object */
+       text = "[\"string\"]";
+       text_len = strlen(text);
+
+       input = i_stream_create_from_data(text, text_len);
+       jinput = json_istream_create_object(input, NULL, 0);
+
+       test_begin("json istream error - root not object");
+
+       ret = json_istream_read(jinput, &jnode);
+       error = json_istream_get_error(jinput);
+       test_out_reason("read failure", (ret < 0 && error != NULL), error);
+
+       test_end();
+
+       json_istream_unref(&jinput);
+       i_stream_unref(&input);
+}
+
+/*
+ * Main
+ */
+
+int main(int argc, char *argv[])
+{
+       int c;
+
+       static void (*test_functions[])(void) = {
+               test_json_istream_read_number,
+               test_json_istream_read_string,
+               test_json_istream_read_buffer,
+               test_json_istream_read_trickle,
+               test_json_istream_finish,
+               test_json_istream_tokens_buffer,
+               test_json_istream_tokens_trickle,
+               test_json_istream_skip_array,
+               test_json_istream_skip_object_fields,
+               test_json_istream_error,
+               NULL
+       };
+
+       while ((c = getopt(argc, argv, "D")) > 0) {
+               switch (c) {
+               case 'D':
+                       debug = TRUE;
+                       break;
+               default:
+                       i_fatal("Usage: %s [-D]", argv[0]);
+               }
+       }
+
+       return test_run(test_functions);
+}