From: Stephan Bosch <stephan.bosch@open-xchange.com>
Date: Wed, 7 Aug 2019 21:16:25 +0000 (+0200)
Subject: lib-json: json-parser - Add support for parsing string values as an input stream
X-Git-Tag: 2.4.0~2391
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5f04d3b0958f6be1f3e40ac6f9fb630f927b92fe;p=thirdparty%2Fdovecot%2Fcore.git

lib-json: json-parser - Add support for parsing string values as an input stream
---

diff --git a/src/lib-json/json-parser.new.c b/src/lib-json/json-parser.new.c
index 06fffd74c8..f0f734f937 100644
--- a/src/lib-json/json-parser.new.c
+++ b/src/lib-json/json-parser.new.c
@@ -171,11 +171,16 @@ struct json_parser {
 	string_t *object_member;
 	struct json_data content_data;
 
+	struct json_string_istream *str_stream;
+	size_t str_stream_threshold;
+	size_t str_stream_max_buffer_size;
+
 	char *error;
 
 	bool parsed_nul_char:1;
 	bool parsed_control_char:1;
 	bool parsed_float:1;
+	bool streaming_string:1;
 	bool callback_interrupted:1;
 	bool callback_running:1;
 	bool finished_level:1;
@@ -184,6 +189,9 @@ struct json_parser {
 	bool have_object_member:1;
 };
 
+static struct istream *
+json_string_stream_create(struct json_parser *parser, bool complete);
+
 static inline bool json_parser_is_busy(struct json_parser *parser)
 {
 	return (parser->level_stack_pos > 0 || parser->cur < parser->end);
@@ -521,9 +529,33 @@ json_parser_callback_string_value(struct json_parser *parser,
 				  void *list_context)
 {
 	struct json_value value;
+	int ret;
+
+	if (parser->str_stream != NULL)
+		return JSON_PARSE_BOUNDARY;
+	if (parser->streaming_string) {
+		parser->streaming_string = FALSE;
+		return JSON_PARSE_OK;
+	}
 
 	i_zero(&value);
 
+	if (parser->str_stream_max_buffer_size > 0) {
+		if (str_len(parser->buffer) >= parser->str_stream_threshold) {
+			value.content_type = JSON_CONTENT_TYPE_STREAM;
+			value.content.stream =
+				json_string_stream_create(parser, TRUE);
+			ret = json_parser_callback_parse_value(
+				parser, list_context, JSON_TYPE_STRING,
+				&value);
+			i_stream_unref(&value.content.stream);
+			parser->streaming_string = TRUE;
+			if (ret < JSON_PARSE_OK)
+				return ret;
+			return JSON_PARSE_INTERRUPTED;
+		}
+	}
+
 	if (parser->parsed_nul_char ||
 	    (parser->flags & JSON_PARSER_FLAG_STRINGS_AS_DATA) != 0) {
 		struct json_data *data = &parser->content_data;
@@ -545,9 +577,30 @@ json_parser_callback_string_value(struct json_parser *parser,
 						JSON_TYPE_STRING, &value);
 }
 
+static int
+json_parser_callback_string_stream(struct json_parser *parser,
+				   void *list_context)
+{
+	struct json_value value;
+	int ret;
+
+	if (parser->streaming_string)
+		return JSON_PARSE_OK;
+	parser->streaming_string = TRUE;
+
+	i_zero(&value);
+	value.content_type = JSON_CONTENT_TYPE_STREAM;
+	value.content.stream = json_string_stream_create(parser, FALSE);
+
+	ret = json_parser_callback_parse_value(parser, list_context,
+					       JSON_TYPE_STRING, &value);
+	i_stream_unref(&value.content.stream);
+	return ret;
+}
+
 static int
 json_parser_callback_true_value(struct json_parser *parser,
-				void *list_context)
+				 void *list_context)
 {
 	struct json_value value;
 
@@ -1566,7 +1619,20 @@ static int
 json_parser_do_parse_string_value(struct json_parser *parser,
 				  struct json_parser_state *state)
 {
-	size_t max_size = parser->limits.max_string_size;
+	size_t max_size;
+
+	if (parser->str_stream_max_buffer_size > 0)
+		max_size = parser->str_stream_max_buffer_size;
+	else
+		max_size = parser->limits.max_string_size;
+
+	if (parser->str_stream == NULL &&
+	    parser->str_stream_max_buffer_size > 0 &&
+	    max_size > parser->str_stream_threshold) {
+		/* Return string stream immediately once the treshold is
+		   crossed */
+		max_size = parser->str_stream_threshold;
+	}
 
 	return json_parser_do_parse_string(parser, state, max_size);
 }
@@ -1904,10 +1970,17 @@ json_parser_do_parse_value(struct json_parser *parser,
 			if (ret < JSON_PARSE_OK) {
 				if (ret != JSON_PARSE_OVERFLOW)
 					return ret;
-				json_parser_error(parser,
-					"Excessive string size (> %zu)",
-					parser->limits.max_string_size);
-				return JSON_PARSE_ERROR;
+				if (parser->str_stream_max_buffer_size == 0) {
+					json_parser_error(parser,
+						"Excessive string size (> %zu)",
+						parser->limits.max_string_size);
+					return JSON_PARSE_ERROR;
+				}
+				ret = json_parser_callback_string_stream(
+					parser,	parent_context);
+				if (ret < JSON_PARSE_OK)
+					return ret;
+				return JSON_PARSE_INTERRUPTED;
 			}
 			state->state = _VALUE_WS;
 			ret = json_parser_callback_string_value(
@@ -1953,6 +2026,7 @@ json_parser_do_parse_value(struct json_parser *parser,
 			ret = json_parser_skip_ws(parser);
 			if (ret < JSON_PARSE_OK)
 				return ret;
+			parser->streaming_string = FALSE;
 			state->state = _VALUE_END;
 			return JSON_PARSE_OK;
 		default:
@@ -2171,6 +2245,8 @@ int json_parse_more(struct json_parser *parser, const char **error_r)
 {
 	int ret;
 
+	i_assert(parser->str_stream == NULL);
+
 	*error_r = NULL;
 
 	ret = json_parser_continue(parser);
@@ -2206,3 +2282,169 @@ void json_parser_get_location(struct json_parser *parser,
 	loc_r->value_line = parser->loc.value_line_number;
 	loc_r->column = parser->loc.column;
 }
+
+/*
+ *
+ */
+
+struct json_string_istream {
+	struct istream_private istream;
+
+	struct json_parser *parser;
+
+	bool buffer_overflowed:1;
+	bool ended:1;
+};
+
+static ssize_t json_string_istream_read(struct istream_private *stream)
+{
+	struct json_string_istream *jstream =
+		(struct json_string_istream *)stream;
+	struct json_parser *parser = jstream->parser;
+	bool read_data;
+	size_t old_pos;
+	int ret;
+
+	if (jstream->ended) {
+		stream->istream.eof = TRUE;
+		return -1;
+	}
+	i_assert(jstream->parser != NULL);
+
+	i_assert(stream->pos == str_len(parser->buffer));
+	i_assert(stream->skip <= stream->pos);
+
+	read_data = FALSE;
+	do {
+		if (jstream->buffer_overflowed) {
+			stream->pos = str_len(parser->buffer);
+			if (stream->skip == stream->pos)
+				str_truncate(parser->buffer, 0);
+			else if (stream->skip > 0)
+				str_delete(parser->buffer, 0, stream->skip);
+			else
+				return -2;
+			stream->skip = 0;
+			jstream->buffer_overflowed = FALSE;
+		}
+
+		old_pos = str_len(parser->buffer);
+		ret = json_parser_continue(parser);
+		i_assert(str_len(parser->buffer) >= old_pos);
+		read_data = str_len(parser->buffer) > old_pos;
+		switch (ret) {
+		case JSON_PARSE_INTERRUPTED:
+			i_assert(stream->skip == 0 ||
+				 !jstream->buffer_overflowed);
+			jstream->buffer_overflowed = TRUE;
+			break;
+		case JSON_PARSE_BOUNDARY:
+			jstream->ended = TRUE;
+			read_data = TRUE;
+			if (str_len(parser->buffer) == old_pos) {
+				stream->istream.eof = TRUE;
+				return -1;
+			}
+			break;
+		case JSON_PARSE_NO_DATA:
+			stream->buffer = str_data(parser->buffer);
+			stream->pos = str_len(parser->buffer);
+			return 0;
+		case JSON_PARSE_ERROR:
+			io_stream_set_error(&stream->iostream,
+					    "%s", parser->error);
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		case JSON_PARSE_UNEXPECTED_EOF:
+			io_stream_set_error(&stream->iostream,
+					    "%s", parser->error);
+			stream->istream.stream_errno = EPIPE;
+			return -1;
+		default:
+			i_unreached();
+		}
+	} while (jstream->buffer_overflowed && !read_data);
+
+	stream->pos = str_len(parser->buffer);
+	stream->buffer = str_data(parser->buffer);
+	return (ssize_t)(stream->pos - old_pos);
+}
+
+static void
+json_string_istream_set_max_buffer_size(struct iostream_private *stream,
+					size_t max_size)
+{
+	struct json_string_istream *jstream =
+		(struct json_string_istream *)stream;
+
+	i_assert(max_size > 0);
+	jstream->parser->str_stream_max_buffer_size = max_size;
+}
+
+static void
+json_string_istream_close(struct iostream_private *stream,
+			  bool close_parent ATTR_UNUSED)
+{
+	struct json_string_istream *jstream =
+		(struct json_string_istream *)stream;
+
+	if (jstream->parser != NULL)
+		jstream->parser->str_stream = NULL;
+}
+
+static struct istream *
+json_string_stream_create(struct json_parser *parser, bool complete)
+{
+	struct json_string_istream *jstream;
+	struct istream *stream;
+	const char *name;
+
+	i_assert(parser->str_stream == NULL);
+
+	jstream = i_new(struct json_string_istream, 1);
+	jstream->parser = parser;
+
+	jstream->ended = complete;
+	jstream->istream.pos = str_len(parser->buffer);
+	jstream->istream.buffer = str_data(parser->buffer);
+
+	jstream->istream.max_buffer_size = parser->str_stream_max_buffer_size;
+	jstream->istream.iostream.set_max_buffer_size =
+		json_string_istream_set_max_buffer_size;
+	jstream->istream.iostream.close =
+		json_string_istream_close;
+	jstream->istream.read = json_string_istream_read;
+
+	jstream->istream.istream.readable_fd = FALSE;
+	jstream->istream.istream.blocking = parser->input->blocking;
+	jstream->istream.istream.seekable = FALSE;
+
+	parser->str_stream = jstream;
+
+	stream = i_stream_create(&jstream->istream, NULL,
+				 i_stream_get_fd(parser->input), 0);
+
+	name = i_stream_get_name(parser->input);
+	if (name == NULL || *name == '\0') {
+		i_stream_set_name(stream, "(JSON string)");
+	} else {
+		i_stream_set_name(stream, t_strdup_printf(
+			"(JSON string parsed from %s)", name));
+	}
+	return stream;
+}
+
+void json_parser_enable_string_stream(struct json_parser *parser,
+				      size_t threshold, size_t max_buffer_size)
+{
+	i_assert(max_buffer_size > 0);
+	if (threshold > max_buffer_size)
+		threshold = max_buffer_size;
+	parser->str_stream_threshold = threshold;
+	parser->str_stream_max_buffer_size = max_buffer_size;
+}
+
+void json_parser_disable_string_stream(struct json_parser *parser)
+{
+	parser->str_stream_max_buffer_size = 0;
+}
diff --git a/src/lib-json/json-parser.new.h b/src/lib-json/json-parser.new.h
index 2079ece9c1..0b5a15c3bc 100644
--- a/src/lib-json/json-parser.new.h
+++ b/src/lib-json/json-parser.new.h
@@ -103,4 +103,12 @@ int json_parse_more(struct json_parser *parser, const char **error_r);
 void json_parser_get_location(struct json_parser *parser,
 			      struct json_parser_location *loc_r);
 
+/* Enable parsing strings as a stream if the length of the string equals or
+   exceeds `threshold' octets. This disables the normal string size limit. The
+   stream will buffer at most `max_buffer_size' bytes. */
+void json_parser_enable_string_stream(struct json_parser *parser,
+				      size_t threshold, size_t max_buffer_size);
+/* Disable parsing strings as a stream. */
+void json_parser_disable_string_stream(struct json_parser *parser);
+
 #endif
diff --git a/src/lib-json/json-types.c b/src/lib-json/json-types.c
index 9a2f37a99c..f7855a6f4d 100644
--- a/src/lib-json/json-types.c
+++ b/src/lib-json/json-types.c
@@ -22,6 +22,7 @@ static const char *json_content_type_names[] = {
 	[JSON_CONTENT_TYPE_LIST] = "<LIST>",
 	[JSON_CONTENT_TYPE_STRING] = "<STRING>",
 	[JSON_CONTENT_TYPE_DATA] = "<DATA>",
+	[JSON_CONTENT_TYPE_STREAM] = "<STREAM>",
 	[JSON_CONTENT_TYPE_INTEGER] = "<INTEGER>",
 };
 static_assert_array_size(json_content_type_names,
diff --git a/src/lib-json/json-types.h b/src/lib-json/json-types.h
index 42d28b4b8d..e3d0efdb5b 100644
--- a/src/lib-json/json-types.h
+++ b/src/lib-json/json-types.h
@@ -1,6 +1,8 @@
 #ifndef JSON_TYPES_H
 #define JSON_TYPES_H
 
+#include "istream.h"
+
 struct json_data;
 struct json_value;
 struct json_node;
@@ -41,6 +43,8 @@ enum json_content_type {
 	JSON_CONTENT_TYPE_STRING,
 	/* data buffer (for string/text containing \u0000) */
 	JSON_CONTENT_TYPE_DATA,
+	/* data stream (for potentially very long string/text) */
+	JSON_CONTENT_TYPE_STREAM,
 	/* integer number */
 	JSON_CONTENT_TYPE_INTEGER,
 };
@@ -63,6 +67,8 @@ struct json_value {
 		const char *str;
 		/* JSON_CONTENT_TYPE_DATA */
 		struct json_data *data;
+		/* JSON_CONTENT_TYPE_STREAM */
+		struct istream *stream;
 		/* JSON_CONTENT_TYPE_INTEGER */
 		intmax_t intnum;
 	} content;
@@ -105,15 +111,15 @@ name(const struct json_value *jvalue, type *num_r)                      \
 }
 
 JSON_VALUE_GET_U__TEMPLATE(json_value_get_uint,
-			    unsigned int, UINT_MAX)
+			   unsigned int, UINT_MAX)
 JSON_VALUE_GET_U__TEMPLATE(json_value_get_ulong,
-			    unsigned long, ULONG_MAX)
+			   unsigned long, ULONG_MAX)
 JSON_VALUE_GET_U__TEMPLATE(json_value_get_ullong,
-			    unsigned long long, ULLONG_MAX)
+			   unsigned long long, ULLONG_MAX)
 JSON_VALUE_GET_U__TEMPLATE(json_value_get_uint32,
-			    uint32_t, UINT32_MAX)
+			   uint32_t, UINT32_MAX)
 JSON_VALUE_GET_U__TEMPLATE(json_value_get_uint64,
-			    uint64_t, UINT64_MAX)
+			   uint64_t, UINT64_MAX)
 
 #define JSON_VALUE_GET_S__TEMPLATE(name, type, int_min, int_max)        \
 static inline int                                                       \
@@ -129,15 +135,15 @@ name(const struct json_value *jvalue, type *num_r)                      \
 }
 
 JSON_VALUE_GET_S__TEMPLATE(json_value_get_int,
-			    int, INT_MIN, INT_MAX)
+			   int, INT_MIN, INT_MAX)
 JSON_VALUE_GET_S__TEMPLATE(json_value_get_long,
-			    long, LONG_MIN, LONG_MAX)
+			   long, LONG_MIN, LONG_MAX)
 JSON_VALUE_GET_S__TEMPLATE(json_value_get_llong,
-			    long long, LLONG_MIN, LLONG_MAX)
+			   long long, LLONG_MIN, LLONG_MAX)
 JSON_VALUE_GET_S__TEMPLATE(json_value_get_int32,
-			    int32_t, INT32_MIN, INT32_MAX)
+			   int32_t, INT32_MIN, INT32_MAX)
 JSON_VALUE_GET_S__TEMPLATE(json_value_get_int64,
-			    int64_t, INT64_MIN, INT64_MAX)
+			   int64_t, INT64_MIN, INT64_MAX)
 
 #define JSON_VALUE_GET_F__TEMPLATE(name, type)                          \
 static inline int                                                       \
@@ -187,6 +193,17 @@ json_value_get_data(const struct json_value *jvalue, size_t *size_r)
 	i_unreached();
 }
 
+static inline ATTR_PURE int
+json_value_get_stream(const struct json_value *jvalue,
+		       struct istream **stream_r)
+{
+	if (jvalue->content_type != JSON_CONTENT_TYPE_STREAM)
+		return -1;
+	*stream_r = jvalue->content.stream;
+	i_stream_ref(*stream_r);
+	return 0;
+}
+
 /*
  * Node
  */
@@ -326,6 +343,16 @@ json_node_get_data(const struct json_node *jnode, size_t *size_r)
 	return json_value_get_data(&jnode->value, size_r);
 }
 
+static inline int
+json_node_get_stream(const struct json_node *jnode,
+		      struct istream **stream_r)
+{
+	if (jnode->type != JSON_TYPE_STRING)
+		return -1;
+
+	return json_value_get_stream(&jnode->value, stream_r);
+}
+
 /* number */
 
 static inline ATTR_PURE bool
@@ -514,6 +541,10 @@ json_node_is_singular(const struct json_node *jnode)
  * Limits
  */
 
+// NOTE: There is currently no support for reading enormous object member names
+//       incrementally. This is usually not needed, but it could sometimes be
+//       nice to have.
+
 #define JSON_DEFAULT_MAX_NAME_SIZE 1024
 #define JSON_DEFAULT_MAX_STRING_SIZE 32*1024
 #define JSON_DEFAULT_MAX_NESTING 32
diff --git a/src/lib-json/test-json-parser.c b/src/lib-json/test-json-parser.c
index 8c5ffbf451..fdbd7ca3a4 100644
--- a/src/lib-json/test-json-parser.c
+++ b/src/lib-json/test-json-parser.c
@@ -2390,6 +2390,447 @@ static void test_json_parse_invalid(void)
 	} T_END;
 }
 
+/*
+ * Test: stream parse tests
+ */
+
+struct json_stream_parse_test {
+	const char *input, *output;
+	struct json_limits limits;
+	enum json_parser_flags flags;
+};
+
+static const struct json_stream_parse_test
+stream_parse_tests[] = {
+	{
+		.input = "\"AABBCC\"",
+		.output = "AABBCC"
+	},{
+		.input = "\""
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"\"",
+		.output =
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+	},{
+		.input = "[\""
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"\"]",
+		.output =
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+	},{
+		.input = "  [ \""
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"\" ]  ",
+		.output =
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+			"AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ"
+	},{
+		.input = "\"foo\\\\\\\"\\b\\f\\n\\r\\t\\u0001\\uffff\"",
+		.output = "foo\\\"\b\f\n\r\t\001\xEF\xBF\xBF"
+	},{
+		.input = "\"\\ud801\\udc37\"",
+		.output = "\xf0\x90\x90\xb7"
+	},{
+		.input = "\"\"",
+		.output = ""
+	}
+};
+
+static const unsigned int stream_parse_test_count =
+	N_ELEMENTS(stream_parse_tests);
+
+static void
+test_parse_stream_parse_value(void *context,
+	void *parent_context ATTR_UNUSED,
+	const char *name ATTR_UNUSED, enum json_type type,
+	const struct json_value *value)
+{
+	struct istream **str_stream_r = (struct istream **)context;
+
+	test_assert(type == JSON_TYPE_STRING);
+	test_assert(value->content_type == JSON_CONTENT_TYPE_STREAM);
+	*str_stream_r = value->content.stream;
+	i_stream_ref(value->content.stream);
+}
+
+static struct json_parser_callbacks parse_stream_callbacks = {
+	.parse_value = test_parse_stream_parse_value
+};
+
+static void test_json_parse_stream(void)
+{
+	static const unsigned int trickle_steps[] = {1,2,3,4,5,10,20};
+	string_t *buffer;
+	unsigned int i, j;
+
+	buffer = str_new(default_pool, 256);
+
+	for (i = 0; i < stream_parse_test_count; i++) T_BEGIN {
+		const struct json_stream_parse_test *test;
+		struct istream *input, *str_input;
+		struct json_parser *parser;
+		const char *text, *error = NULL;
+		unsigned int pos, text_len;
+		int ret = 0;
+
+		test = &stream_parse_tests[i];
+
+		text = test->input;
+		text_len = strlen(text);
+
+		input = test_istream_create_data(text, text_len);
+
+		test_begin(t_strdup_printf("json parse stream [%u]", i));
+
+		/* trickle tests */
+		for (j = 0; j < N_ELEMENTS(trickle_steps); j++) {
+			unsigned int trickle_step = trickle_steps[j];
+
+			i_stream_seek(input, 0);
+			str_input = NULL;
+			str_truncate(buffer, 0);
+
+			parser = json_parser_init(input,
+				NULL, 0, &parse_stream_callbacks, &str_input);
+			json_parser_enable_string_stream(parser, 0, 10);
+
+			ret = 0;
+			for (pos = 0; pos <= text_len+1000 && ret == 0; pos += trickle_step) {
+				test_istream_set_size(input, pos);
+				if (str_input == NULL) {
+					ret = json_parse_more(parser, &error);
+					if (ret < 0)
+						break;
+				}
+				if (str_input != NULL) {
+					const unsigned char *data;
+					size_t size;
+
+					while ((ret = i_stream_read_more(str_input,
+									 &data, &size)) > 0) {
+						buffer_append(buffer, data, size);
+						i_stream_skip(str_input, size);
+					}
+					if (ret < 0) {
+						i_assert(!i_stream_have_bytes_left(str_input));
+						i_stream_skip(str_input, size);
+						i_stream_unref(&str_input);
+						ret = 0;
+					}
+				}
+			}
+			test_out_reason_quiet(
+				t_strdup_printf("parse success "
+						"(trickle, step=%u)",
+						trickle_step),
+				ret > 0, error);
+			test_out_quiet("stream output",
+				       strcmp(str_c(buffer),
+					      test->output) == 0);
+			json_parser_deinit(&parser);
+		}
+
+		/* buffered test */
+		i_stream_seek(input, 0);
+		str_truncate(buffer, 0);
+
+		parser = json_parser_init(input,
+			NULL, 0, &parse_stream_callbacks, &str_input);
+		json_parser_enable_string_stream(parser, 0, 10);
+
+		test_istream_set_size(input, text_len);
+		ret = json_parse_more(parser, &error);
+		test_out_reason_quiet("parse success (buffered) #1",
+				      ret == 0, error);
+		if (ret == 0 && str_input != NULL) {
+			const unsigned char *data;
+			size_t size;
+
+			while ((ret = i_stream_read_more(str_input,
+							 &data, &size)) > 0) {
+				buffer_append(buffer, data, size);
+				i_stream_skip(str_input, size);
+			}
+			i_assert (ret != 0);
+			if (ret < 0) {
+				i_assert(!i_stream_have_bytes_left(str_input));
+				i_stream_skip(str_input, size);
+				i_stream_unref(&str_input);
+				ret = 0;
+			}
+		}
+		if (ret == 0) {
+			ret = json_parse_more(parser, &error);
+			test_out_reason_quiet("parse success (buffered) #2",
+					      ret > 0, error);
+		}
+		test_out_quiet("stream output",
+			       strcmp(str_c(buffer), test->output) == 0);
+		json_parser_deinit(&parser);
+
+		test_end();
+
+		i_stream_unref(&input);
+	} T_END;
+
+	str_free(&buffer);
+}
+
+/*
+ * Test: stream parse error tests
+ */
+
+struct json_stream_parse_error_test {
+	const char *input;
+	struct json_limits limits;
+	enum json_parser_flags flags;
+	int stream_errno;
+};
+
+static const struct json_stream_parse_error_test
+stream_parse_error_tests[] = {
+	/* invalid escape */
+	{
+		.input = "\"foo\\?\"",
+		.stream_errno = EINVAL,
+	/* just a DQUOTE */
+	},{
+		.input = "\"",
+		.stream_errno = EPIPE,
+	/* unterminated string, escaped DQUOTE */
+	},{
+		.input = "\"\\\"",
+		.stream_errno = EPIPE,
+	/* unterminated string */
+	},{
+		.input = "\"foo",
+		.stream_errno = EPIPE,
+	/* high surrogate alone, unterminated string */
+	},{
+		.input = "\"\\ud801",
+		.stream_errno = EPIPE,
+	/* high surrogate alone */
+	},{
+		.input = "\"\\ud801\"",
+		.stream_errno = EINVAL,
+	/* low surrogate before high */
+	},{
+		.input = "\"\\udced\\udc37\"",
+		.stream_errno = EINVAL,
+	/* has extra 1 in middle */
+	},{
+		.input = "\"\\ud8011\\udc37\"",
+		.stream_errno = EINVAL,
+	/* has extra TAB in middle */
+	},{
+		.input = "\"\\ud801\\t\\udc37\"",
+		.stream_errno = EINVAL,
+	/* low surrogate before high with valid prefix*/
+	},{
+		.input = "\"hello \\udc37\"",
+		.stream_errno = EINVAL,
+	/* high surrogate alone with valid prefix */
+	},{
+		.input = "\"hello \\ud801",
+		.stream_errno = EPIPE,
+	/* invalid hex value */
+	},{
+		.input = "\"\\uabcg",
+		.stream_errno = EINVAL,
+	/* invalid escape */
+	},{
+		.input = "\"\\xFF\\xFF\\xFF\"",
+		.stream_errno = EINVAL,
+	}
+};
+
+static const unsigned int stream_parse_error_test_count =
+	N_ELEMENTS(stream_parse_error_tests);
+
+static void
+test_parse_stream_parse_error_value(void *context,
+				    void *parent_context ATTR_UNUSED,
+				    const char *name ATTR_UNUSED,
+				    enum json_type type,
+				    const struct json_value *value)
+{
+	struct istream **str_stream_r = (struct istream **)context;
+
+	test_assert(type == JSON_TYPE_STRING);
+	test_assert(value->content_type == JSON_CONTENT_TYPE_STREAM);
+	*str_stream_r = value->content.stream;
+	i_stream_ref(value->content.stream);
+}
+
+static struct json_parser_callbacks parse_stream_error_callbacks = {
+	.parse_value = test_parse_stream_parse_error_value
+};
+
+static void test_json_parse_stream_error(void)
+{
+	static const unsigned int trickle_steps[] = {1,2,3,4,5,10,20};
+	string_t *buffer;
+	unsigned int i, j;
+
+	buffer = str_new(default_pool, 256);
+
+	for (i = 0; i < stream_parse_error_test_count; i++) T_BEGIN {
+		const struct json_stream_parse_error_test *test;
+		struct istream *input, *str_input;
+		struct json_parser *parser;
+		const char *text, *error = NULL;
+		unsigned int pos, text_len;
+		int ret = 0;
+
+		test = &stream_parse_error_tests[i];
+
+		text = test->input;
+		text_len = strlen(text);
+
+		input = test_istream_create_data(text, text_len);
+
+		test_begin(t_strdup_printf("json parse stream error [%u]", i));
+
+		/* trickle tests */
+		for (j = 0; j < N_ELEMENTS(trickle_steps); j++) {
+			unsigned int trickle_step = trickle_steps[j];
+
+			i_stream_seek(input, 0);
+			str_input = NULL;
+			str_truncate(buffer, 0);
+
+			parser = json_parser_init(input,
+				NULL, 0, &parse_stream_error_callbacks, &str_input);
+			json_parser_enable_string_stream(parser, 0, 10);
+
+			ret = 0;
+			for (pos = 0; pos <= text_len+1000 && ret == 0; pos += trickle_step) {
+				test_istream_set_size(input, pos);
+				if (str_input == NULL) {
+					ret = json_parse_more(parser, &error);
+					if (ret < 0)
+						break;
+				}
+				if (str_input != NULL) {
+					const unsigned char *data;
+					size_t size;
+
+					while ((ret = i_stream_read_more(str_input,
+									 &data, &size)) > 0) {
+						buffer_append(buffer, data, size);
+						i_stream_skip(str_input, size);
+					}
+					if (ret < 0) {
+						test_assert(str_input->stream_errno != 0);
+						test_out_quiet("stream errno",
+							       str_input->stream_errno == test->stream_errno);
+						i_stream_skip(str_input, size);
+						i_stream_unref(&str_input);
+						ret = 0;
+					}
+				}
+			}
+			test_out_reason_quiet(
+				t_strdup_printf("parse failure "
+						"(trickle, step=%u)",
+						trickle_step),
+				ret < 0, error);
+			json_parser_deinit(&parser);
+		}
+
+		/* buffered test */
+		i_stream_seek(input, 0);
+		str_truncate(buffer, 0);
+
+		parser = json_parser_init(input,
+			NULL, 0, &parse_stream_error_callbacks, &str_input);
+		json_parser_enable_string_stream(parser, 0, 10);
+
+		test_istream_set_size(input, text_len);
+		ret = json_parse_more(parser, &error);
+		test_out_reason_quiet("parse failure (buffered) #1",
+				      ret <= 0, error);
+		if (ret == 0 && str_input != NULL) {
+			const unsigned char *data;
+			size_t size;
+
+			while ((ret = i_stream_read_more(str_input,
+							 &data, &size)) > 0) {
+				buffer_append(buffer, data, size);
+				i_stream_skip(str_input, size);
+			}
+			i_assert (ret != 0);
+			if (ret < 0) {
+				test_assert(str_input->stream_errno != 0);
+				test_out_quiet("stream errno",
+					       str_input->stream_errno == test->stream_errno);
+				i_stream_skip(str_input, size);
+				i_stream_unref(&str_input);
+				ret = 0;
+			}
+		}
+		if (ret == 0) {
+			ret = json_parse_more(parser, &error);
+			test_out_reason_quiet("parse failure (buffered) #2",
+					      ret < 0, error);
+		}
+		json_parser_deinit(&parser);
+
+		test_end();
+
+		i_stream_unref(&input);
+	} T_END;
+
+	str_free(&buffer);
+}
+
 int main(int argc, char *argv[])
 {
 	int c;
@@ -2397,6 +2838,8 @@ int main(int argc, char *argv[])
 	static void (*test_functions[])(void) = {
 		test_json_parse_valid,
 		test_json_parse_invalid,
+		test_json_parse_stream,
+		test_json_parse_stream_error,
 		NULL
 	};