From: Stephan Bosch Date: Wed, 7 Aug 2019 21:34:27 +0000 (+0200) Subject: lib-json: json-generator - Add support for writing a JSON string from an input stream X-Git-Tag: 2.4.0~2390 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c1331bf2785777db4cfcc8f1cc8c87cd7a6952b5;p=thirdparty%2Fdovecot%2Fcore.git lib-json: json-generator - Add support for writing a JSON string from an input stream --- diff --git a/src/lib-json/json-generator.c b/src/lib-json/json-generator.c index 858bfb91a9..bc22a85210 100644 --- a/src/lib-json/json-generator.c +++ b/src/lib-json/json-generator.c @@ -4,6 +4,7 @@ #include "str.h" #include "array.h" #include "hex-dec.h" +#include "istream.h" #include "ostream-private.h" #include "json-syntax.h" @@ -49,13 +50,21 @@ struct json_generator { /* Write state: stack position of written syntax levels */ unsigned int level_stack_written; + /* Currently pending string input stream */ + struct istream *value_input; + /* We are in an object */ bool object_level_written:1; /* write state */ bool object_level:1; /* API state */ /* We closed an empty string */ bool string_empty:1; /* API state */ + /* We opened an input stream string */ + bool string_stream:1; /* API state */ + bool string_stream_written:1; /* write state */ }; +static int json_generator_flush_string_input(struct json_generator *generator); + static struct json_generator * json_generator_new(enum json_generator_flags flags) { @@ -100,6 +109,7 @@ void json_generator_deinit(struct json_generator **_generator) return; *_generator = NULL; + i_stream_unref(&generator->value_input); if (generator->output != NULL) { o_stream_unref(&generator->output); str_free(&generator->buf); @@ -361,6 +371,31 @@ int json_generator_flush(struct json_generator *generator) if (generator->state == JSON_GENERATOR_STATE_TEXT && generator->write_state != JSON_GENERATOR_STATE_TEXT) generator->write_state = JSON_GENERATOR_STATE_TEXT; + /* Flush string stream */ + if (generator->string_stream) { + i_assert(generator->value_input != NULL); + if (!generator->string_stream_written) { + ret = json_generator_write_all(generator, "\"", 1); + if (ret <= 0) + return ret; + generator->string_stream_written = TRUE; + } + /* Flush the stream */ + ret = json_generator_flush_string_input(generator); + if (ret <= 0) + return ret; + generator->string_stream = FALSE; + generator->string_stream_written = FALSE; + /* Close the string */ + ret = json_generator_write_all(generator, "\"", 1); + if (ret < 0) + return -1; + if (ret == 0) { + generator->write_state = JSON_GENERATOR_STATE_STRING; + return 0; + } + generator->write_state = JSON_GENERATOR_STATE_VALUE_END; + } return 1; } @@ -571,6 +606,7 @@ ssize_t json_generate_string_more(struct json_generator *generator, { int ret; + i_assert(generator->value_input == NULL); i_assert(generator->state == JSON_GENERATOR_STATE_STRING); ret = json_generator_flush(generator); if (ret <= 0) @@ -583,6 +619,7 @@ ssize_t json_generate_string_more(struct json_generator *generator, void json_generate_string_close(struct json_generator *generator) { + i_assert(generator->value_input == NULL); i_assert(generator->state == JSON_GENERATOR_STATE_STRING); if (generator->write_state != JSON_GENERATOR_STATE_STRING) { /* This function does not flush first before changing state, nor @@ -634,6 +671,57 @@ int json_generate_string(struct json_generator *generator, const char *str) strlen(str)); } +static int +json_generator_flush_string_input(struct json_generator *generator) +{ + const unsigned char *data; + size_t size; + ssize_t sret; + int ret; + + while ((ret = i_stream_read_more(generator->value_input, + &data, &size)) > 0) { + sret = json_generate_string_write_data( + generator, data, size, FALSE, + generator->value_input->eof); + if (sret < 0) + return -1; + if (sret == 0) + return 0; + i_stream_skip(generator->value_input, (size_t)sret); + } + if (ret < 0) { + if (generator->value_input->stream_errno != 0) + return -1; + + i_assert(!i_stream_have_bytes_left(generator->value_input)); + i_stream_unref(&generator->value_input); + return 1; + } + return 0; +} + +int json_generate_string_stream(struct json_generator *generator, + struct istream *input) +{ + i_assert(generator->value_input == NULL); + json_generator_value_begin(generator); + generator->value_input = input; + i_stream_ref(input); + generator->string_stream = TRUE; + if (generator->write_state == JSON_GENERATOR_STATE_VALUE_END) + generator->write_state = JSON_GENERATOR_STATE_VALUE_NEXT; + if (generator->level_stack_pos == 0) + generator->state = JSON_GENERATOR_STATE_END; + else if (generator->object_level) + generator->state = JSON_GENERATOR_STATE_OBJECT_MEMBER; + else + generator->state = JSON_GENERATOR_STATE_VALUE; + if (json_generator_flush(generator) < 0) + return -1; + return 1; +} + /* * null, true, false */ @@ -899,8 +987,7 @@ int json_generate_text(struct json_generator *generator, const char *str) */ int json_generate_value(struct json_generator *generator, - enum json_type type, - const struct json_value *value) + enum json_type type, const struct json_value *value) { switch (type) { /* string */ @@ -913,6 +1000,9 @@ int json_generate_value(struct json_generator *generator, return json_generate_string_data( generator, value->content.data->data, value->content.data->size); + case JSON_CONTENT_TYPE_STREAM: + return json_generate_string_stream( + generator, value->content.stream); default: break; } diff --git a/src/lib-json/json-generator.h b/src/lib-json/json-generator.h index 1670f99f50..b296bcfede 100644 --- a/src/lib-json/json-generator.h +++ b/src/lib-json/json-generator.h @@ -46,6 +46,9 @@ int json_generate_string_data(struct json_generator *generator, const void *data, size_t size); int json_generate_string(struct json_generator *generator, const char *str); +int json_generate_string_stream(struct json_generator *generator, + struct istream *input); + /* null */ int json_generate_null(struct json_generator *generator); diff --git a/src/lib-json/test-json-generator.c b/src/lib-json/test-json-generator.c index c3f33fdc77..8ebe91bc84 100644 --- a/src/lib-json/test-json-generator.c +++ b/src/lib-json/test-json-generator.c @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "istream.h" #include "ostream.h" #include "unichar.h" #include "test-common.h" @@ -16,6 +17,8 @@ static void test_json_generate_buffer(void) { string_t *buffer; struct ostream *output; + struct istream *input; + const char *data; struct json_generator *generator; unsigned int state, pos; ssize_t sret; @@ -231,6 +234,22 @@ static void test_json_generate_buffer(void) str_truncate(buffer, 0); output->offset = 0; + /* string - input stream */ + test_begin("json write string - input stream"); + generator = json_generator_init(output, 0); + data = "ABC\tDEF\nGHI\tJKL\nMNO\x19PQR\nSTU\tVWX\nYZ"; + input = i_stream_create_from_data(data, strlen(data)); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + json_generator_flush(generator); + test_assert(ret > 0); + json_generator_deinit(&generator); + test_assert(strcmp("\"ABC\\tDEF\\nGHI\\tJKL\\nMNO\\u0019PQR\\nSTU\\tVWX\\nYZ\"", str_c(buffer)) == 0); + test_end(); + str_truncate(buffer, 0); + output->offset = 0; + /* */ test_begin("json write "); generator = json_generator_init(output, 0); @@ -647,6 +666,50 @@ static void test_json_generate_buffer(void) str_truncate(buffer, 0); output->offset = 0; + /* array - string input stream nested */ + test_begin("json write array - string input stream nested"); + generator = json_generator_init(output, 0); + data = "ABC\tDEF\nGHI\tJKL\nMNO\x19PQR\nSTU\tVWX\nYZ"; + input = i_stream_create_from_data(data, strlen(data)); + json_generate_array_open(generator); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + ret = json_generate_array_close(generator); + test_assert(ret > 0); + json_generator_flush(generator); + test_assert(ret > 0); + json_generator_deinit(&generator); + test_assert(strcmp( + "[\"ABC\\tDEF\\nGHI\\tJKL\\nMNO\\u0019PQR\\nSTU\\tVWX\\nYZ\"]", + str_c(buffer)) == 0); + test_end(); + str_truncate(buffer, 0); + output->offset = 0; + + /* array - string input stream nested, second */ + test_begin("json write array - string input stream nested, second"); + generator = json_generator_init(output, 0); + data = "ABC\tDEF\nGHI\tJKL\nMNO\x19PQR\nSTU\tVWX\nYZ"; + input = i_stream_create_from_data(data, strlen(data)); + json_generate_array_open(generator); + ret = json_generate_string(generator, "frop"); + test_assert(ret > 0); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + ret = json_generate_array_close(generator); + test_assert(ret > 0); + json_generator_flush(generator); + test_assert(ret > 0); + json_generator_deinit(&generator); + test_assert(strcmp( + "[\"frop\",\"ABC\\tDEF\\nGHI\\tJKL\\nMNO\\u0019PQR\\nSTU\\tVWX\\nYZ\"]", + str_c(buffer)) == 0); + test_end(); + str_truncate(buffer, 0); + output->offset = 0; + /* { } */ test_begin("json write object - { }"); generator = json_generator_init(output, 0); @@ -951,6 +1014,27 @@ static void test_json_generate_buffer(void) str_truncate(buffer, 0); output->offset = 0; + /* object - string input stream nested */ + test_begin("json write object - string input stream nested"); + generator = json_generator_init(output, 0); + data = "ABC\tDEF\nGHI\tJKL\nMNO\x19PQR\nSTU\tVWX\nYZ"; + input = i_stream_create_from_data(data, strlen(data)); + json_generate_object_open(generator); + ret = json_generate_object_member(generator, "a"); + test_assert(ret > 0); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + ret = json_generate_object_close(generator); + test_assert(ret > 0); + json_generator_flush(generator); + test_assert(ret > 0); + json_generator_deinit(&generator); + test_assert(strcmp("{\"a\":\"ABC\\tDEF\\nGHI\\tJKL\\nMNO\\u0019PQR\\nSTU\\tVWX\\nYZ\"}", str_c(buffer)) == 0); + test_end(); + str_truncate(buffer, 0); + output->offset = 0; + /* trickle [1] */ test_begin("json write object - trickle[1]"); o_stream_set_max_buffer_size(output, 0); @@ -1377,6 +1461,134 @@ static void test_json_generate_buffer(void) str_truncate(buffer, 0); output->offset = 0; + /* trickle [4] */ + test_begin("json write object - trickle[4]"); + o_stream_set_max_buffer_size(output, 0); + generator = json_generator_init(output, 0); + json_generate_object_open(generator); + state = 0; + for (pos = 0; pos < 65535 && state <= 15; pos++) { + o_stream_set_max_buffer_size(output, pos); + switch (state) { + case 0: + ret = json_generate_object_member(generator, "aaaaaa"); + test_assert(ret >= 0); + if (ret == 0) break; + json_generate_array_open(generator); + json_generate_object_open(generator); + state++; + continue; + case 1: + ret = json_generate_object_member(generator, "dddddd"); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 2: + data = "AAAAA"; + input = i_stream_create_from_data(data, strlen(data)); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + state++; + continue; + case 3: + ret = json_generate_object_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 4: + ret = json_generate_array_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 5: + ret = json_generate_object_member(generator, "bbbbbb"); + test_assert(ret >= 0); + if (ret == 0) break; + json_generate_array_open(generator); + json_generate_object_open(generator); + state++; + continue; + case 6: + ret = json_generate_object_member(generator, "eeeeee"); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 7: + data = "BBBBB"; + input = i_stream_create_from_data(data, strlen(data)); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + state++; + continue; + case 8: + ret = json_generate_object_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 9: + ret = json_generate_array_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 10: + ret = json_generate_object_member(generator, "cccccc"); + test_assert(ret >= 0); + if (ret == 0) break; + json_generate_array_open(generator); + json_generate_object_open(generator); + state++; + continue; + case 11: + ret = json_generate_object_member(generator, "ffffff"); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 12: + data = "CCCCC"; + input = i_stream_create_from_data(data, strlen(data)); + ret = json_generate_string_stream(generator, input); + test_assert(ret > 0); + i_stream_unref(&input); + state++; + continue; + case 13: + ret = json_generate_object_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 14: + ret = json_generate_array_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + case 15: + ret = json_generate_object_close(generator); + test_assert(ret >= 0); + if (ret == 0) break; + state++; + continue; + } + } + json_generator_deinit(&generator); + test_assert(state == 16); + test_assert(strcmp("{\"aaaaaa\":[{\"dddddd\":\"AAAAA\"}]," + "\"bbbbbb\":[{\"eeeeee\":\"BBBBB\"}]," + "\"cccccc\":[{\"ffffff\":\"CCCCC\"}]}", str_c(buffer)) == 0); + test_end(); + str_truncate(buffer, 0); + output->offset = 0; + o_stream_destroy(&output); str_free(&buffer); }