From: Stephan Bosch Date: Mon, 1 Apr 2019 23:49:57 +0000 (+0200) Subject: lib: base64 - Add support for adding line breaks to encoded output. X-Git-Tag: 2.3.8~128 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c8c9de1d5f20f71ddbd8298287d01242dced8d15;p=thirdparty%2Fdovecot%2Fcore.git lib: base64 - Add support for adding line breaks to encoded output. --- diff --git a/src/lib/base64.c b/src/lib/base64.c index 96ecde5cc4..53a10dc368 100644 --- a/src/lib/base64.c +++ b/src/lib/base64.c @@ -8,14 +8,25 @@ * Low-level Base64 encoder */ -off_t base64_get_full_encoded_size(struct base64_encoder *enc ATTR_UNUSED, - off_t src_size) +off_t base64_get_full_encoded_size(struct base64_encoder *enc, off_t src_size) { + bool crlf = HAS_ALL_BITS(enc->flags, BASE64_ENCODE_FLAG_CRLF); off_t out_size; + off_t newlines; /* calculate size of encoded data */ out_size = MAX_BASE64_ENCODED_SIZE(src_size); + /* newline between each full line */ + newlines = (out_size / enc->max_line_len) - 1; + /* an extra newline to separate the partial last line from the previous + full line */ + if ((out_size % enc->max_line_len) != 0) + newlines++; + + /* update size with added newlines */ + out_size += newlines * (crlf ? 2 : 1); + return out_size; } @@ -68,7 +79,9 @@ base64_encode_get_out_size(struct base64_encoder *enc, size_t src_size) size_t base64_encode_get_size(struct base64_encoder *enc, size_t src_size) { + bool crlf = HAS_ALL_BITS(enc->flags, BASE64_ENCODE_FLAG_CRLF); size_t out_size = base64_encode_get_out_size(enc, src_size); + size_t line_part, lines; if (src_size == 0) { /* last block */ @@ -86,6 +99,16 @@ size_t base64_encode_get_size(struct base64_encoder *enc, size_t src_size) } } + if (enc->max_line_len == SIZE_MAX) + return out_size; + + /* Calculate how many line endings must be added */ + lines = out_size / enc->max_line_len; + line_part = out_size % enc->max_line_len; + if (enc->cur_line_len > (enc->max_line_len - line_part)) + lines++; + + out_size += lines * (crlf ? 2 : 1); return out_size; } @@ -251,22 +274,59 @@ bool base64_encode_more(struct base64_encoder *enc, const void *src, size_t src_size, size_t *src_pos_r, buffer_t *dest) { - const unsigned char *src_c = src; - size_t src_pos, dst_avail; + const unsigned char *src_c, *src_p; + size_t src_pos; i_assert(!enc->finished); - /* determine how much we can write in destination buffer */ - dst_avail = buffer_get_avail_size(dest); - if (dst_avail == 0) { - i_assert(src_pos_r != NULL); - *src_pos_r = 0; - return FALSE; - } + src_p = src_c = src; + while (src_size > 0) { + size_t dst_avail, dst_pos, line_avail, written; - base64_encode_more_data(enc, src_c, src_size, &src_pos, - dst_avail, dest); + /* determine how much we can write in destination buffer */ + dst_avail = buffer_get_avail_size(dest); + if (dst_avail == 0) + break; + i_assert(enc->max_line_len > 0); + i_assert(enc->cur_line_len <= enc->max_line_len); + line_avail = I_MIN(enc->max_line_len - enc->cur_line_len, + dst_avail); + + if (line_avail > 0) { + dst_pos = dest->used; + base64_encode_more_data(enc, src_p, src_size, &src_pos, + line_avail, dest); + i_assert(src_pos <= src_size); + src_p += src_pos; + src_size -= src_pos; + i_assert(dest->used >= dst_pos); + written = dest->used - dst_pos; + + i_assert(written <= line_avail); + i_assert(written <= enc->max_line_len); + i_assert(enc->cur_line_len <= + (enc->max_line_len - written)); + enc->cur_line_len += written; + dst_avail -= written; + } + + if (src_size > 0 && enc->cur_line_len == enc->max_line_len) { + if (HAS_ALL_BITS(enc->flags, BASE64_ENCODE_FLAG_CRLF)) { + if (dst_avail < 2) + break; + buffer_append(dest, "\r\n", 2); + } else { + if (dst_avail < 1) + break; + buffer_append_c(dest, '\n'); + } + enc->cur_line_len = 0; + } + } + + i_assert(src_p >= src_c); + src_pos = (src_p - src_c); if (src_pos_r != NULL) *src_pos_r = src_pos; return (src_pos == src_size); @@ -276,9 +336,11 @@ bool base64_encode_finish(struct base64_encoder *enc, buffer_t *dest) { const struct base64_scheme *b64 = enc->b64; const char *b64enc = b64->encmap; - size_t dst_avail; - unsigned char w_buf[7]; - unsigned int w_buf_len = 0; + bool crlf = HAS_ALL_BITS(enc->flags, BASE64_ENCODE_FLAG_CRLF); + unsigned char *ptr, *end; + size_t dst_avail, line_avail, write; + unsigned char w_buf[9]; + unsigned int w_buf_len = 0, w_buf_pos = 0; dst_avail = 0; if (dest != NULL) @@ -294,6 +356,10 @@ bool base64_encode_finish(struct base64_encoder *enc, buffer_t *dest) w_buf_len += enc->w_buf_len; } + i_assert(enc->max_line_len > 0); + i_assert(enc->cur_line_len <= enc->max_line_len); + line_avail = enc->max_line_len - enc->cur_line_len; + switch (enc->sub_pos) { case 0: break; @@ -313,16 +379,46 @@ bool base64_encode_finish(struct base64_encoder *enc, buffer_t *dest) } enc->sub_pos = 0; - if (w_buf_len == 0) { + write = w_buf_len; + if (enc->max_line_len < SIZE_MAX && line_avail < write) { + unsigned int lines; + + lines = I_MAX((write - line_avail) / enc->max_line_len, 1); + write += lines * (crlf ? 2 : 1); + } else { + line_avail = write; + } + + if (write == 0) { enc->finished = TRUE; return TRUE; } i_assert(dest != NULL); - if (dst_avail < w_buf_len) + if (dst_avail < write) return FALSE; - buffer_append(dest, w_buf, w_buf_len); + ptr = buffer_append_space_unsafe(dest, write); + end = ptr + write; + if (line_avail > 0) { + memcpy(ptr, w_buf, line_avail); + ptr += line_avail; + w_buf_pos += line_avail; + } + while (w_buf_pos < w_buf_len) { + if (crlf) { + ptr[0] = '\r'; + ptr++; + } + ptr[0] = '\n'; + ptr++; + + write = I_MIN(w_buf_len - w_buf_pos, enc->max_line_len); + memcpy(ptr, &w_buf[w_buf_pos], write); + ptr += write; + w_buf_pos += write; + } + i_assert(ptr == end); enc->finished = TRUE; return TRUE; } diff --git a/src/lib/base64.h b/src/lib/base64.h index 03f699e4c9..e9c2922e2a 100644 --- a/src/lib/base64.h +++ b/src/lib/base64.h @@ -21,12 +21,20 @@ struct base64_scheme { * Low-level Base64 encoder */ +enum base64_encode_flags { + /* Use CRLF instead of the default LF as line ending. */ + BASE64_ENCODE_FLAG_CRLF = BIT(0), +}; + struct base64_encoder { const struct base64_scheme *b64; + enum base64_encode_flags flags; + size_t max_line_len; /* state */ unsigned int sub_pos; unsigned char buf; + size_t cur_line_len; unsigned char w_buf[4]; unsigned int w_buf_len; @@ -46,10 +54,14 @@ base64_encode_is_finished(struct base64_encoder *enc) */ static inline void base64_encode_init(struct base64_encoder *enc, - const struct base64_scheme *b64) + const struct base64_scheme *b64, + enum base64_encode_flags flags, + size_t max_line_len) { i_zero(enc); enc->b64 = b64; + enc->flags = flags; + enc->max_line_len = (max_line_len == 0 ? SIZE_MAX : max_line_len); } /* Reset the Base64 encoder to its initial state. */ @@ -57,8 +69,10 @@ static inline void base64_encode_reset(struct base64_encoder *enc) { const struct base64_scheme *b64 = enc->b64; + enum base64_encode_flags flags = enc->flags; + size_t max_line_len = enc->max_line_len; - base64_encode_init(enc, b64); + base64_encode_init(enc, b64, flags, max_line_len); } /* Translate the size of the full encoder input to the size of the encoder @@ -181,7 +195,7 @@ base64_scheme_encode(const struct base64_scheme *b64, { struct base64_encoder enc; - base64_encode_init(&enc, b64); + base64_encode_init(&enc, b64, 0, 0); base64_encode_more(&enc, src, src_size, NULL, dest); base64_encode_finish(&enc, dest); } diff --git a/src/lib/test-base64.c b/src/lib/test-base64.c index 2926649283..0bd54a571c 100644 --- a/src/lib/test-base64.c +++ b/src/lib/test-base64.c @@ -244,6 +244,9 @@ static void test_base64url_random(void) struct test_base64_encode_lowlevel { const struct base64_scheme *scheme; + enum base64_encode_flags flags; + size_t max_line_len; + const char *input; const char *output; }; @@ -310,6 +313,87 @@ tests_base64_encode_lowlevel[] = { "\x99", .output = "6KeS44KS55-v44KB44Gm54mb44KS5q6644GZ", }, + { + .scheme = &base64_scheme, + .flags = BASE64_ENCODE_FLAG_CRLF, + .input = "just niin", + .output = "anVzdCBuaWlu", + }, + { + .scheme = &base64_scheme, + .flags = BASE64_ENCODE_FLAG_CRLF, + .max_line_len = 80, + .input = "just niin", + .output = "anVzdCBuaWlu", + }, + { + .scheme = &base64_scheme, + .flags = BASE64_ENCODE_FLAG_CRLF, + .max_line_len = 48, + .input = + "Passer, deliciae meae puellae,\n" + "quicum ludere, quem in sinu tenere,\n" + "cui primum digitum dare appetenti\n" + "et acris solet incitare morsus,\n" + "cum desiderio meo nitenti\n" + "carum nescio quid lubet iocari,\n" + "credo ut, cum gravis acquiescet ardor,\n" + "sit solaciolum sui doloris,\n" + "tecum ludere sicut ipsa possem\n" + "et tristis animi levare curas!\n", + .output = + "UGFzc2VyLCBkZWxpY2lhZSBtZWFlIHB1ZWxsYWUsCnF1aWN1\r\n" + "bSBsdWRlcmUsIHF1ZW0gaW4gc2ludSB0ZW5lcmUsCmN1aSBw\r\n" + "cmltdW0gZGlnaXR1bSBkYXJlIGFwcGV0ZW50aQpldCBhY3Jp\r\n" + "cyBzb2xldCBpbmNpdGFyZSBtb3JzdXMsCmN1bSBkZXNpZGVy\r\n" + "aW8gbWVvIG5pdGVudGkKY2FydW0gbmVzY2lvIHF1aWQgbHVi\r\n" + "ZXQgaW9jYXJpLApjcmVkbyB1dCwgY3VtIGdyYXZpcyBhY3F1\r\n" + "aWVzY2V0IGFyZG9yLApzaXQgc29sYWNpb2x1bSBzdWkgZG9s\r\n" + "b3JpcywKdGVjdW0gbHVkZXJlIHNpY3V0IGlwc2EgcG9zc2Vt\r\n" + "CmV0IHRyaXN0aXMgYW5pbWkgbGV2YXJlIGN1cmFzIQo=", + }, + { + .scheme = &base64_scheme, + .max_line_len = 48, + .input = + "Lugete, o Veneres Cupidinesque,\n" + "et quantum est hominum venustiorum:\n" + "passer mortuus est meae puellae, \n" + "passer, deliciae meae puellae\n" + "quem plus amat illa oculis suis amabat.\n" + "Nam mellitus erat suamque norat\n" + "ipsam tam bene quam puella matrem,\n" + "nec sese a gremio illius movebat,\n" + "sed circumsiliens modo huc modo illuc\n" + "ad solam dominam usque pipiabat;\n" + "qui nunc it per iter tenebricosum\n" + "illuc, unde negant redire quemquam.\n" + "At vobis male sint, malae tenebrae\n" + "Orci, quae omnia bella devoratis:\n" + "tam bellum mihi passerem abstulistis.\n" + "O factum male! O miselle passer!\n" + "Tua nunc opera meae puellae\n" + "flendo turgiduli rubent ocelli.\n", + .output = + "THVnZXRlLCBvIFZlbmVyZXMgQ3VwaWRpbmVzcXVlLApldCBx\n" + "dWFudHVtIGVzdCBob21pbnVtIHZlbnVzdGlvcnVtOgpwYXNz\n" + "ZXIgbW9ydHV1cyBlc3QgbWVhZSBwdWVsbGFlLCAKcGFzc2Vy\n" + "LCBkZWxpY2lhZSBtZWFlIHB1ZWxsYWUKcXVlbSBwbHVzIGFt\n" + "YXQgaWxsYSBvY3VsaXMgc3VpcyBhbWFiYXQuCk5hbSBtZWxs\n" + "aXR1cyBlcmF0IHN1YW1xdWUgbm9yYXQKaXBzYW0gdGFtIGJl\n" + "bmUgcXVhbSBwdWVsbGEgbWF0cmVtLApuZWMgc2VzZSBhIGdy\n" + "ZW1pbyBpbGxpdXMgbW92ZWJhdCwKc2VkIGNpcmN1bXNpbGll\n" + "bnMgbW9kbyBodWMgbW9kbyBpbGx1YwphZCBzb2xhbSBkb21p\n" + "bmFtIHVzcXVlIHBpcGlhYmF0OwpxdWkgbnVuYyBpdCBwZXIg\n" + "aXRlciB0ZW5lYnJpY29zdW0KaWxsdWMsIHVuZGUgbmVnYW50\n" + "IHJlZGlyZSBxdWVtcXVhbS4KQXQgdm9iaXMgbWFsZSBzaW50\n" + "LCBtYWxhZSB0ZW5lYnJhZQpPcmNpLCBxdWFlIG9tbmlhIGJl\n" + "bGxhIGRldm9yYXRpczoKdGFtIGJlbGx1bSBtaWhpIHBhc3Nl\n" + "cmVtIGFic3R1bGlzdGlzLgpPIGZhY3R1bSBtYWxlISBPIG1p\n" + "c2VsbGUgcGFzc2VyIQpUdWEgbnVuYyBvcGVyYSBtZWFlIHB1\n" + "ZWxsYWUKZmxlbmRvIHR1cmdpZHVsaSBydWJlbnQgb2NlbGxp\n" + "Lgo=", + }, }; static void test_base64_encode_lowlevel(void) @@ -323,18 +407,23 @@ static void test_base64_encode_lowlevel(void) const struct test_base64_encode_lowlevel *test = &tests_base64_encode_lowlevel[i]; struct base64_encoder enc; + uoff_t out_size; str_truncate(str, 0); - base64_encode_init(&enc, test->scheme); + base64_encode_init(&enc, test->scheme, test->flags, + test->max_line_len); + out_size = base64_get_full_encoded_size( + &enc, strlen(test->input)); base64_encode_more(&enc, test->input, strlen(test->input), NULL, str); base64_encode_finish(&enc, str); test_assert_idx(strcmp(test->output, str_c(str)) == 0, i); - test_assert_idx( + test_assert_idx(test->flags != 0 || test->max_line_len != 0 || str_len(str) == MAX_BASE64_ENCODED_SIZE( strlen(test->input)), i); + test_assert_idx(str_len(str) == out_size, i); } test_end(); } @@ -578,7 +667,9 @@ static void test_base64_decode_lowlevel(void) static void test_base64_random_lowlevel_one_block(const struct base64_scheme *b64, + enum base64_encode_flags enc_flags, enum base64_decode_flags dec_flags, + size_t max_line_len, unsigned int test_idx, const unsigned char *in_buf, size_t in_buf_size, @@ -591,7 +682,7 @@ test_base64_random_lowlevel_one_block(const struct base64_scheme *b64, buffer_set_used_size(buf1, 0); buffer_set_used_size(buf2, 0); - base64_encode_init(&enc, b64); + base64_encode_init(&enc, b64, enc_flags, max_line_len); base64_encode_more(&enc, in_buf, in_buf_size, NULL, buf1); base64_encode_finish(&enc, buf1); @@ -608,8 +699,9 @@ test_base64_random_lowlevel_one_block(const struct base64_scheme *b64, static void test_base64_random_lowlevel_stream(const struct base64_scheme *b64, + enum base64_encode_flags enc_flags, enum base64_decode_flags dec_flags, - unsigned int test_idx, + size_t max_line_len, unsigned int test_idx, const unsigned char *in_buf, size_t in_buf_size, buffer_t *buf1, buffer_t *buf2, @@ -629,7 +721,7 @@ test_base64_random_lowlevel_stream(const struct base64_scheme *b64, buffer_set_used_size(buf1, 0); buffer_set_used_size(buf2, 0); - base64_encode_init(&enc, b64); + base64_encode_init(&enc, b64, enc_flags, max_line_len); out_space = 0; for (buf_p = buf_begin; buf_p < buf_end; ) { size_t buf_ch, out_ch; @@ -710,7 +802,9 @@ test_base64_random_lowlevel_stream(const struct base64_scheme *b64, static void test_base64_random_lowlevel_case(const struct base64_scheme *b64, - enum base64_decode_flags dec_flags) + enum base64_encode_flags enc_flags, + enum base64_decode_flags dec_flags, + size_t max_line_len) { unsigned char in_buf[512]; size_t in_buf_size; @@ -726,7 +820,8 @@ test_base64_random_lowlevel_case(const struct base64_scheme *b64, for (j = 0; j < in_buf_size; j++) in_buf[j] = i_rand(); - test_base64_random_lowlevel_one_block(b64, dec_flags, i, + test_base64_random_lowlevel_one_block(b64, enc_flags, dec_flags, + max_line_len, i, in_buf, in_buf_size, buf1, buf2); } @@ -737,7 +832,8 @@ test_base64_random_lowlevel_case(const struct base64_scheme *b64, for (j = 0; j < in_buf_size; j++) in_buf[j] = i_rand(); - test_base64_random_lowlevel_stream(b64, dec_flags, i, + test_base64_random_lowlevel_stream(b64, enc_flags, dec_flags, + max_line_len, i, in_buf, in_buf_size, buf1, buf2, 1); } @@ -748,7 +844,8 @@ test_base64_random_lowlevel_case(const struct base64_scheme *b64, for (j = 0; j < in_buf_size; j++) in_buf[j] = i_rand(); - test_base64_random_lowlevel_stream(b64, dec_flags, i, + test_base64_random_lowlevel_stream(b64, enc_flags, dec_flags, + max_line_len, i, in_buf, in_buf_size, buf1, buf2, 0); } @@ -758,16 +855,115 @@ static void test_base64_random_lowlevel(void) { test_begin("base64 encode/decode low-level with random input"); - test_base64_random_lowlevel_case(&base64_scheme, 0); - test_base64_random_lowlevel_case(&base64url_scheme, 0); + test_base64_random_lowlevel_case(&base64_scheme, 0, 0, 0); + test_base64_random_lowlevel_case(&base64url_scheme, 0, 0, 0); + test_base64_random_lowlevel_case(&base64_scheme, 0, + BASE64_DECODE_FLAG_EXPECT_BOUNDARY, 0); + test_base64_random_lowlevel_case(&base64url_scheme, 0, + BASE64_DECODE_FLAG_EXPECT_BOUNDARY, 0); + test_base64_random_lowlevel_case(&base64_scheme, 0, + BASE64_DECODE_FLAG_NO_WHITESPACE, 0); + test_base64_random_lowlevel_case(&base64url_scheme, 0, + BASE64_DECODE_FLAG_NO_WHITESPACE, 0); + test_base64_random_lowlevel_case(&base64_scheme, 0, 0, 10); + test_base64_random_lowlevel_case(&base64url_scheme, 0, 0, 10); test_base64_random_lowlevel_case(&base64_scheme, - BASE64_DECODE_FLAG_EXPECT_BOUNDARY); + BASE64_ENCODE_FLAG_CRLF, 0, 10); test_base64_random_lowlevel_case(&base64url_scheme, - BASE64_DECODE_FLAG_EXPECT_BOUNDARY); - test_base64_random_lowlevel_case(&base64_scheme, - BASE64_DECODE_FLAG_NO_WHITESPACE); - test_base64_random_lowlevel_case(&base64url_scheme, - BASE64_DECODE_FLAG_NO_WHITESPACE); + BASE64_ENCODE_FLAG_CRLF, 0, 10); + test_end(); +} + +static void +_add_lines(const char *in, size_t max_line_len, bool crlf, string_t *out) +{ + size_t in_len = strlen(in); + + while (max_line_len > 0 && in_len > max_line_len) { + str_append_data(out, in, max_line_len); + if (crlf) + str_append(out, "\r\n"); + else + str_append_c(out, '\n'); + in += max_line_len; + in_len -= max_line_len; + } + + str_append_data(out, in, in_len); +} + +static void test_base64_encode_lines(void) +{ + static const char *input[] = { + "Passer, deliciae meae puellae,\n" + "quicum ludere, quem in sinu tenere,\n" + "cui primum digitum dare appetenti\n" + "et acris solet incitare morsus,\n" + "cum desiderio meo nitenti\n" + "carum nescio quid lubet iocari,\n" + "credo ut, cum gravis acquiescet ardor,\n" + "sit solaciolum sui doloris,\n" + "tecum ludere sicut ipsa possem\n" + "et tristis animi levare curas!\n" + }; + static const char *output[] = { + "UGFzc2VyLCBkZWxpY2lhZSBtZWFlIHB1ZWxsYWUsCnF1aWN1" + "bSBsdWRlcmUsIHF1ZW0gaW4gc2ludSB0ZW5lcmUsCmN1aSBw" + "cmltdW0gZGlnaXR1bSBkYXJlIGFwcGV0ZW50aQpldCBhY3Jp" + "cyBzb2xldCBpbmNpdGFyZSBtb3JzdXMsCmN1bSBkZXNpZGVy" + "aW8gbWVvIG5pdGVudGkKY2FydW0gbmVzY2lvIHF1aWQgbHVi" + "ZXQgaW9jYXJpLApjcmVkbyB1dCwgY3VtIGdyYXZpcyBhY3F1" + "aWVzY2V0IGFyZG9yLApzaXQgc29sYWNpb2x1bSBzdWkgZG9s" + "b3JpcywKdGVjdW0gbHVkZXJlIHNpY3V0IGlwc2EgcG9zc2Vt" + "CmV0IHRyaXN0aXMgYW5pbWkgbGV2YXJlIGN1cmFzIQo=", + }; + string_t *out_test, *out_ref; + unsigned int i, n; + + out_test = t_str_new(256); + out_ref = t_str_new(256); + + test_begin("base64 encode lines (LF)"); + for (i = 0; i < N_ELEMENTS(input); i++) { + struct base64_encoder b64enc; + + for (n = 0; n <= 80; n++) { + str_truncate(out_test, 0); + base64_encode_init(&b64enc, &base64_scheme, 0, n); + base64_encode_more(&b64enc, input[i], strlen(input[i]), + NULL, out_test); + base64_encode_finish(&b64enc, out_test); + + str_truncate(out_ref, 0); + _add_lines(output[i], n, FALSE, out_ref); + + test_assert(strcmp(str_c(out_ref), + str_c(out_test)) == 0); + + } + } + test_end(); + + test_begin("base64 encode lines (CRLF)"); + for (i = 0; i < N_ELEMENTS(input); i++) { + struct base64_encoder b64enc; + + for (n = 0; n <= 80; n++) { + str_truncate(out_test, 0); + base64_encode_init(&b64enc, &base64_scheme, + BASE64_ENCODE_FLAG_CRLF, n); + base64_encode_more(&b64enc, input[i], strlen(input[i]), + NULL, out_test); + base64_encode_finish(&b64enc, out_test); + + str_truncate(out_ref, 0); + _add_lines(output[i], n, TRUE, out_ref); + + test_assert(strcmp(str_c(out_ref), + str_c(out_test)) == 0); + + } + } test_end(); } @@ -782,4 +978,5 @@ void test_base64(void) test_base64_encode_lowlevel(); test_base64_decode_lowlevel(); test_base64_random_lowlevel(); + test_base64_encode_lines(); }