Merge branch 'master' into userinfo

Resolve conflict in src/libserver/maps/map.c: take master's zstd
decompression refactor (rspamd_zstd_decompress_bounded) and re-apply
credential redaction (uri -> uri_log) in the new error paths.

diff --cc src/libserver/maps/map.c
index bbd9b1a4f6cfe59166881611b20423cfe49bb426,bfac513f56aaca94e512579539bd0b7d75a2d584..9fa303107d4e5375eb3ac62e0cbe4a7db69b8307
--- 1/src/libserver/maps/map.c
--- 2/src/libserver/maps/map.c
+++ b/src/libserver/maps/map.c
@@@ -789,63 -794,37 +794,37 @@@ http_map_finish(struct rspamd_http_conn
  
                /* If compressed flag is set OR payload looks like zstd, decompress */
                if (cbd->bk->is_compressed || rspamd_map_payload_is_zstd(payload, payload_len)) {
-                       ZSTD_DStream *zstream;
-                       ZSTD_inBuffer zin;
-                       ZSTD_outBuffer zout;
-                       gsize outlen, r;
- 
-                       zstream = ZSTD_createDStream();
-                       ZSTD_initDStream(zstream);
- 
-                       zin.pos = 0;
-                       zin.src = payload;
-                       zin.size = payload_len;
+                       gsize max_size = rspamd_map_effective_max_size(cbd);
+                       GError *derr = NULL;
+                       rspamd_fstring_t *decompressed;
  
-                       if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
-                               outlen = ZSTD_DStreamOutSize();
-                       }
- 
-                       final_out = g_malloc(outlen);
- 
-                       zout.dst = final_out;
-                       zout.pos = 0;
-                       zout.size = outlen;
+                       decompressed = rspamd_zstd_decompress_bounded(NULL, payload, payload_len,
+                                                                                                                 max_size, &derr);
  
-                       while (zin.pos < zin.size) {
-                               r = ZSTD_decompressStream(zstream, &zout, &zin);
- 
-                               if (ZSTD_isError(r)) {
-                                       msg_err_map("%s(%s): cannot decompress data: %s",
-                                                               cbd->bk->uri_log,
-                                                               rspamd_inet_address_to_string_pretty(cbd->addr),
-                                                               ZSTD_getErrorName(r));
-                                       ZSTD_freeDStream(zstream);
-                                       g_free(final_out);
-                                       if (cbd->bk->is_encrypted && payload && payload != (unsigned char *) in) {
-                                               rspamd_explicit_memzero(payload, payload_len);
-                                               g_free(payload);
-                                       }
-                                       MAP_RELEASE(cbd->shmem_data, "shmem_data");
-                                       goto err;
-                               }
- 
-                               if (zout.pos == zout.size) {
-                                       /* We need to extend output buffer */
-                                       zout.size = zout.size * 2 + 1.0;
-                                       final_out = g_realloc(zout.dst, zout.size);
-                                       zout.dst = final_out;
+                       if (decompressed == NULL) {
+                               msg_err_map("%s(%s): cannot decompress data: %s",
 -                                                      cbd->bk->uri,
++                                                      cbd->bk->uri_log,
+                                                       rspamd_inet_address_to_string_pretty(cbd->addr),
+                                                       derr ? derr->message : "unknown error");
+                               g_clear_error(&derr);
+                               if (cbd->bk->is_encrypted && payload && payload != (unsigned char *) in) {
+                                       rspamd_explicit_memzero(payload, payload_len);
+                                       g_free(payload);
                                }
+                               MAP_RELEASE(cbd->shmem_data, "shmem_data");
+                               goto err;
                        }
  
-                       ZSTD_freeDStream(zstream);
                        msg_info_map("%s(%s): read map data %z bytes compressed, "
                                                 "%z uncompressed, next check at %s",
                                                 cbd->bk->uri,
                                                 rspamd_inet_address_to_string_pretty(cbd->addr),
-                                                payload_len, zout.pos, next_check_date);
-                       if (!rspamd_map_save_http_cached_file(map, bk, cbd->data, final_out, zout.pos)) {
+                                                payload_len, decompressed->len, next_check_date);
+                       if (!rspamd_map_save_http_cached_file(map, bk, cbd->data,
+                                                                                                 (const unsigned char *) decompressed->str,
+                                                                                                 decompressed->len)) {
 -                              msg_err_map("%s: failed to save cache file", bk->uri);
 +                              msg_err_map("%s: failed to save cache file", bk->uri_log);
-                               g_free(final_out);
+                               rspamd_fstring_free(decompressed);
                                MAP_RELEASE(cbd->shmem_data, "shmem_data");
                                goto err;
                        }
@@@ -1885,55 -1814,26 +1814,26 @@@ rspamd_map_read_cached(struct rspamd_ma
  
                /* If compressed flag is set OR payload looks like zstd, decompress */
                if (bk->is_compressed || rspamd_map_payload_is_zstd(payload, payload_len)) {
-                       ZSTD_DStream *zstream;
-                       ZSTD_inBuffer zin;
-                       ZSTD_outBuffer zout;
-                       unsigned char *out;
-                       gsize outlen, r;
- 
-                       zstream = ZSTD_createDStream();
-                       ZSTD_initDStream(zstream);
- 
-                       zin.pos = 0;
-                       zin.src = payload;
-                       zin.size = payload_len;
- 
-                       if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
-                               outlen = ZSTD_DStreamOutSize();
-                       }
- 
-                       out = g_malloc(outlen);
- 
-                       zout.dst = out;
-                       zout.pos = 0;
-                       zout.size = outlen;
- 
-                       while (zin.pos < zin.size) {
-                               r = ZSTD_decompressStream(zstream, &zout, &zin);
- 
-                               if (ZSTD_isError(r)) {
-                                       msg_err_map("%s: cannot decompress data: %s",
-                                                               bk->uri_log,
-                                                               ZSTD_getErrorName(r));
-                                       ZSTD_freeDStream(zstream);
-                                       g_free(out);
-                                       if (dec) {
-                                               rspamd_explicit_memzero(dec, declen);
-                                               g_free(dec);
-                                       }
-                                       munmap(in, mmap_len);
-                                       return FALSE;
-                               }
- 
-                               if (zout.pos == zout.size) {
-                                       /* We need to extend output buffer */
-                                       zout.size = zout.size * 2 + 1;
-                                       out = g_realloc(zout.dst, zout.size);
-                                       zout.dst = out;
+                       GError *derr = NULL;
+                       rspamd_fstring_t *decompressed;
+ 
+                       decompressed = rspamd_zstd_decompress_bounded(NULL, payload, payload_len,
+                                                                                                                 map->cfg ? map->cfg->max_map_size : 0,
+                                                                                                                 &derr);
+ 
+                       if (decompressed == NULL) {
+                               msg_err_map("%s: cannot decompress data: %s",
 -                                                      bk->uri,
++                                                      bk->uri_log,
+                                                       derr ? derr->message : "unknown error");
+                               g_clear_error(&derr);
+                               if (dec) {
+                                       rspamd_explicit_memzero(dec, declen);
+                                       g_free(dec);
                                }
+                               munmap(in, mmap_len);
+                               return FALSE;
                        }
  
-                       ZSTD_freeDStream(zstream);
                        msg_info_map("%s: read map data cached %z bytes compressed, "
                                                 "%z uncompressed",
                                                 bk->uri,

diff --cc src/libserver/maps/map_private.h
Simple merge

diff --cc test/rspamd_cxx_unit.cxx
Simple merge