[Feature] Add content negotiation for /stat endpoint and zstd compression

- Update /stat handler to use rspamd_controller_send_ucl_negotiated
  for Accept header content-type negotiation (JSON/msgpack)
- Add zstd compression support to rspamd_controller_maybe_compress,
  preferred over gzip when client supports it
- Add functional robot tests for content negotiation covering:
  - OpenMetrics/text/plain Accept headers for /metrics
  - JSON/msgpack Accept headers for /stat
  - gzip/zstd Accept-Encoding compression
  - Quality factor parsing

diff --git a/src/controller.c b/src/controller.c
index a1e8db9e64d1559f3682438db74dc458b2d9794e..286dbebe5423ab7a51939775cb660727a19c5f2e 100644 (file)
--- a/src/controller.c
+++ b/src/controller.c
@@ -2783,7 +2783,7 @@ rspamd_controller_stat_fin_task(void *ud)
                ucl_object_insert_key(top, ar, "fuzzy_hashes", 0, false);
        }
 
-       rspamd_controller_send_ucl(conn_ent, top);
+       rspamd_controller_send_ucl_negotiated(conn_ent, cbdata->req_msg, top);
 
 
        return TRUE;
@@ -2837,6 +2837,7 @@ rspamd_controller_handle_stat_common(
        cbdata->conn_ent = conn_ent;
        cbdata->task = task;
        cbdata->ctx = ctx;
+       cbdata->req_msg = rspamd_http_message_ref(msg);
        top = ucl_object_typed_new(UCL_OBJECT);
        cbdata->top = top;
 

diff --git a/src/libserver/http/http_router.c b/src/libserver/http/http_router.c
index b46321085ba4cfd12003e8686d77890e181fd4b7..fd472460508a5697fb8edfd2dbc8fb2b8dc70ee2 100644 (file)
--- a/src/libserver/http/http_router.c
+++ b/src/libserver/http/http_router.c
@@ -17,6 +17,7 @@
 #include "http_router.h"
 #include "http_connection.h"
 #include "http_private.h"
+#include "libserver/http_content_negotiation.h"
 #include "libutil/regexp.h"
 #include "libutil/printf.h"
 #include "libserver/logger.h"
@@ -339,15 +340,15 @@ rspamd_http_router_finish_handler(struct rspamd_http_connection *conn,
                        if (rspamd_substring_search(encoding->begin, encoding->len,
                                                                                "gzip", 4) != -1) {
                                entry->support_gzip = TRUE;
-                               entry->compression_flags |= (1 << 0); /* RSPAMD_HTTP_COMPRESS_GZIP */
+                               entry->compression_flags |= RSPAMD_HTTP_COMPRESS_GZIP;
                        }
                        if (rspamd_substring_search(encoding->begin, encoding->len,
                                                                                "zstd", 4) != -1) {
-                               entry->compression_flags |= (1 << 1); /* RSPAMD_HTTP_COMPRESS_ZSTD */
+                               entry->compression_flags |= RSPAMD_HTTP_COMPRESS_ZSTD;
                        }
                        if (rspamd_substring_search(encoding->begin, encoding->len,
                                                                                "deflate", 7) != -1) {
-                               entry->compression_flags |= (1 << 2); /* RSPAMD_HTTP_COMPRESS_DEFLATE */
+                               entry->compression_flags |= RSPAMD_HTTP_COMPRESS_DEFLATE;
                        }
                }
 

diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index 4b21d754002cc9d300bec6628ee7821407503f41..4a06018af47be2a93a8e001a2cca366107c302d6 100644 (file)
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -46,6 +46,11 @@
 #include <libutil.h>
 #endif
 #include "zlib.h"
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#else
+#include "contrib/zstd/zstd.h"
+#endif
 
 #ifdef HAVE_UCONTEXT_H
 #include <ucontext.h>
@@ -609,7 +614,23 @@ static rspamd_fstring_t *
 rspamd_controller_maybe_compress(struct rspamd_http_connection_entry *entry,
                                                                 rspamd_fstring_t *buf, struct rspamd_http_message *msg)
 {
-       if (entry->support_gzip) {
+       /* Prefer zstd over gzip if client supports it */
+       if (entry->compression_flags & RSPAMD_HTTP_COMPRESS_ZSTD) {
+               gsize compressed_size = ZSTD_compressBound(buf->len);
+               rspamd_fstring_t *compressed = rspamd_fstring_sized_new(compressed_size);
+
+               compressed->len = ZSTD_compress(compressed->str, compressed->allocated,
+                                                                               buf->str, buf->len, 1);
+
+               if (!ZSTD_isError(compressed->len) && compressed->len < buf->len) {
+                       rspamd_fstring_free(buf);
+                       rspamd_http_message_add_header(msg, CONTENT_ENCODING_HEADER, "zstd");
+                       return compressed;
+               }
+
+               rspamd_fstring_free(compressed);
+       }
+       else if (entry->support_gzip) {
                if (rspamd_fstring_gzip(&buf)) {
                        rspamd_http_message_add_header(msg, CONTENT_ENCODING_HEADER, "gzip");
                }

diff --git a/test/functional/cases/001_merged/420_content_negotiation.robot b/test/functional/cases/001_merged/420_content_negotiation.robot
new file mode 100644 (file)
index 0000000..4eb4af3
--- /dev/null
+++ b/test/functional/cases/001_merged/420_content_negotiation.robot
@@ -0,0 +1,85 @@
+*** Settings ***
+Library         ${RSPAMD_TESTDIR}/lib/rspamd.py
+Resource        ${RSPAMD_TESTDIR}/lib/rspamd.robot
+Variables       ${RSPAMD_TESTDIR}/lib/vars.py
+
+*** Test Cases ***
+Metrics Default Content Type
+  [Documentation]  Without Accept header, should return OpenMetrics format
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/openmetrics-text
+  Should Contain  ${result}[1].decode('utf-8')  \# EOF
+
+Metrics With OpenMetrics Accept
+  [Documentation]  With Accept: application/openmetrics-text, should return OpenMetrics
+  &{headers} =  Create Dictionary  Accept=application/openmetrics-text
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/openmetrics-text
+  Should Contain  ${result}[1].decode('utf-8')  \# EOF
+
+Metrics With Text Plain Accept
+  [Documentation]  With Accept: text/plain, should return Prometheus 0.0.4 format
+  &{headers} =  Create Dictionary  Accept=text/plain
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  text/plain
+  Should Contain  ${result}[1].decode('utf-8')  \# EOF
+
+Metrics With Wildcard Accept
+  [Documentation]  With Accept: */*, should return default (OpenMetrics)
+  &{headers} =  Create Dictionary  Accept=*/*
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/openmetrics-text
+
+Metrics With Quality Factor
+  [Documentation]  Accept header with quality factors should prefer higher quality
+  &{headers} =  Create Dictionary  Accept=text/plain;q=0.9, application/openmetrics-text;q=1.0
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/openmetrics-text
+
+Metrics Fallback For Unknown Accept
+  [Documentation]  With unsupported Accept type, should fallback to text/plain
+  &{headers} =  Create Dictionary  Accept=application/xml
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /metrics  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  text/plain
+
+Stat With Msgpack Accept
+  [Documentation]  With Accept: application/msgpack, should return msgpack format
+  &{headers} =  Create Dictionary  Accept=application/msgpack
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /stat  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/msgpack
+
+Stat With JSON Accept
+  [Documentation]  With Accept: application/json, should return JSON format
+  &{headers} =  Create Dictionary  Accept=application/json
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /stat  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/json
+
+Stat With Zstd Encoding
+  [Documentation]  With Accept-Encoding: zstd, should return zstd compressed response
+  &{headers} =  Create Dictionary  Accept-Encoding=zstd
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /stat  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Encoding]  zstd
+
+Stat With Gzip Encoding
+  [Documentation]  With Accept-Encoding: gzip, should return gzip compressed response
+  &{headers} =  Create Dictionary  Accept-Encoding=gzip
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /stat  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Encoding]  gzip
+
+Stat With Msgpack And Zstd
+  [Documentation]  With Accept: msgpack and Accept-Encoding: zstd, should return compressed msgpack
+  &{headers} =  Create Dictionary  Accept=application/msgpack  Accept-Encoding=zstd
+  @{result} =  HTTP With Headers  GET  ${RSPAMD_LOCAL_ADDR}  ${RSPAMD_PORT_CONTROLLER}  /stat  headers=${headers}
+  Should Be Equal As Integers  ${result}[0]  200
+  Should Contain  ${result}[2][Content-Type]  application/msgpack
+  Should Contain  ${result}[2][Content-Encoding]  zstd

diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py
index ea9c6204b8c08ad0e28239f6a284473fb8a07ef2..9c869df0ef2e6ce1166dc200dc63e93f9f7f1217 100644 (file)
--- a/test/functional/lib/rspamd.py
+++ b/test/functional/lib/rspamd.py
@@ -148,6 +148,20 @@ def HTTP(method, host, port, path, data=None, headers={}):
     return [s, t]
 
 
+def HTTP_With_Headers(method, host, port, path, data=None, headers={}):
+    """HTTP request that returns response headers.
+    Returns [status, body, headers_dict]
+    """
+    c = http.client.HTTPConnection("%s:%s" % (host, port))
+    c.request(method, path, data, headers)
+    r = c.getresponse()
+    t = r.read()
+    s = r.status
+    h = dict(r.getheaders())
+    c.close()
+    return [s, t, h]
+
+
 def hard_link(src, dst):
     os.link(src, dst)