doc/userguide: generate eve documentation

Add EVE documentation for QUIC and Pgsql to their respective sections of
the userguide.

Also add a complete EVE reference as an appendix.

Other protocols can be done, but its a manual process to document in the
schema, then add the glue to pull them into the documentation.

The documentation is generated during "make dist", or if it doesn't
exist, "conf.py" will attempt to generate the eve documentation for
building on Readthedocs.

diff --git a/Makefile.am b/Makefile.am
index 6e2f77cadc2415b36ef50b3377267afb57eeb4d8..20e50bdc4a03addecbf29a4da876028cda53d70a 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \
             scripts/generate-images.sh \
             scripts/docs-almalinux9-minimal-build.sh \
             scripts/docs-ubuntu-debian-minimal-build.sh \
+       scripts/evedoc.py \
             examples/plugins
 SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \
           $(SURICATA_UPDATE_DIR)

diff --git a/doc/userguide/.gitignore b/doc/userguide/.gitignore
index e35d8850c9688b1ce82711694692cc574a799396..1f45e1e2c5a8f5cfba9e05b575545dce3c560be4 100644 (file)
--- a/doc/userguide/.gitignore
+++ b/doc/userguide/.gitignore
@@ -1 +1,2 @@
 _build
+_generated

diff --git a/doc/userguide/Makefile.am b/doc/userguide/Makefile.am
index 53f4907eb74366a8a6374c3c4ac5869fd6cd867c..aeab4768b086311fb77794f1408ab01d7374cb57 100644 (file)
--- a/doc/userguide/Makefile.am
+++ b/doc/userguide/Makefile.am
@@ -1,7 +1,9 @@
 EXTRA_DIST = \
+       _generated \
        _static \
        3rd-party-integration \
        acknowledgements.rst \
+       appendix \
        capture-hardware \
        command-line-options.rst \
        conf.py \
@@ -47,14 +49,14 @@ endif
 
 SPHINX_BUILD = sphinx-build -q
 
-html:
+html: _generated
        sysconfdir=$(sysconfdir) \
        localstatedir=$(localstatedir) \
        version=$(PACKAGE_VERSION) \
                $(SPHINX_BUILD) -W -b html -d _build/doctrees \
                $(top_srcdir)/doc/userguide _build/html
 
-_build/latex/Suricata.pdf:
+_build/latex/Suricata.pdf: _generated
        sysconfdir=$(sysconfdir) \
        localstatedir=$(localstatedir) \
        version=$(PACKAGE_VERSION) \
@@ -74,7 +76,7 @@ userguide.pdf: _build/latex/Suricata.pdf
 
 pdf: userguide.pdf
 
-_build/man: manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
+_build/man: _generated manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
        RELEASE_DATE=$(RELEASE_DATE) \
        sysconfdir=$(sysconfdir) \
        localstatedir=$(localstatedir) \
@@ -95,3 +97,6 @@ clean-local:
        rm -f $(top_builddir)/doc/userguide/userguide.pdf
 
 endif # SPHINX_BUILD
+
+_generated:
+       ./generate-evedoc.sh

diff --git a/doc/userguide/appendix/eve-index.rst b/doc/userguide/appendix/eve-index.rst
new file mode 100644 (file)
index 0000000..0789c04
--- /dev/null
+++ b/doc/userguide/appendix/eve-index.rst
@@ -0,0 +1,7 @@
+EVE Index
+=========
+
+.. toctree::
+   :maxdepth: 1
+
+.. include:: ../_generated/eve-index.rst

diff --git a/doc/userguide/appendix/index.rst b/doc/userguide/appendix/index.rst
new file mode 100644 (file)
index 0000000..56cf792
--- /dev/null
+++ b/doc/userguide/appendix/index.rst
@@ -0,0 +1,7 @@
+Appendix
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   eve-index

diff --git a/doc/userguide/conf.py b/doc/userguide/conf.py
index 959744e88b7b83ff49d73e1f26932f49598ca0ba..48d5e24dad26603376225c3e752e97cebbb783bf 100644 (file)
--- a/doc/userguide/conf.py
+++ b/doc/userguide/conf.py
@@ -151,6 +151,11 @@ def setup(app):
     else:
         app.add_stylesheet('css/suricata.css')
 
+    # Build generated pages if they don't exist. For example, when on
+    # RTD and we're build from git instead of a distribution package.
+    if not os.path.exists("./_generated"):
+        os.system("./generate-evedoc.sh")
+
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
@@ -338,3 +343,11 @@ rst_epilog = """
     "sysconfdir": os.getenv("sysconfdir", "/etc"),
     "localstatedir": os.getenv("localstatedir", "/var"),
 }
+
+# Custom code generate some documentation.
+# evedoc = "./evedoc.py"
+# eve_schema = "../../etc/schema.json"
+# os.makedirs("_generated", exist_ok=True)
+# subprocess.call([evedoc, "--output", "_generated/eve-index.rst", eve_schema])
+# for proto in ["quic", "pgsql"]:
+#     subprocess.call([evedoc, "--output", "_generated/{}.rst".format(proto), "--object", proto, eve_schema])

diff --git a/doc/userguide/generate-evedoc.sh b/doc/userguide/generate-evedoc.sh
new file mode 100755 (executable)
index 0000000..b9dca78
--- /dev/null
+++ b/doc/userguide/generate-evedoc.sh
@@ -0,0 +1,13 @@
+#! /bin/sh
+#
+# Generate RST EVE documentation.
+#
+# This has been broken out of the Makefile so it can be called by
+# make, and Sphinx via conf.py.
+
+set -e
+
+mkdir -p _generated
+../../scripts/evedoc.py --output _generated/eve-index.rst ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/quic.rst --object quic ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/pgsql.rst --object pgsql ../../etc/schema.json

diff --git a/doc/userguide/index.rst b/doc/userguide/index.rst
index 1440fa82801a2d04ae7f1eed1e4ba90920ebed49..7bfead762759143d9fdad35fe8c3fde6dbbd976d 100644 (file)
--- a/doc/userguide/index.rst
+++ b/doc/userguide/index.rst
@@ -34,4 +34,5 @@ This is the documentation for Suricata |version|.
    acknowledgements
    licenses/index.rst
    devguide/index.rst
-   verifying-source-files
\ No newline at end of file
+   verifying-source-files
+   appendix/index.rst

diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst
index e8a5ffc0311922b37a1f63e9a8184afa9d306b3f..952945dffc98671b918cad25590294df9cb3ea78 100644 (file)
--- a/doc/userguide/output/eve/eve-json-format.rst
+++ b/doc/userguide/output/eve/eve-json-format.rst
@@ -2682,6 +2682,10 @@ References:
 .. _PostgreSQL message format - BackendKeyData: https://www.postgresql.org/docs
    /current/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BACKENDKEYDATA
 
+Field Reference
+~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/pgsql.rst
 
 Event type: IKE
 ---------------
@@ -2937,6 +2941,11 @@ Example of QUIC logging with CYU, JA3 and JA4 hashes (note that the JA4 hash is
     "ja4": "q13d0310h3_55b375c5d22e_cd85d2d88918"
   }
 
+Output Reference
+~~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/quic.rst
+
 Event type: DHCP
 -----------------
 
@@ -3048,4 +3057,3 @@ Example of ARP logging: request and response
     "dest_mac": "00:1d:09:f0:92:ab",
     "dest_ip": "10.10.10.1"
   }
-

diff --git a/etc/Makefile.am b/etc/Makefile.am
index 0466d5b7dd42b3611a451d7194134aeb2ea76d99..586cb51ce1cb38454e46b05b02bb2a22878cfc52 100644 (file)
--- a/etc/Makefile.am
+++ b/etc/Makefile.am
@@ -1,6 +1,7 @@
 etcdatadir =   $(datadir)/suricata
 
-EXTRA_DIST =   suricata.logrotate.in \
+EXTRA_DIST =   schema.json \
+               suricata.logrotate.in \
                suricata.service.in
 
 dist_etcdata_DATA =    classification.config \