From: Philippe Antoine Date: Mon, 22 Nov 2021 16:16:08 +0000 (+0100) Subject: jsonschema: use rust utility X-Git-Tag: suricata-5.0.10~33 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9ffb63314824acdf9d8e487011389018f426653d;p=thirdparty%2Fsuricata-verify.git jsonschema: use rust utility As it is faster than python --- diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 99cea7b79..124c6af7a 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -59,6 +59,7 @@ jobs: libjansson-dev \ libpython2.7 \ libnss3-dev \ + libssl-dev \ make \ parallel \ python3-distutils \ @@ -71,6 +72,10 @@ jobs: - run: cargo install --force --debug cbindgen - run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH - uses: actions/checkout@v2 + - name: Build eve-validator + working-directory: eve-validator + run: | + cargo install --path . - run: python3 ./run.py --self-test - run: git clone https://github.com/OISF/suricata -b ${{ matrix.branch }} - run: git clone https://github.com/OISF/libhtp suricata/libhtp @@ -126,6 +131,7 @@ jobs: libevent-devel \ libmaxminddb-devel \ libpcap-devel \ + openssl-devel \ libtool \ lz4-devel \ make \ @@ -144,6 +150,10 @@ jobs: - run: cargo install --force --debug cbindgen - run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH - uses: actions/checkout@v2 + - name: Build eve-validator + working-directory: eve-validator + run: | + cargo install --path . - run: python3 ./run.py --self-test - run: git clone https://github.com/OISF/suricata -b ${{ matrix.branch }} - run: git clone https://github.com/OISF/libhtp suricata/libhtp diff --git a/check-eve.py b/check-eve.py index 18e08e9a9..92cebd768 100755 --- a/check-eve.py +++ b/check-eve.py @@ -27,26 +27,29 @@ import os import os.path import argparse import json +import subprocess from jsonschema import validate from jsonschema.exceptions import ValidationError -def validate_json(args, dirpath, schema, isDirectory): - json_filename = dirpath - if isDirectory: - json_filename = os.path.join(dirpath, 'eve.json') - +def validate_json(args, json_filename, schema): status = "OK" errors = [] - with open(json_filename) as f: - for line in f: - obj = json.loads(line) - try: - validate(instance = obj, schema=schema) - except ValidationError as err: - status = "FAIL" - errors.append(err.message) - + if not args.python_validator: + cp = subprocess.run(["eve-validator", "-q", "-s", schema, "--", json_filename]) + if cp.returncode != 0: + status = "FAIL" + errors.append(cp.stdout) + else: + with open(json_filename) as f: + for line in f: + obj = json.loads(line) + try: + validate(instance = obj, schema=schema) + except ValidationError as err: + status = "FAIL" + errors.append(err.message) + if not args.quiet: if status == "FAIL": print("===> %s: FAIL " % json_filename) @@ -63,6 +66,7 @@ def main(): parser = argparse.ArgumentParser(description="Validation schema") parser.add_argument("-v", dest="verbose", action="store_true") + parser.add_argument("-p", dest="python_validator", action="store_true", help="use python validator") parser.add_argument("file", nargs="?", default=[]) parser.add_argument("-q", dest="quiet", action="store_true") args = parser.parse_args() @@ -70,7 +74,11 @@ def main(): tdir = os.path.join(TOPDIR, "tests") json_path = "{}/schema.json".format(TOPDIR) - schema = json.load(open(json_path)) + + if args.python_validator: + schema = json.load(open(json_path)) + else: + schema = json_path checked = 0 passed = 0 @@ -78,12 +86,12 @@ def main(): isDirectory = True argfile = args.file - + if argfile: # if the argument is a single file if os.path.isfile(argfile): isDirectory = False - status = validate_json(args, argfile, schema, isDirectory) + status = validate_json(args, argfile, schema) checked += 1 if status == "OK": passed += 1 @@ -93,12 +101,12 @@ def main(): # if the argument is a directory elif os.path.isdir(argfile): tdir = argfile - + if isDirectory: # os.walk for eve.json files and validate each one for dirpath, dirnames, filenames in os.walk(tdir): if 'eve.json' in filenames: - status = validate_json(args, dirpath, schema, isDirectory) + status = validate_json(args, os.path.join(dirpath, 'eve.json'), schema) checked += 1 if status == "OK": passed += 1 diff --git a/eve-validator/Cargo.toml b/eve-validator/Cargo.toml new file mode 100644 index 000000000..33502f0cc --- /dev/null +++ b/eve-validator/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "eve-validator" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +jsonschema = "0.13" +clap = { version = "3.1.6", features = ["derive"] } +serde_json = "1" diff --git a/eve-validator/src/main.rs b/eve-validator/src/main.rs new file mode 100644 index 000000000..db3b7e771 --- /dev/null +++ b/eve-validator/src/main.rs @@ -0,0 +1,79 @@ +use std::{error::Error, fs, path::PathBuf, process}; + +use clap::Parser; +use jsonschema::JSONSchema; + +type BoxErrorResult = Result>; + +use std::fs::File; +use std::io::BufReader; + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Cli { + /// A path to a JSON instance (i.e. filename.json) to validate (may be specified multiple times). + #[clap(last = true)] + instances: Vec, + + /// The JSON Schema to validate with (i.e. schema.json). + #[clap(short, long)] + schema: PathBuf, + + /// Quiet output + #[clap(short, long)] + quiet: bool, +} + +pub fn main() -> BoxErrorResult<()> { + let config = Cli::parse(); + + let success = validate_instances(&config.instances, config.schema, config.quiet)?; + + if !success { + process::exit(1); + } + + Ok(()) +} + +fn validate_instances(instances: &[PathBuf], schema: PathBuf, quiet: bool) -> BoxErrorResult { + let mut success = true; + + let schema_json = fs::read_to_string(schema)?; + let schema_json = serde_json::from_str(&schema_json)?; + match JSONSchema::compile(&schema_json) { + Ok(schema) => { + for instance in instances { + let instance_path_name = instance.to_str().unwrap(); + let file = File::open(instance_path_name)?; + let reader = BufReader::new(file); + let deserializer = serde_json::Deserializer::from_reader(reader); + let iterator = deserializer.into_iter::(); + let mut success_i = true; + for item in iterator { + let instance_json = item?; + let validation = schema.validate(&instance_json); + match validation { + Ok(_) => {} + Err(errors) => { + success = false; + success_i = false; + println!("{} - INVALID. Errors:", instance_path_name); + for (i, e) in errors.enumerate() { + println!("{}.{} {}", i + 1, e.instance_path, e); + } + } + } + } + if success_i && !quiet { + println!("{} - VALID", instance_path_name); + } + } + } + Err(error) => { + println!("Schema is invalid. Error: {}", error); + success = false; + } + } + Ok(success) +} diff --git a/run.py b/run.py index 292924e87..0c007c827 100755 --- a/run.py +++ b/run.py @@ -46,8 +46,10 @@ import yaml # Check if we can validate EVE files against the schema. try: - import jsonschema VALIDATE_EVE = True + check_output = subprocess.run(["eve-validator", "-V"], capture_output=True) + if check_output.returncode != 0: + VALIDATE_EVE = False except: VALIDATE_EVE = False @@ -949,7 +951,7 @@ def main(): return unittest.main(argv=[sys.argv[0]]) if not VALIDATE_EVE: - print("Warning: EVE files will not be valided: jsonschema module not found.") + print("Warning: EVE files will not be valided: eve-validator program not found.") TOPDIR = os.path.abspath(os.path.dirname(sys.argv[0]))