hsbench: add stream compress functionality

diff --git a/tools/hsbench/common.h b/tools/hsbench/common.h
index efff3f99da558b39b217a18488b03435770b3314..a82959110f983ddff74d7552411e343b105bb77f 100644 (file)
--- a/tools/hsbench/common.h
+++ b/tools/hsbench/common.h
@@ -40,5 +40,6 @@ extern std::string serializePath;
 extern unsigned int somPrecisionMode;
 extern bool forceEditDistance;
 extern unsigned editDistance;
+extern bool printCompressSize;
 
 #endif // COMMON_H

diff --git a/tools/hsbench/engine_hyperscan.cpp b/tools/hsbench/engine_hyperscan.cpp
index 9674e5c8493100c5e410c7390b2f2494482af89b..5f188472c2639687d1fdf3e1546c3ee221dcedc1 100644 (file)
--- a/tools/hsbench/engine_hyperscan.cpp
+++ b/tools/hsbench/engine_hyperscan.cpp
@@ -205,6 +205,35 @@ void EngineHyperscan::streamScan(EngineStream &stream, const char *data,
     }
 }
 
+void EngineHyperscan::streamCompressExpand(EngineStream &stream,
+                                           vector<char> &temp) const {
+    size_t used = 0;
+    hs_error_t err = hs_compress_stream(stream.id, temp.data(), temp.size(),
+                                        &used);
+    if (err == HS_INSUFFICIENT_SPACE) {
+        temp.resize(used);
+        err = hs_compress_stream(stream.id, temp.data(), temp.size(), &used);
+    }
+
+    if (err != HS_SUCCESS) {
+        printf("Fatal error: hs_compress_stream returned error %d\n", err);
+        abort();
+    }
+
+    if (printCompressSize) {
+        printf("stream %u: compressed to %zu\n", stream.sn, used);
+    }
+
+    err = hs_reset_and_expand_stream(stream.id, temp.data(), temp.size(),
+                                     nullptr, nullptr, nullptr);
+
+    if (err != HS_SUCCESS) {
+        printf("Fatal error: hs_reset_and expand_stream returned error %d\n",
+               err);
+        abort();
+    }
+}
+
 static
 unsigned makeModeFlags(ScanMode scan_mode) {
     switch (scan_mode) {

diff --git a/tools/hsbench/engine_hyperscan.h b/tools/hsbench/engine_hyperscan.h
index 7875decc2946a1abac5e851c7fae462f3afce743..2c93959b83a737bb415d7a08b3b072d5f293681c 100644 (file)
--- a/tools/hsbench/engine_hyperscan.h
+++ b/tools/hsbench/engine_hyperscan.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Intel Corporation
+ * Copyright (c) 2016-2017, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -34,6 +34,7 @@
 #include "hs_runtime.h"
 
 #include <memory>
+#include <vector>
 
 /** Structure for the result of a single complete scan. */
 struct ResultEntry {
@@ -79,6 +80,9 @@ public:
     void streamClose(std::unique_ptr<EngineStream> stream,
                      ResultEntry &result) const;
 
+    void streamCompressExpand(EngineStream &stream,
+                              std::vector<char> &temp) const;
+
     void streamScan(EngineStream &stream, const char *data, unsigned int len,
                     unsigned int id, ResultEntry &result) const;
 

diff --git a/tools/hsbench/main.cpp b/tools/hsbench/main.cpp
index 9c5fd6cbc848bc1a1da252559c1f75305f89d64a..f2ea8e7e734d3dca44e8be89a713cb2c95e3aef6 100644 (file)
--- a/tools/hsbench/main.cpp
+++ b/tools/hsbench/main.cpp
@@ -77,10 +77,13 @@ string serializePath("");
 unsigned int somPrecisionMode = HS_MODE_SOM_HORIZON_LARGE;
 bool forceEditDistance = false;
 unsigned editDistance = 0;
+bool printCompressSize = false;
+
+// Globals local to this file.
+static bool compressStream = false;
 
 namespace /* anonymous */ {
 
-// Globals local to this file.
 bool display_per_scan = false;
 ScanMode scan_mode = ScanMode::STREAMING;
 unsigned repeats = 20;
@@ -212,11 +215,15 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
     int in_sigfile = 0;
     int do_per_scan = 0;
     int do_echo_matches = 0;
+    int do_compress = 0;
+    int do_compress_size = 0;
     vector<string> sigFiles;
 
     static struct option longopts[] = {
         {"per-scan", 0, &do_per_scan, 1},
         {"echo-matches", 0, &do_echo_matches, 1},
+        {"compress-stream", 0, &do_compress, 1},
+        {"print-compress-size", 0, &do_compress_size, 1},
         {nullptr, 0, nullptr, 0}
     };
 
@@ -338,6 +345,12 @@ void processArgs(int argc, char *argv[], vector<BenchmarkSigs> &sigSets,
     if (do_per_scan) {
         display_per_scan = true;
     }
+    if (do_compress) {
+        compressStream = true;
+    }
+    if (do_compress_size) {
+        printCompressSize = true;
+    }
 
     if (exprPath.empty() && !sigFiles.empty()) {
         /* attempt to infer an expression directory */
@@ -470,10 +483,12 @@ vector<StreamInfo> prepStreamingData(const ThreadContext *ctx) {
 }
 
 static
-void benchStreamingInternal(ThreadContext *ctx, vector<StreamInfo> &streams) {
+void benchStreamingInternal(ThreadContext *ctx, vector<StreamInfo> &streams,
+                            bool do_compress) {
     assert(ctx);
     const EngineHyperscan &e = ctx->engine;
     const vector<DataBlock> &blocks = ctx->corpus_data;
+    vector<char> compress_buf(do_compress ? 1000 : 0);
 
     for (ResultEntry &r : ctx->results) {
         ctx->timer.start();
@@ -491,6 +506,8 @@ void benchStreamingInternal(ThreadContext *ctx, vector<StreamInfo> &streams) {
                     printf("Fatal error: stream open failed!\n");
                     exit(1);
                 }
+            } else if (do_compress) {
+                e.streamCompressExpand(*stream.eng_handle, compress_buf);
             }
 
             assert(stream.eng_handle);
@@ -521,7 +538,7 @@ void benchStreaming(void *context) {
 
     startTotalTimer(ctx);
 
-    benchStreamingInternal(ctx, streams);
+    benchStreamingInternal(ctx, streams, false);
 
     // Synchronization point
     ctx->barrier();
@@ -530,6 +547,26 @@ void benchStreaming(void *context) {
     stopTotalTimer(ctx);
 }
 
+static
+void benchStreamingCompress(void *context) {
+    ThreadContext *ctx = (ThreadContext *)context;
+    vector<StreamInfo> streams = prepStreamingData(ctx);
+
+    // Synchronization point
+    ctx->barrier();
+
+    startTotalTimer(ctx);
+
+    benchStreamingInternal(ctx, streams, true);
+
+    // Synchronization point
+    ctx->barrier();
+
+    // Now that all threads are finished, we can stop the clock.
+    stopTotalTimer(ctx);
+}
+
+
 /** In-memory structure for a data block to be scanned in vectored mode. */
 struct VectoredInfo {
     vector<const char *> data;
@@ -704,7 +741,11 @@ unique_ptr<ThreadContext> makeThreadContext(const EngineHyperscan &db,
     thread_func_t fn = nullptr;
     switch (scan_mode) {
     case ScanMode::STREAMING:
-        fn = benchStreaming;
+        if (compressStream) {
+            fn = benchStreamingCompress;
+        } else {
+            fn = benchStreaming;
+        }
         break;
     case ScanMode::VECTORED:
         fn = benchVectored;