src/nfa/goughcompile_accel.cpp
src/nfa/goughcompile_internal.h
src/nfa/goughcompile_reg.cpp
+ src/nfa/goughcompile_util.cpp
+ src/nfa/goughcompile_util.h
src/nfa/mcclellan.h
src/nfa/mcclellan_internal.h
src/nfa/mcclellancompile.cpp
#include "util/make_unique.h"
#include "util/multibit_build.h"
#include "util/multibit_internal.h"
+#include "util/report_manager.h"
#include "util/ue2_containers.h"
#include "util/verify_types.h"
#include "grey.h"
return g;
}
+static
+void remapReportsToPrograms(PureRepeat &pr, const ReportManager &rm) {
+ if (pr.reports.empty()) {
+ return;
+ }
+ auto old_reports = pr.reports;
+ pr.reports.clear();
+ for (const auto &r : old_reports) {
+ pr.reports.insert(rm.getProgramOffset(r));
+ }
+}
+
+void remapReportsToPrograms(CastleProto &castle, const ReportManager &rm) {
+ for (auto &m : castle.repeats) {
+ remapReportsToPrograms(m.second, rm);
+ }
+
+ auto old_report_map = castle.report_map;
+ castle.report_map.clear();
+ for (auto &m : old_report_map) {
+ u32 program = rm.getProgramOffset(m.first);
+ castle.report_map[program].insert(begin(m.second), end(m.second));
+ }
+}
+
} // namespace ue2
/*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
class CharReach;
class NGHolder;
+class ReportManager;
struct CompileContext;
/**
std::unique_ptr<NGHolder> makeHolder(const CastleProto &castle, nfa_kind kind,
const CompileContext &cc);
+void remapReportsToPrograms(CastleProto &castle, const ReportManager &rm);
+
} // namespace ue2
#endif // NFA_CASTLECOMPILE_H
--- /dev/null
+/*
+ * Copyright (c) 2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "goughcompile.h"
+#include "goughcompile_util.h"
+#include "mcclellancompile_util.h"
+#include "util/report_manager.h"
+
+#include "ue2common.h"
+
+using namespace std;
+using namespace ue2;
+
+namespace ue2 {
+
+static
+void remapReportsToPrograms(set<som_report> &reports,
+ const ReportManager &rm) {
+ if (reports.empty()) {
+ return;
+ }
+ auto old_reports = reports;
+ reports.clear();
+ for (const auto &r : old_reports) {
+ u32 program = rm.getProgramOffset(r.report);
+ reports.emplace(program, r.slot);
+ }
+}
+
+void remapReportsToPrograms(raw_som_dfa &haig, const ReportManager &rm) {
+ DEBUG_PRINTF("remap haig reports\n");
+
+ for (auto &ds : haig.state_som) {
+ remapReportsToPrograms(ds.reports, rm);
+ remapReportsToPrograms(ds.reports_eod, rm);
+ }
+
+ // McClellan-style reports too.
+ raw_dfa &rdfa = haig;
+ remapReportsToPrograms(rdfa, rm);
+}
+
+} // namespace ue2
--- /dev/null
+/*
+ * Copyright (c) 2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef GOUGHCOMPILE_UTIL_H
+#define GOUGHCOMPILE_UTIL_H
+
+namespace ue2 {
+
+struct raw_som_dfa;
+class ReportManager;
+
+void remapReportsToPrograms(raw_som_dfa &haig, const ReportManager &rm);
+
+} // namespace ue2
+
+#endif // GOUGHCOMPILE_UTIL_H
}
}
+static
+void remapReportsToPrograms(flat_set<ReportID> &reports,
+ const ReportManager &rm) {
+ if (reports.empty()) {
+ return;
+ }
+ auto old_reports = reports;
+ reports.clear();
+ for (const ReportID &id : old_reports) {
+ u32 program = rm.getProgramOffset(id);
+ reports.insert(program);
+ }
+}
+
+void remapReportsToPrograms(raw_dfa &rdfa, const ReportManager &rm) {
+ DEBUG_PRINTF("remap dfa reports\n");
+ for (auto &ds : rdfa.states) {
+ remapReportsToPrograms(ds.reports, rm);
+ remapReportsToPrograms(ds.reports_eod, rm);
+ }
+}
+
+
} // namespace ue2
#include "rdfa.h"
#include "ue2common.h"
+#include "util/report_manager.h"
#include <set>
dstate_id_t get_sds_or_proxy(const raw_dfa &raw);
+void remapReportsToPrograms(raw_dfa &rdfa, const ReportManager &rm);
+
} // namespace ue2
#endif
/*
- * Copyright (c) 2015, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
#include "util/compile_context.h"
#include "util/container.h"
#include "util/graph_range.h"
-#include "util/verify_types.h"
+#include "util/report_manager.h"
#include "util/ue2_containers.h"
+#include "util/verify_types.h"
#include <map>
#include <vector>
return h;
}
+static
+void remapReportsToPrograms(NGHolder &h, const ReportManager &rm) {
+ for (const auto &v : vertices_range(h)) {
+ auto &reports = h[v].reports;
+ if (reports.empty()) {
+ continue;
+ }
+ auto old_reports = reports;
+ reports.clear();
+ for (const ReportID &id : old_reports) {
+ u32 program = rm.getProgramOffset(id);
+ reports.insert(program);
+ }
+ DEBUG_PRINTF("vertex %u: remapped reports {%s} to programs {%s}\n",
+ h[v].index, as_string_list(old_reports).c_str(),
+ as_string_list(reports).c_str());
+ }
+}
+
static
aligned_unique_ptr<NFA>
constructNFA(const NGHolder &h_in, const ReportManager *rm,
set<NFAVertex> zombies = findZombies(*h, br_cyclic, state_ids, cc);
+ if (generates_callbacks(*h)) {
+ assert(rm);
+ remapReportsToPrograms(*h, *rm);
+ }
+
if (!cc.streaming || !cc.grey.compressNFAState) {
compress_state = false;
}
static really_inline
int roseNfaRunProgram(const struct RoseEngine *rose, struct hs_scratch *scratch,
u64a som, u64a offset, ReportID id, const char from_mpv) {
- assert(id < rose->reportProgramCount);
- const u32 *programs = getByOffset(rose, rose->reportProgramOffset);
+ const u32 program = id;
+ assert(program > 0);
+ assert(program % ROSE_INSTR_MIN_ALIGN == 0);
const size_t match_len = 0; // Unused in this path.
const char in_anchored = 0;
const char in_catchup = 1;
- roseRunProgram(rose, scratch, programs[id], som, offset, match_len,
- in_anchored, in_catchup, from_mpv, 0);
+ roseRunProgram(rose, scratch, program, som, offset, match_len, in_anchored,
+ in_catchup, from_mpv, 0);
return can_stop_matching(scratch) ? MO_HALT_MATCHING : MO_CONTINUE_MATCHING;
}
const struct RoseEngine *rose = scratch->core_info.rose;
- assert(id < rose->reportProgramCount);
- const u32 *programs = getByOffset(rose, rose->reportProgramOffset);
-
+ // Our match ID is the program offset.
+ const u32 program = id;
const size_t match_len = 0; // Unused in this path.
const char in_anchored = 0;
const char in_catchup = 0;
const char from_mpv = 0;
const char skip_mpv_catchup = 1;
hwlmcb_rv_t rv =
- roseRunProgram(rose, scratch, programs[id], som, offset, match_len,
+ roseRunProgram(rose, scratch, program, som, offset, match_len,
in_anchored, in_catchup, from_mpv, skip_mpv_catchup);
if (rv == HWLM_TERMINATE_MATCHING) {
return MO_HALT_MATCHING;
#include "hwlm/hwlm.h" /* engine types */
#include "nfa/castlecompile.h"
#include "nfa/goughcompile.h"
+#include "nfa/goughcompile_util.h"
#include "nfa/mcclellancompile.h"
+#include "nfa/mcclellancompile_util.h"
#include "nfa/nfa_api_queue.h"
#include "nfa/nfa_build_util.h"
#include "nfa/nfa_internal.h"
const map<u32, vector<vector<CharReach>>> &triggers,
suffix_id suff, const CompileContext &cc) {
if (suff.castle()) {
- auto n = buildRepeatEngine(*suff.castle(), triggers, cc);
+ auto remapped_castle = *suff.castle();
+ remapReportsToPrograms(remapped_castle, rm);
+ auto n = buildRepeatEngine(remapped_castle, triggers, cc);
assert(n);
return n;
}
if (suff.haig()) {
- auto n = goughCompile(*suff.haig(), ssm.somPrecision(), cc);
+ auto remapped_haig = *suff.haig();
+ remapReportsToPrograms(remapped_haig, rm);
+ auto n = goughCompile(remapped_haig, ssm.somPrecision(), cc);
assert(n);
return n;
}
if (suff.dfa()) {
- auto d = mcclellanCompile(*suff.dfa(), cc);
+ auto remapped_rdfa = *suff.dfa();
+ remapReportsToPrograms(remapped_rdfa, rm);
+ auto d = mcclellanCompile(remapped_rdfa, cc);
assert(d);
return d;
}
// Take a shot at the LBR engine.
if (oneTop) {
- auto lbr = constructLBR(holder, triggers.at(0), cc);
+ auto remapped_holder = cloneHolder(holder);
+ remapReportsToPrograms(*remapped_holder, rm);
+ auto lbr = constructLBR(*remapped_holder, triggers.at(0), cc);
if (lbr) {
return lbr;
}
auto rdfa = buildMcClellan(holder, &rm, false, triggers.at(0),
cc.grey);
if (rdfa) {
+ remapReportsToPrograms(*rdfa, rm);
auto d = mcclellanCompile(*rdfa, cc);
assert(d);
if (cc.grey.roseMcClellanSuffix != 2) {
aligned_unique_ptr<NFA> operator()(unique_ptr<raw_dfa> &rdfa) const {
// Unleash the McClellan!
- return mcclellanCompile(*rdfa, build.cc);
+ raw_dfa tmp(*rdfa);
+ remapReportsToPrograms(tmp, build.rm);
+ return mcclellanCompile(tmp, build.cc);
}
aligned_unique_ptr<NFA> operator()(unique_ptr<raw_som_dfa> &haig) const {
// Unleash the Goughfish!
- return goughCompile(*haig, build.ssm.somPrecision(), build.cc);
+ raw_som_dfa tmp(*haig);
+ remapReportsToPrograms(tmp, build.rm);
+ return goughCompile(tmp, build.ssm.somPrecision(), build.cc);
}
aligned_unique_ptr<NFA> operator()(unique_ptr<NGHolder> &holder) const {
return n;
}
+static
+void remapReportsToPrograms(MpvProto &mpv, const ReportManager &rm) {
+ for (auto &puff : mpv.puffettes) {
+ puff.report = rm.getProgramOffset(puff.report);
+ }
+ for (auto &puff : mpv.triggered_puffettes) {
+ puff.report = rm.getProgramOffset(puff.report);
+ }
+}
+
static
void prepMpv(RoseBuildImpl &tbi, build_context &bc, size_t *historyRequired,
bool *mpv_as_outfix) {
}
auto *mpv = mpv_outfix->mpv();
- auto nfa = mpvCompile(mpv->puffettes, mpv->triggered_puffettes);
+ auto tmp = *mpv; // copy
+ remapReportsToPrograms(tmp, tbi.rm);
+ auto nfa = mpvCompile(tmp.puffettes, tmp.triggered_puffettes);
assert(nfa);
if (!nfa) {
throw CompileError("Unable to generate bytecode.");
auto boundary_out = makeBoundaryPrograms(*this, bc, boundary, dboundary);
+ u32 reportProgramOffset = buildReportPrograms(*this, bc);
+
// Build NFAs
set<u32> no_retrigger_queues;
bool mpv_as_outfix;
u32 eodIterOffset;
tie(eodIterProgramOffset, eodIterOffset) = buildEodAnchorProgram(*this, bc);
- u32 reportProgramOffset = buildReportPrograms(*this, bc);
-
vector<mmbit_sparse_iter> activeLeftIter;
buildActiveLeftIter(leftInfoTable, activeLeftIter);
DEBUG_PRINTF("building rdfa %p\n", rdfa.get());
+ remapReportsToPrograms(*rdfa, rm);
+
u32 start_offset;
u32 small_region;
auto nfa = prepEngine(*rdfa, roseQuality, cc, &start_offset, &small_region);
using namespace testing;
using namespace ue2;
-static const string SCAN_DATA = "___foo______\n___foofoo_foo_^^^^^^^^^^^^^^^^^^^^^^__bar_bar______0_______z_____bar";
+static const string SCAN_DATA = "___foo______\n___foofoo_foo_^^^^^^^^^^^^^^^^^^"
+ "^^^^__bar_bar______0_______z_____bar";
+static const u32 MATCH_REPORT = 1024;
static
int onMatch(u64a, ReportID, void *ctx) {
unique_ptr<NGWrapper> g = buildWrapper(rm, cc, parsed);
ASSERT_TRUE(g != nullptr);
+ rm.setProgramOffset(0, MATCH_REPORT);
+
const map<u32, u32> fixed_depth_tops;
const map<u32, vector<vector<CharReach>>> triggers;
bool compress_state = false;
char rv = nfaQueueExecToMatch(nfa.get(), &q, end);
ASSERT_EQ(MO_MATCHES_PENDING, rv);
ASSERT_EQ(0, matches);
- ASSERT_NE(0, nfaInAcceptState(nfa.get(), 0, &q));
+ ASSERT_NE(0, nfaInAcceptState(nfa.get(), MATCH_REPORT, &q));
nfaReportCurrentMatches(nfa.get(), &q);
ASSERT_EQ(1, matches);
rv = nfaQueueExecToMatch(nfa.get(), &q, end);
ASSERT_EQ(MO_MATCHES_PENDING, rv);
ASSERT_EQ(1, matches);
- ASSERT_NE(0, nfaInAcceptState(nfa.get(), 0, &q));
+ ASSERT_NE(0, nfaInAcceptState(nfa.get(), MATCH_REPORT, &q));
nfaReportCurrentMatches(nfa.get(), &q);
ASSERT_EQ(2, matches);
rv = nfaQueueExecToMatch(nfa.get(), &q, end);
ASSERT_EQ(MO_MATCHES_PENDING, rv);
ASSERT_EQ(2, matches);
- ASSERT_NE(0, nfaInAcceptState(nfa.get(), 0, &q));
+ ASSERT_NE(0, nfaInAcceptState(nfa.get(), MATCH_REPORT, &q));
nfaReportCurrentMatches(nfa.get(), &q);
ASSERT_EQ(3, matches);
pushQueue(&q, MQE_TOP, 0);
pushQueue(&q, MQE_END, end);
- char rv = nfaQueueExecRose(nfa.get(), &q, 0 /* report id */);
+ char rv = nfaQueueExecRose(nfa.get(), &q, MATCH_REPORT);
ASSERT_EQ(MO_MATCHES_PENDING, rv);
pushQueue(&q, MQE_START, end);
- ASSERT_NE(0, nfaInAcceptState(nfa.get(), 0, &q));
+ ASSERT_NE(0, nfaInAcceptState(nfa.get(), MATCH_REPORT, &q));
}
TEST_P(LimExModelTest, CheckFinalState) {
unique_ptr<NGWrapper> g = buildWrapper(rm, cc, parsed);
ASSERT_TRUE(g != nullptr);
+ rm.setProgramOffset(0, MATCH_REPORT);
+
const map<u32, u32> fixed_depth_tops;
const map<u32, vector<vector<CharReach>>> triggers;
bool compress_state = false;
projects / thirdparty / vectorscan.git / commitdiff
