src/hwlm/noodle_build.h
src/hwlm/noodle_internal.h
src/nfa/accel.h
+ src/nfa/accel_dfa_build_strat.cpp
+ src/nfa/accel_dfa_build_strat.h
src/nfa/accelcompile.cpp
src/nfa/accelcompile.h
src/nfa/callback.h
src/nfa/castlecompile.cpp
src/nfa/castlecompile.h
+ src/nfa/dfa_build_strat.cpp
+ src/nfa/dfa_build_strat.h
src/nfa/dfa_min.cpp
src/nfa/dfa_min.h
src/nfa/goughcompile.cpp
src/nfa/mcclellan_internal.h
src/nfa/mcclellancompile.cpp
src/nfa/mcclellancompile.h
- src/nfa/mcclellancompile_accel.cpp
- src/nfa/mcclellancompile_accel.h
src/nfa/mcclellancompile_util.cpp
src/nfa/mcclellancompile_util.h
src/nfa/limex_compile.cpp
/*
- * Copyright (c) 2016, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* POSSIBILITY OF SUCH DAMAGE.
*/
-#include "mcclellancompile_accel.h"
-
-#include "mcclellancompile_util.h"
+#include "accel_dfa_build_strat.h"
+#include "accel.h"
#include "grey.h"
#include "nfagraph/ng_limex_accel.h"
+#include "shufticompile.h"
+#include "trufflecompile.h"
#include "util/charreach.h"
#include "util/container.h"
#include "util/dump_charclass.h"
+#include "util/verify_types.h"
-#include <vector>
#include <sstream>
+#include <vector>
#define PATHS_LIMIT 500
namespace ue2 {
namespace {
-
struct path {
vector<CharReach> reach;
dstate_id_t dest = DEAD_STATE;
- explicit path(dstate_id_t base) : dest(base) {}
+ explicit path(dstate_id_t base) : dest(base) {
+ }
+};
};
-
-}
static UNUSED
string describeClasses(const vector<CharReach> &v) {
goto next;
}
}
- DEBUG_PRINTF("better: [%s] -> %u\n",
- describeClasses(g.reach).c_str(), g.dest);
+ DEBUG_PRINTF("better: [%s] -> %u\n", describeClasses(g.reach).c_str(),
+ g.dest);
return false;
next:;
static
void extend(const raw_dfa &rdfa, const path &p,
- map<u32, vector<path> > &all,
- vector<path> &out) {
+ map<u32, vector<path>> &all, vector<path> &out) {
dstate s = rdfa.states[p.dest];
if (!p.reach.empty() && p.reach.back().none()) {
}
DEBUG_PRINTF("----good: [%s] -> %u\n",
- describeClasses(pp.reach).c_str(), pp.dest);
+ describeClasses(pp.reach).c_str(), pp.dest);
all[e.first].push_back(pp);
out.push_back(pp);
}
}
static
-vector<vector<CharReach> > generate_paths(const raw_dfa &rdfa, dstate_id_t base,
- u32 len) {
- vector<path> paths{ path(base) };
- map<u32, vector<path> > all;
+vector<vector<CharReach>> generate_paths(const raw_dfa &rdfa,
+ dstate_id_t base, u32 len) {
+ vector<path> paths{path(base)};
+ map<u32, vector<path>> all;
all[base].push_back(path(base));
for (u32 i = 0; i < len && paths.size() < PATHS_LIMIT; i++) {
vector<path> next_gen;
dump_paths(paths);
- vector<vector<CharReach> > rv;
+ vector<vector<CharReach>> rv;
for (auto &p : paths) {
rv.push_back(move(p.reach));
}
AccelScheme look_for_offset_accel(const raw_dfa &rdfa, dstate_id_t base,
u32 max_allowed_accel_offset) {
DEBUG_PRINTF("looking for accel for %hu\n", base);
- vector<vector<CharReach> > paths = generate_paths(rdfa, base,
- max_allowed_accel_offset + 1);
+ vector<vector<CharReach>> paths =
+ generate_paths(rdfa, base, max_allowed_accel_offset + 1);
AccelScheme as = findBestAccelScheme(paths, CharReach(), true);
DEBUG_PRINTF("found %s + %u\n", describeClass(as.cr).c_str(), as.offset);
return as;
}
+static UNUSED
+bool better(const AccelScheme &a, const AccelScheme &b) {
+ if (!a.double_byte.empty() && b.double_byte.empty()) {
+ return true;
+ }
+
+ if (!b.double_byte.empty()) {
+ return false;
+ }
+
+ return a.cr.count() < b.cr.count();
+}
+
+static
+vector<CharReach> reverse_alpha_remapping(const raw_dfa &rdfa) {
+ vector<CharReach> rv(rdfa.alpha_size - 1); /* TOP not required */
+
+ for (u32 i = 0; i < N_CHARS; i++) {
+ rv.at(rdfa.alpha_remap[i]).set(i);
+ }
+
+ return rv;
+}
+
+static
+bool double_byte_ok(const AccelScheme &info) {
+ return !info.double_byte.empty() &&
+ info.double_cr.count() < info.double_byte.size() &&
+ info.double_cr.count() <= 2 && !info.double_byte.empty();
+}
+
+static
+bool has_self_loop(dstate_id_t s, const raw_dfa &raw) {
+ u16 top_remap = raw.alpha_remap[TOP];
+ for (u32 i = 0; i < raw.states[s].next.size(); i++) {
+ if (i != top_remap && raw.states[s].next[i] == s) {
+ return true;
+ }
+ }
+ return false;
+}
+
static
vector<u16> find_nonexit_symbols(const raw_dfa &rdfa,
- const CharReach &escape) {
+ const CharReach &escape) {
set<u16> rv;
CharReach nonexit = ~escape;
for (auto i = nonexit.find_first(); i != CharReach::npos;
return vector<u16>(rv.begin(), rv.end());
}
+static
+dstate_id_t get_sds_or_proxy(const raw_dfa &raw) {
+ if (raw.start_floating != DEAD_STATE) {
+ DEBUG_PRINTF("has floating start\n");
+ return raw.start_floating;
+ }
+
+ DEBUG_PRINTF("looking for SDS proxy\n");
+
+ dstate_id_t s = raw.start_anchored;
+
+ if (has_self_loop(s, raw)) {
+ return s;
+ }
+
+ u16 top_remap = raw.alpha_remap[TOP];
+
+ ue2::unordered_set<dstate_id_t> seen;
+ while (true) {
+ seen.insert(s);
+ DEBUG_PRINTF("basis %hu\n", s);
+
+ /* check if we are connected to a state with a self loop */
+ for (u32 i = 0; i < raw.states[s].next.size(); i++) {
+ dstate_id_t t = raw.states[s].next[i];
+ if (i != top_remap && t != DEAD_STATE && has_self_loop(t, raw)) {
+ return t;
+ }
+ }
+
+ /* find a neighbour to use as a basis for looking for the sds proxy */
+ dstate_id_t t = DEAD_STATE;
+ for (u32 i = 0; i < raw.states[s].next.size(); i++) {
+ dstate_id_t tt = raw.states[s].next[i];
+ if (i != top_remap && tt != DEAD_STATE && !contains(seen, tt)) {
+ t = tt;
+ break;
+ }
+ }
+
+ if (t == DEAD_STATE) {
+ /* we were unable to find a state to use as a SDS proxy */
+ return DEAD_STATE;
+ }
+
+ s = t;
+ }
+}
+
static
set<dstate_id_t> find_region(const raw_dfa &rdfa, dstate_id_t base,
- const AccelScheme &ei) {
+ const AccelScheme &ei) {
DEBUG_PRINTF("looking for region around %hu\n", base);
set<dstate_id_t> region = {base};
return region;
}
-static
-bool better(const AccelScheme &a, const AccelScheme &b) {
- if (!a.double_byte.empty() && b.double_byte.empty()) {
- return true;
- }
-
- if (!b.double_byte.empty()) {
- return false;
- }
-
- return a.cr.count() < b.cr.count();
-}
-
-static
-vector<CharReach> reverse_alpha_remapping(const raw_dfa &rdfa) {
- vector<CharReach> rv(rdfa.alpha_size - 1); /* TOP not required */
-
- for (u32 i = 0; i < N_CHARS; i++) {
- rv.at(rdfa.alpha_remap[i]).set(i);
- }
-
- return rv;
-}
-
-map<dstate_id_t, AccelScheme> populateAccelerationInfo(const raw_dfa &rdfa,
- const dfa_build_strat &strat,
- const Grey &grey) {
- map<dstate_id_t, AccelScheme> rv;
- if (!grey.accelerateDFA) {
- return rv;
- }
-
- dstate_id_t sds_proxy = get_sds_or_proxy(rdfa);
- DEBUG_PRINTF("sds %hu\n", sds_proxy);
-
- for (size_t i = 0; i < rdfa.states.size(); i++) {
- if (i == DEAD_STATE) {
- continue;
- }
-
- /* Note on report acceleration states: While we can't accelerate while we
- * are spamming out callbacks, the QR code paths don't raise reports
- * during scanning so they can accelerate report states. */
- if (generates_callbacks(rdfa.kind) && !rdfa.states[i].reports.empty()) {
- continue;
- }
-
- size_t single_limit = i == sds_proxy ? ACCEL_DFA_MAX_FLOATING_STOP_CHAR
- : ACCEL_DFA_MAX_STOP_CHAR;
- DEBUG_PRINTF("inspecting %zu/%hu: %zu\n", i, sds_proxy, single_limit);
-
- AccelScheme ei = strat.find_escape_strings(i);
- if (ei.cr.count() > single_limit) {
- DEBUG_PRINTF("state %zu is not accelerable has %zu\n", i,
- ei.cr.count());
- continue;
- }
-
- DEBUG_PRINTF("state %zu should be accelerable %zu\n",
- i, ei.cr.count());
-
- rv[i] = ei;
- }
-
- /* provide accleration states to states in the region of sds */
- if (contains(rv, sds_proxy)) {
- AccelScheme sds_ei = rv[sds_proxy];
- sds_ei.double_byte.clear(); /* region based on single byte scheme
- * may differ from double byte */
- DEBUG_PRINTF("looking to expand offset accel to nearby states, %zu\n",
- sds_ei.cr.count());
- auto sds_region = find_region(rdfa, sds_proxy, sds_ei);
- for (auto s : sds_region) {
- if (!contains(rv, s) || better(sds_ei, rv[s])) {
- rv[s] = sds_ei;
- }
- }
- }
-
- return rv;
-}
-
-static
-bool double_byte_ok(const AccelScheme &info) {
- return !info.double_byte.empty()
- && info.double_cr.count() < info.double_byte.size()
- && info.double_cr.count() <= 2 && !info.double_byte.empty();
-}
-
-AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa, dstate_id_t this_idx,
- u32 max_allowed_accel_offset) {
+AccelScheme
+accel_dfa_build_strat::find_escape_strings(dstate_id_t this_idx) const {
AccelScheme rv;
+ const raw_dfa &rdfa = get_raw();
rv.cr.clear();
rv.offset = 0;
const dstate &raw = rdfa.states[this_idx];
if (!raw_next.reports.empty() && generates_callbacks(rdfa.kind)) {
DEBUG_PRINTF("leads to report\n");
- outs2_broken = true; /* cannot accelerate over reports */
+ outs2_broken = true; /* cannot accelerate over reports */
continue;
}
succs[next_id] |= cr_i;
DEBUG_PRINTF("this %u, sds proxy %hu\n", this_idx, get_sds_or_proxy(rdfa));
DEBUG_PRINTF("broken %d\n", outs2_broken);
- if (!double_byte_ok(rv) && !is_triggered(rdfa.kind)
- && this_idx == rdfa.start_floating
- && this_idx != DEAD_STATE) {
+ if (!double_byte_ok(rv) && !is_triggered(rdfa.kind) &&
+ this_idx == rdfa.start_floating && this_idx != DEAD_STATE) {
DEBUG_PRINTF("looking for offset accel at %u\n", this_idx);
- auto offset = look_for_offset_accel(rdfa, this_idx,
- max_allowed_accel_offset);
- DEBUG_PRINTF("width %zu vs %zu\n", offset.cr.count(),
- rv.cr.count());
+ auto offset =
+ look_for_offset_accel(rdfa, this_idx, max_allowed_offset_accel());
+ DEBUG_PRINTF("width %zu vs %zu\n", offset.cr.count(), rv.cr.count());
if (double_byte_ok(offset) || offset.cr.count() < rv.cr.count()) {
DEBUG_PRINTF("using offset accel\n");
rv = offset;
return rv;
}
+void
+accel_dfa_build_strat::buildAccel(UNUSED dstate_id_t this_idx,
+ const AccelScheme &info,
+ void *accel_out) {
+ AccelAux *accel = (AccelAux *)accel_out;
+
+ DEBUG_PRINTF("accelerations scheme has offset s%u/d%u\n", info.offset,
+ info.double_offset);
+ accel->generic.offset = verify_u8(info.offset);
+
+ if (double_byte_ok(info) && info.double_cr.none() &&
+ info.double_byte.size() == 1) {
+ accel->accel_type = ACCEL_DVERM;
+ accel->dverm.c1 = info.double_byte.begin()->first;
+ accel->dverm.c2 = info.double_byte.begin()->second;
+ accel->dverm.offset = verify_u8(info.double_offset);
+ DEBUG_PRINTF("state %hu is double vermicelli\n", this_idx);
+ return;
+ }
+
+ if (double_byte_ok(info) && info.double_cr.none() &&
+ (info.double_byte.size() == 2 || info.double_byte.size() == 4)) {
+ bool ok = true;
+
+ assert(!info.double_byte.empty());
+ u8 firstC = info.double_byte.begin()->first & CASE_CLEAR;
+ u8 secondC = info.double_byte.begin()->second & CASE_CLEAR;
+
+ for (const pair<u8, u8> &p : info.double_byte) {
+ if ((p.first & CASE_CLEAR) != firstC ||
+ (p.second & CASE_CLEAR) != secondC) {
+ ok = false;
+ break;
+ }
+ }
+
+ if (ok) {
+ accel->accel_type = ACCEL_DVERM_NOCASE;
+ accel->dverm.c1 = firstC;
+ accel->dverm.c2 = secondC;
+ accel->dverm.offset = verify_u8(info.double_offset);
+ DEBUG_PRINTF("state %hu is nc double vermicelli\n", this_idx);
+ return;
+ }
+
+ u8 m1;
+ u8 m2;
+ if (buildDvermMask(info.double_byte, &m1, &m2)) {
+ accel->accel_type = ACCEL_DVERM_MASKED;
+ accel->dverm.offset = verify_u8(info.double_offset);
+ accel->dverm.c1 = info.double_byte.begin()->first & m1;
+ accel->dverm.c2 = info.double_byte.begin()->second & m2;
+ accel->dverm.m1 = m1;
+ accel->dverm.m2 = m2;
+ DEBUG_PRINTF(
+ "building maskeddouble-vermicelli for 0x%02hhx%02hhx\n",
+ accel->dverm.c1, accel->dverm.c2);
+ return;
+ }
+ }
+
+ if (double_byte_ok(info) &&
+ shuftiBuildDoubleMasks(info.double_cr, info.double_byte,
+ &accel->dshufti.lo1, &accel->dshufti.hi1,
+ &accel->dshufti.lo2, &accel->dshufti.hi2)) {
+ accel->accel_type = ACCEL_DSHUFTI;
+ accel->dshufti.offset = verify_u8(info.double_offset);
+ DEBUG_PRINTF("state %hu is double shufti\n", this_idx);
+ return;
+ }
+
+ if (info.cr.none()) {
+ accel->accel_type = ACCEL_RED_TAPE;
+ DEBUG_PRINTF("state %hu is a dead end full of bureaucratic red tape"
+ " from which there is no escape\n",
+ this_idx);
+ return;
+ }
+
+ if (info.cr.count() == 1) {
+ accel->accel_type = ACCEL_VERM;
+ accel->verm.c = info.cr.find_first();
+ DEBUG_PRINTF("state %hu is vermicelli\n", this_idx);
+ return;
+ }
+
+ if (info.cr.count() == 2 && info.cr.isCaselessChar()) {
+ accel->accel_type = ACCEL_VERM_NOCASE;
+ accel->verm.c = info.cr.find_first() & CASE_CLEAR;
+ DEBUG_PRINTF("state %hu is caseless vermicelli\n", this_idx);
+ return;
+ }
+
+ if (info.cr.count() > max_floating_stop_char()) {
+ accel->accel_type = ACCEL_NONE;
+ DEBUG_PRINTF("state %hu is too broad\n", this_idx);
+ return;
+ }
+
+ accel->accel_type = ACCEL_SHUFTI;
+ if (-1 != shuftiBuildMasks(info.cr, &accel->shufti.lo, &accel->shufti.hi)) {
+ DEBUG_PRINTF("state %hu is shufti\n", this_idx);
+ return;
+ }
+
+ assert(!info.cr.none());
+ accel->accel_type = ACCEL_TRUFFLE;
+ truffleBuildMasks(info.cr, &accel->truffle.mask1, &accel->truffle.mask2);
+ DEBUG_PRINTF("state %hu is truffle\n", this_idx);
}
+
+map<dstate_id_t, AccelScheme>
+accel_dfa_build_strat::getAccelInfo(const Grey &grey) {
+ map<dstate_id_t, AccelScheme> rv;
+ raw_dfa &rdfa = get_raw();
+ if (!grey.accelerateDFA) {
+ return rv;
+ }
+
+ dstate_id_t sds_proxy = get_sds_or_proxy(rdfa);
+ DEBUG_PRINTF("sds %hu\n", sds_proxy);
+
+ for (size_t i = 0; i < rdfa.states.size(); i++) {
+ if (i == DEAD_STATE) {
+ continue;
+ }
+
+ /* Note on report acceleration states: While we can't accelerate while
+ * we
+ * are spamming out callbacks, the QR code paths don't raise reports
+ * during scanning so they can accelerate report states. */
+ if (generates_callbacks(rdfa.kind) && !rdfa.states[i].reports.empty()) {
+ continue;
+ }
+
+ size_t single_limit =
+ i == sds_proxy ? max_floating_stop_char() : max_stop_char();
+ DEBUG_PRINTF("inspecting %zu/%hu: %zu\n", i, sds_proxy, single_limit);
+
+ AccelScheme ei = find_escape_strings(i);
+ if (ei.cr.count() > single_limit) {
+ DEBUG_PRINTF("state %zu is not accelerable has %zu\n", i,
+ ei.cr.count());
+ continue;
+ }
+
+ DEBUG_PRINTF("state %zu should be accelerable %zu\n", i, ei.cr.count());
+
+ rv[i] = ei;
+ }
+
+ /* provide accleration states to states in the region of sds */
+ if (contains(rv, sds_proxy)) {
+ AccelScheme sds_ei = rv[sds_proxy];
+ sds_ei.double_byte.clear(); /* region based on single byte scheme
+ * may differ from double byte */
+ DEBUG_PRINTF("looking to expand offset accel to nearby states, %zu\n",
+ sds_ei.cr.count());
+ auto sds_region = find_region(rdfa, sds_proxy, sds_ei);
+ for (auto s : sds_region) {
+ if (!contains(rv, s) || better(sds_ei, rv[s])) {
+ rv[s] = sds_ei;
+ }
+ }
+ }
+
+ return rv;
+}
+};
/*
- * Copyright (c) 2016, Intel Corporation
+ * Copyright (c) 2015-2016, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* POSSIBILITY OF SUCH DAMAGE.
*/
-#ifndef MCCLELLANCOMPILE_ACCEL_H
-#define MCCLELLANCOMPILE_ACCEL_H
+#ifndef ACCEL_DFA_BUILD_STRAT_H
+#define ACCEL_DFA_BUILD_STRAT_H
-#include "mcclellancompile.h"
+#include "rdfa.h"
+#include "dfa_build_strat.h"
+#include "ue2common.h"
+#include "util/accel_scheme.h"
#include <map>
namespace ue2 {
+class ReportManager;
struct Grey;
-#define ACCEL_DFA_MAX_OFFSET_DEPTH 4
-
-/** Maximum tolerated number of escape character from an accel state.
- * This is larger than nfa, as we don't have a budget and the nfa cheats on stop
- * characters for sets of states */
-#define ACCEL_DFA_MAX_STOP_CHAR 160
-
-/** Maximum tolerated number of escape character from a sds accel state. Larger
- * than normal states as accelerating sds is important. Matches NFA value */
-#define ACCEL_DFA_MAX_FLOATING_STOP_CHAR 192
-
-std::map<dstate_id_t, AccelScheme> populateAccelerationInfo(const raw_dfa &rdfa,
- const dfa_build_strat &strat,
- const Grey &grey);
-
-AccelScheme find_mcclellan_escape_info(const raw_dfa &rdfa,
- dstate_id_t this_idx,
- u32 max_allowed_accel_offset);
-
-}
-
-#endif
+class accel_dfa_build_strat : public dfa_build_strat {
+public:
+ explicit accel_dfa_build_strat(const ReportManager &rm_in)
+ : dfa_build_strat(rm_in) {}
+ virtual AccelScheme find_escape_strings(dstate_id_t this_idx) const;
+ virtual size_t accelSize(void) const = 0;
+ virtual u32 max_allowed_offset_accel() const = 0;
+ virtual u32 max_stop_char() const = 0;
+ virtual u32 max_floating_stop_char() const = 0;
+ virtual void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
+ void *accel_out);
+ virtual std::map<dstate_id_t, AccelScheme> getAccelInfo(const Grey &grey);
+};
+
+} // namespace ue2
+
+#endif // ACCEL_DFA_BUILD_STRAT_H
--- /dev/null
+/*
+ * Copyright (c) 2015-2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "dfa_build_strat.h"
+
+#include "accel.h"
+#include "accelcompile.h"
+#include "grey.h"
+#include "mcclellan_internal.h"
+#include "mcclellancompile_util.h"
+#include "nfa_internal.h"
+#include "shufticompile.h"
+#include "trufflecompile.h"
+#include "ue2common.h"
+#include "util/alloc.h"
+#include "util/bitutils.h"
+#include "util/charreach.h"
+#include "util/compare.h"
+#include "util/compile_context.h"
+#include "util/container.h"
+#include "util/make_unique.h"
+#include "util/order_check.h"
+#include "util/report_manager.h"
+#include "util/ue2_containers.h"
+#include "util/unaligned.h"
+#include "util/verify_types.h"
+
+#include <vector>
+
+using namespace std;
+
+namespace ue2 {
+
+// prevent weak vtables for raw_report_info, dfa_build_strat and raw_dfa
+raw_report_info::~raw_report_info() {
+}
+
+dfa_build_strat::~dfa_build_strat() {
+}
+
+raw_dfa::~raw_dfa() {
+}
+
+namespace {
+
+struct raw_report_list {
+ flat_set<ReportID> reports;
+
+ raw_report_list(const flat_set<ReportID> &reports_in,
+ const ReportManager &rm, bool do_remap) {
+ if (do_remap) {
+ for (auto &id : reports_in) {
+ reports.insert(rm.getProgramOffset(id));
+ }
+ } else {
+ reports = reports_in;
+ }
+ }
+
+ bool operator<(const raw_report_list &b) const {
+ return reports < b.reports;
+ }
+};
+
+struct raw_report_info_impl : public raw_report_info {
+ vector<raw_report_list> rl;
+ u32 getReportListSize() const override;
+ size_t size() const override;
+ void fillReportLists(NFA *n, size_t base_offset,
+ std::vector<u32> &ro /* out */) const override;
+};
+}
+
+unique_ptr<raw_report_info>
+dfa_build_strat::gatherReports(vector<u32> &reports, vector<u32> &reports_eod,
+ u8 *isSingleReport, ReportID *arbReport) const {
+ auto &rdfa = get_raw();
+ DEBUG_PRINTF("gathering reports\n");
+
+ const bool remap_reports = has_managed_reports(rdfa.kind);
+
+ auto ri = ue2::make_unique<raw_report_info_impl>();
+ map<raw_report_list, u32> rev;
+
+ for (const dstate &s : rdfa.states) {
+ if (s.reports.empty()) {
+ reports.push_back(MO_INVALID_IDX);
+ continue;
+ }
+
+ raw_report_list rrl(s.reports, rm, remap_reports);
+ DEBUG_PRINTF("non empty r\n");
+ if (rev.find(rrl) != rev.end()) {
+ reports.push_back(rev[rrl]);
+ } else {
+ DEBUG_PRINTF("adding to rl %zu\n", ri->size());
+ rev[rrl] = ri->size();
+ reports.push_back(ri->size());
+ ri->rl.push_back(rrl);
+ }
+ }
+
+ for (const dstate &s : rdfa.states) {
+ if (s.reports_eod.empty()) {
+ reports_eod.push_back(MO_INVALID_IDX);
+ continue;
+ }
+
+ DEBUG_PRINTF("non empty r eod\n");
+ raw_report_list rrl(s.reports_eod, rm, remap_reports);
+ if (rev.find(rrl) != rev.end()) {
+ reports_eod.push_back(rev[rrl]);
+ continue;
+ }
+
+ DEBUG_PRINTF("adding to rl eod %zu\n", s.reports_eod.size());
+ rev[rrl] = ri->size();
+ reports_eod.push_back(ri->size());
+ ri->rl.push_back(rrl);
+ }
+
+ assert(!ri->rl.empty()); /* all components should be able to generate
+ reports */
+ if (!ri->rl.empty()) {
+ *arbReport = *ri->rl.begin()->reports.begin();
+ } else {
+ *arbReport = 0;
+ }
+
+ /* if we have only a single report id generated from all accepts (not eod)
+ * we can take some short cuts */
+ set<ReportID> reps;
+
+ for (u32 rl_index : reports) {
+ if (rl_index == MO_INVALID_IDX) {
+ continue;
+ }
+ assert(rl_index < ri->size());
+ insert(&reps, ri->rl[rl_index].reports);
+ }
+
+ if (reps.size() == 1) {
+ *isSingleReport = 1;
+ *arbReport = *reps.begin();
+ DEBUG_PRINTF("single -- %u\n", *arbReport);
+ } else {
+ *isSingleReport = 0;
+ }
+
+ return move(ri);
+}
+
+u32 raw_report_info_impl::getReportListSize() const {
+ u32 rv = 0;
+
+ for (const auto &reps : rl) {
+ rv += sizeof(report_list);
+ rv += sizeof(ReportID) * reps.reports.size();
+ }
+
+ return rv;
+}
+
+size_t raw_report_info_impl::size() const {
+ return rl.size();
+}
+
+void raw_report_info_impl::fillReportLists(NFA *n, size_t base_offset,
+ vector<u32> &ro) const {
+ for (const auto &reps : rl) {
+ ro.push_back(base_offset);
+
+ report_list *p = (report_list *)((char *)n + base_offset);
+
+ u32 i = 0;
+ for (const ReportID report : reps.reports) {
+ p->report[i++] = report;
+ }
+ p->count = verify_u32(reps.reports.size());
+
+ base_offset += sizeof(report_list);
+ base_offset += sizeof(ReportID) * reps.reports.size();
+ }
+}
+
+} // namespace ue2
--- /dev/null
+/*
+ * Copyright (c) 2015-2016, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DFA_BUILD_STRAT_H
+#define DFA_BUILD_STRAT_H
+
+#include "rdfa.h"
+#include "ue2common.h"
+
+#include <memory>
+#include <vector>
+
+struct NFA;
+
+namespace ue2 {
+
+class ReportManager;
+
+struct raw_report_info {
+ virtual ~raw_report_info();
+ virtual u32 getReportListSize() const = 0; /* in bytes */
+ virtual size_t size() const = 0; /* number of lists */
+ virtual void fillReportLists(NFA *n, size_t base_offset,
+ std::vector<u32> &ro /* out */) const = 0;
+};
+
+class dfa_build_strat {
+public:
+ explicit dfa_build_strat(const ReportManager &rm_in) : rm(rm_in) {}
+ virtual ~dfa_build_strat();
+ virtual raw_dfa &get_raw() const = 0;
+ virtual std::unique_ptr<raw_report_info> gatherReports(
+ std::vector<u32> &reports /* out */,
+ std::vector<u32> &reports_eod /* out */,
+ u8 *isSingleReport /* out */,
+ ReportID *arbReport /* out */) const = 0;
+protected:
+ const ReportManager &rm;
+};
+
+} // namespace ue2
+
+#endif // DFA_BUILD_STRAT_H
#include "accelcompile.h"
#include "grey.h"
#include "mcclellan_internal.h"
-#include "mcclellancompile_accel.h"
#include "mcclellancompile_util.h"
#include "nfa_internal.h"
#include "shufticompile.h"
using namespace std;
using boost::adaptors::map_keys;
+#define ACCEL_DFA_MAX_OFFSET_DEPTH 4
+
+/** Maximum tolerated number of escape character from an accel state.
+ * This is larger than nfa, as we don't have a budget and the nfa cheats on stop
+ * characters for sets of states */
+#define ACCEL_DFA_MAX_STOP_CHAR 160
+
+/** Maximum tolerated number of escape character from a sds accel state. Larger
+ * than normal states as accelerating sds is important. Matches NFA value */
+#define ACCEL_DFA_MAX_FLOATING_STOP_CHAR 192
+
namespace ue2 {
namespace /* anon */ {
};
struct dfa_info {
- dfa_build_strat &strat;
+ accel_dfa_build_strat &strat;
raw_dfa &raw;
vector<dstate> &states;
vector<dstate_extra> extra;
u8 getAlphaShift() const;
- explicit dfa_info(dfa_build_strat &s)
+ explicit dfa_info(accel_dfa_build_strat &s)
: strat(s),
raw(s.get_raw()),
states(raw.states),
return aux;
}
-static
-bool double_byte_ok(const AccelScheme &info) {
- return !info.double_byte.empty()
- && info.double_cr.count() < info.double_byte.size()
- && info.double_cr.count() <= 2 && !info.double_byte.empty();
-}
-
static
void markEdges(NFA *n, u16 *succ_table, const dfa_info &info) {
assert((size_t)succ_table % 2 == 0);
return ACCEL_DFA_MAX_OFFSET_DEPTH;
}
-AccelScheme mcclellan_build_strat::find_escape_strings(dstate_id_t this_idx)
- const {
- return find_mcclellan_escape_info(rdfa, this_idx,
- max_allowed_offset_accel());
+u32 mcclellan_build_strat::max_stop_char() const {
+ return ACCEL_DFA_MAX_STOP_CHAR;
}
-/** builds acceleration schemes for states */
-void mcclellan_build_strat::buildAccel(UNUSED dstate_id_t this_idx,
- const AccelScheme &info,
- void *accel_out) {
- AccelAux *accel = (AccelAux *)accel_out;
-
- DEBUG_PRINTF("accelerations scheme has offset s%u/d%u\n", info.offset,
- info.double_offset);
- accel->generic.offset = verify_u8(info.offset);
-
- if (double_byte_ok(info) && info.double_cr.none()
- && info.double_byte.size() == 1) {
- accel->accel_type = ACCEL_DVERM;
- accel->dverm.c1 = info.double_byte.begin()->first;
- accel->dverm.c2 = info.double_byte.begin()->second;
- accel->dverm.offset = verify_u8(info.double_offset);
- DEBUG_PRINTF("state %hu is double vermicelli\n", this_idx);
- return;
- }
-
- if (double_byte_ok(info) && info.double_cr.none()
- && (info.double_byte.size() == 2 || info.double_byte.size() == 4)) {
- bool ok = true;
-
- assert(!info.double_byte.empty());
- u8 firstC = info.double_byte.begin()->first & CASE_CLEAR;
- u8 secondC = info.double_byte.begin()->second & CASE_CLEAR;
-
- for (const pair<u8, u8> &p : info.double_byte) {
- if ((p.first & CASE_CLEAR) != firstC
- || (p.second & CASE_CLEAR) != secondC) {
- ok = false;
- break;
- }
- }
-
- if (ok) {
- accel->accel_type = ACCEL_DVERM_NOCASE;
- accel->dverm.c1 = firstC;
- accel->dverm.c2 = secondC;
- accel->dverm.offset = verify_u8(info.double_offset);
- DEBUG_PRINTF("state %hu is nc double vermicelli\n", this_idx);
- return;
- }
-
- u8 m1;
- u8 m2;
- if (buildDvermMask(info.double_byte, &m1, &m2)) {
- accel->accel_type = ACCEL_DVERM_MASKED;
- accel->dverm.offset = verify_u8(info.double_offset);
- accel->dverm.c1 = info.double_byte.begin()->first & m1;
- accel->dverm.c2 = info.double_byte.begin()->second & m2;
- accel->dverm.m1 = m1;
- accel->dverm.m2 = m2;
- DEBUG_PRINTF("building maskeddouble-vermicelli for 0x%02hhx%02hhx\n",
- accel->dverm.c1, accel->dverm.c2);
- return;
- }
- }
-
- if (double_byte_ok(info)
- && shuftiBuildDoubleMasks(info.double_cr, info.double_byte,
- &accel->dshufti.lo1, &accel->dshufti.hi1,
- &accel->dshufti.lo2, &accel->dshufti.hi2)) {
- accel->accel_type = ACCEL_DSHUFTI;
- accel->dshufti.offset = verify_u8(info.double_offset);
- DEBUG_PRINTF("state %hu is double shufti\n", this_idx);
- return;
- }
-
- if (info.cr.none()) {
- accel->accel_type = ACCEL_RED_TAPE;
- DEBUG_PRINTF("state %hu is a dead end full of bureaucratic red tape"
- " from which there is no escape\n", this_idx);
- return;
- }
-
- if (info.cr.count() == 1) {
- accel->accel_type = ACCEL_VERM;
- accel->verm.c = info.cr.find_first();
- DEBUG_PRINTF("state %hu is vermicelli\n", this_idx);
- return;
- }
-
- if (info.cr.count() == 2 && info.cr.isCaselessChar()) {
- accel->accel_type = ACCEL_VERM_NOCASE;
- accel->verm.c = info.cr.find_first() & CASE_CLEAR;
- DEBUG_PRINTF("state %hu is caseless vermicelli\n", this_idx);
- return;
- }
-
- if (info.cr.count() > ACCEL_DFA_MAX_FLOATING_STOP_CHAR) {
- accel->accel_type = ACCEL_NONE;
- DEBUG_PRINTF("state %hu is too broad\n", this_idx);
- return;
- }
-
- accel->accel_type = ACCEL_SHUFTI;
- if (-1 != shuftiBuildMasks(info.cr, &accel->shufti.lo,
- &accel->shufti.hi)) {
- DEBUG_PRINTF("state %hu is shufti\n", this_idx);
- return;
- }
-
- assert(!info.cr.none());
- accel->accel_type = ACCEL_TRUFFLE;
- truffleBuildMasks(info.cr, &accel->truffle.mask1, &accel->truffle.mask2);
- DEBUG_PRINTF("state %hu is truffle\n", this_idx);
+u32 mcclellan_build_strat::max_floating_stop_char() const {
+ return ACCEL_DFA_MAX_FLOATING_STOP_CHAR;
}
static
}
}
-raw_dfa::~raw_dfa() {
-}
-
-raw_report_info::raw_report_info() {
-}
-
-raw_report_info::~raw_report_info() {
-}
-
namespace {
struct raw_report_list {
auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb);
map<dstate_id_t, AccelScheme> accel_escape_info
- = populateAccelerationInfo(info.raw, info.strat, cc.grey);
+ = info.strat.getAccelInfo(cc.grey);
size_t tran_size = (1 << info.getAlphaShift())
* sizeof(u16) * count_real_states;
auto ri = info.strat.gatherReports(reports, reports_eod, &single, &arb);
map<dstate_id_t, AccelScheme> accel_escape_info
- = populateAccelerationInfo(info.raw, info.strat, cc.grey);
+ = info.strat.getAccelInfo(cc.grey);
size_t tran_size = sizeof(u8) * (1 << info.getAlphaShift()) * info.size();
size_t aux_size = sizeof(mstate_aux) * info.size();
return false;
}
-aligned_unique_ptr<NFA> mcclellanCompile_i(raw_dfa &raw, dfa_build_strat &strat,
+aligned_unique_ptr<NFA> mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat,
const CompileContext &cc,
set<dstate_id_t> *accel_states) {
u16 total_daddy = 0;
return m->has_accel;
}
-dfa_build_strat::~dfa_build_strat() {
-}
-
} // namespace ue2
#ifndef MCCLELLANCOMPILE_H
#define MCCLELLANCOMPILE_H
+#include "accel_dfa_build_strat.h"
#include "rdfa.h"
#include "ue2common.h"
#include "util/accel_scheme.h"
class ReportManager;
struct CompileContext;
-struct raw_report_info {
- raw_report_info();
- virtual ~raw_report_info();
- virtual u32 getReportListSize() const = 0; /* in bytes */
- virtual size_t size() const = 0; /* number of lists */
- virtual void fillReportLists(NFA *n, size_t base_offset,
- std::vector<u32> &ro /* out */) const = 0;
-};
-
-class dfa_build_strat {
-public:
- explicit dfa_build_strat(const ReportManager &rm_in) : rm(rm_in) {}
- virtual ~dfa_build_strat();
- virtual raw_dfa &get_raw() const = 0;
- virtual std::unique_ptr<raw_report_info> gatherReports(
- std::vector<u32> &reports /* out */,
- std::vector<u32> &reports_eod /* out */,
- u8 *isSingleReport /* out */,
- ReportID *arbReport /* out */) const = 0;
- virtual AccelScheme find_escape_strings(dstate_id_t this_idx) const = 0;
- virtual size_t accelSize(void) const = 0;
- virtual void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
- void *accel_out) = 0;
-protected:
- const ReportManager &rm;
-};
-
-class mcclellan_build_strat : public dfa_build_strat {
+class mcclellan_build_strat : public accel_dfa_build_strat {
public:
mcclellan_build_strat(raw_dfa &rdfa_in, const ReportManager &rm_in)
- : dfa_build_strat(rm_in), rdfa(rdfa_in) {}
+ : accel_dfa_build_strat(rm_in), rdfa(rdfa_in) {}
raw_dfa &get_raw() const override { return rdfa; }
std::unique_ptr<raw_report_info> gatherReports(
std::vector<u32> &reports /* out */,
std::vector<u32> &reports_eod /* out */,
u8 *isSingleReport /* out */,
ReportID *arbReport /* out */) const override;
- AccelScheme find_escape_strings(dstate_id_t this_idx) const override;
size_t accelSize(void) const override;
- void buildAccel(dstate_id_t this_idx,const AccelScheme &info,
- void *accel_out) override;
- virtual u32 max_allowed_offset_accel() const;
+ u32 max_allowed_offset_accel() const override;
+ u32 max_stop_char() const override;
+ u32 max_floating_stop_char() const override;
private:
raw_dfa &rdfa;
/* used internally by mcclellan/haig/gough compile process */
ue2::aligned_unique_ptr<NFA>
-mcclellanCompile_i(raw_dfa &raw, dfa_build_strat &strat,
+mcclellanCompile_i(raw_dfa &raw, accel_dfa_build_strat &strat,
const CompileContext &cc,
std::set<dstate_id_t> *accel_states = nullptr);
return v;
}
-static
-bool has_self_loop(dstate_id_t s, const raw_dfa &raw) {
- u16 top_remap = raw.alpha_remap[TOP];
- for (u32 i = 0; i < raw.states[s].next.size(); i++) {
- if (i != top_remap && raw.states[s].next[i] == s) {
- return true;
- }
- }
- return false;
-}
-
-dstate_id_t get_sds_or_proxy(const raw_dfa &raw) {
- if (raw.start_floating != DEAD_STATE) {
- DEBUG_PRINTF("has floating start\n");
- return raw.start_floating;
- }
-
- DEBUG_PRINTF("looking for SDS proxy\n");
-
- dstate_id_t s = raw.start_anchored;
-
- if (has_self_loop(s, raw)) {
- return s;
- }
-
- u16 top_remap = raw.alpha_remap[TOP];
-
- ue2::unordered_set<dstate_id_t> seen;
- while (true) {
- seen.insert(s);
- DEBUG_PRINTF("basis %hu\n", s);
-
- /* check if we are connected to a state with a self loop */
- for (u32 i = 0; i < raw.states[s].next.size(); i++) {
- dstate_id_t t = raw.states[s].next[i];
- if (i != top_remap && t != DEAD_STATE && has_self_loop(t, raw)) {
- return t;
- }
- }
-
- /* find a neighbour to use as a basis for looking for the sds proxy */
- dstate_id_t t = DEAD_STATE;
- for (u32 i = 0; i < raw.states[s].next.size(); i++) {
- dstate_id_t tt = raw.states[s].next[i];
- if (i != top_remap && tt != DEAD_STATE && !contains(seen, tt)) {
- t = tt;
- break;
- }
- }
-
- if (t == DEAD_STATE) {
- /* we were unable to find a state to use as a SDS proxy */
- return DEAD_STATE;
- }
-
- s = t;
- }
-}
-
static
bool can_die_early(const raw_dfa &raw, dstate_id_t s,
map<dstate_id_t, u32> &visited, u32 age_limit) {
/** \brief Compute a simple hash of this raw_dfa, including its reports. */
size_t hash_dfa(const raw_dfa &rdfa);
-dstate_id_t get_sds_or_proxy(const raw_dfa &raw);
-
bool can_die_early(const raw_dfa &raw, u32 age_limit);
} // namespace ue2
projects / thirdparty / vectorscan.git / commitdiff
