From 911f179f1c8018b363b1eba5bbcd8d2e7df169d2 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Mon, 31 Jul 2017 18:51:14 +0200 Subject: [PATCH] policy aho-corasick: makefile tweaks - install the library with executable bit - use native library extension (i.e. .dylib on Macs) - kill their fancy CFLAGS to get better portability (e.g. -msse4.1) - gitlab-ci: this submodule is needed before building already - actually remove aho-corasick.lua --- .gitlab-ci.yml | 4 +- Makefile | 1 - modules/policy/aho-corasick.lua | 159 -------------------------------- modules/policy/policy.mk | 17 +++- 4 files changed, 14 insertions(+), 167 deletions(-) delete mode 100644 modules/policy/aho-corasick.lua diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c235e5bd..5969a1064 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,10 +3,11 @@ image: cznic/ubuntu:16.04 variables: DEBIAN_FRONTEND: noninteractive LC_ALL: C.UTF-8 - + build:linux:amd64: stage: build script: + - git submodule update --init --recursive - PREFIX=$(pwd)/.local make -k all - PREFIX=$(pwd)/.local make install artifacts: @@ -30,7 +31,6 @@ test:linux:amd64: deckard:linux:amd64: stage: test script: - - git submodule update --init --recursive - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc) --keep-going" make check-integration dependencies: - build:linux:amd64 diff --git a/Makefile b/Makefile index e265fc904..a6b3f2b2d 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,6 @@ info: $(info [$(HAS_libfstrm)] libfstrm (modules/dnstap)) $(info [$(HAS_libprotobuf-c)] libprotobuf-c (modules/dnstap)) $(info [$(HAS_protoc-c)] proto-c (modules/dnstap)) - $(info [$(HAS_ahocorasick)] ahocorasick (modules/policy)) $(info ) # Verify required dependencies are met, as listed above diff --git a/modules/policy/aho-corasick.lua b/modules/policy/aho-corasick.lua deleted file mode 100644 index af4330bcc..000000000 --- a/modules/policy/aho-corasick.lua +++ /dev/null @@ -1,159 +0,0 @@ --- A Lua implementation of the Aho-Corasick string matching algorithm --- --- Copyright (c) 2013 CloudFlare, Inc. All rights reserved. --- --- Redistribution and use in source and binary forms, with or without --- modification, are permitted provided that the following conditions are --- met: --- --- * Redistributions of source code must retain the above copyright --- notice, this list of conditions and the following disclaimer. --- * Redistributions in binary form must reproduce the above --- copyright notice, this list of conditions and the following disclaimer --- in the documentation and/or other materials provided with the --- distribution. --- * Neither the name of CloudFlare, Inc. nor the names of its --- contributors may be used to endorse or promote products derived from --- this software without specific prior written permission. --- --- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS --- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT --- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR --- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT --- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, --- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT --- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, --- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY --- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT --- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE --- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- --- Usage: --- --- local AC = require 'aho-corasick' --- --- t = AC.build({'words', 'to', 'find'}) --- r = AC.match(t, 'try to find in this string') --- r == {'to', 'find'} - -local M = {} -local byte = string.byte -local char = string.char - -local root = "" - --- make: creates a new entry in t for the given string c with optional fail --- state -local function make(t, c, f) - t[c] = {} - t[c].to = {} - t[c].fail = f - t[c].hit = root - t[c].word = false -end - --- build: builds the Aho-Corasick data structure from an array of strings -function M.build(m) - local t = {} - make(t, root, root) - - for i = 1, #m do - local current = root - - -- Build the tos which capture the transitions within the tree - - for j = 1, m[i]:len() do - local c = byte(m[i], j) - local path = current .. char(c) - - if t[current].to[c] == nil then - t[current].to[c] = path - - if current == root then - make(t, path, root) - else - make(t, path) - end - end - - current = path - end - - t[m[i]].word = true - end - - -- Build the fails which show how to backtrack when a fail matches and - -- build the hits which connect nodes to suffixes that are words - - local q = {root} - - while #q > 0 do - local path = table.remove(q, 1) - - for c, p in pairs(t[path].to) do - table.insert(q, p) - - local fail = p:sub(2) - while fail ~= "" and t[fail] == nil do - fail = fail:sub(2) - end - if fail == "" then fail = root end - t[p].fail = fail - - local hit = p:sub(2) - while hit ~= "" and (t[hit] == nil or not t[hit].word) do - hit = hit:sub(2) - end - if hit == "" then hit = root end - t[p].hit = hit - end - end - - return t -end - --- match: checks to see if the passed in string matches the passed in tree --- created with build. If all is true (the default) an array of all matches is --- returned. If all is false then only the first match is returned. -function M.match(t, s, all) - if all == nil then - all = true - end - - local path = root - local hits = {} - local hits_idx = 0 - - for i = 1,s:len() do - local c = byte(s, i) - - while t[path].to[c] == nil and path ~= root do - path = t[path].fail - end - - local n = t[path].to[c] - - if n ~= nil then - path = n - - if t[n].word then - hits_idx = hits_idx + 1 - hits[hits_idx] = n - end - - while t[n].hit ~= root do - n = t[n].hit - hits_idx = hits_idx + 1 - hits[hits_idx] = n - end - - if all == false and hits_idx > 0 then - return hits - end - end - end - - return hits -end - -return M \ No newline at end of file diff --git a/modules/policy/policy.mk b/modules/policy/policy.mk index 2e485a9b2..cd49a1a27 100644 --- a/modules/policy/policy.mk +++ b/modules/policy/policy.mk @@ -1,9 +1,16 @@ -policy_SOURCES := policy.lua lua-aho-corasick/ahocorasick.so -policy_DEPEND := modules/policy/lua-aho-corasick/ahocorasick.so +AHOCORASICK_DIR = modules/policy/lua-aho-corasick/ + +policy_SOURCES := policy.lua +policy_DEPEND := $(AHOCORASICK_DIR)ahocorasick$(LIBEXT) $(call make_lua_module,policy) -AHOCORASICK_DIR = modules/policy/lua-aho-corasick/ policy-clean: $(MAKE) -C $(AHOCORASICK_DIR) clean -$(AHOCORASICK_DIR)ahocorasick.so: $(AHOCORASICK_DIR)Makefile - $(MAKE) -C $(AHOCORASICK_DIR) MY_CFLAGS=$(lua_CFLAGS) SO_EXT=so \ No newline at end of file +$(AHOCORASICK_DIR)ahocorasick$(LIBEXT): $(AHOCORASICK_DIR)Makefile + $(MAKE) -C $(AHOCORASICK_DIR) ahocorasick$(LIBEXT) CFLAGS="$(lua_CFLAGS) -O2 $(CFLAGS)" + @# CFLAGS overridden to get rid of -msse4.1 etc. + +policy-install: ahocorasick-install +ahocorasick-install: $(AHOCORASICK_DIR)ahocorasick$(LIBEXT) $(DESTDIR)$(MODULEDIR) + $(INSTALL) -m 755 $(AHOCORASICK_DIR)ahocorasick$(LIBEXT) $(DESTDIR)$(MODULEDIR) + -- 2.47.2