src/util-mpm-hs.c

   1 /* Copyright (C) 2007-2016 Open Information Security Foundation
   2  *
   3  * You can copy, redistribute or modify this Program under the terms of
   4  * the GNU General Public License version 2 as published by the Free
   5  * Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * version 2 along with this program; if not, write to the Free Software
  14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  15  * 02110-1301, USA.
  16  */
  17
  18 /**
  19  * \file
  20  *
  21  * \author Jim Xu <jim.xu@windriver.com>
  22  * \author Justin Viiret <justin.viiret@intel.com>
  23  *
  24  * MPM pattern matcher that calls the Hyperscan regex matcher.
  25  */
  26
  27 #include "suricata-common.h"
  28 #include "suricata.h"
  29
  30 #include "detect.h"
  31 #include "detect-parse.h"
  32 #include "detect-engine.h"
  33
  34 #include "conf.h"
  35 #include "util-debug.h"
  36 #include "util-unittest.h"
  37 #include "util-unittest-helper.h"
  38 #include "util-memcmp.h"
  39 #include "util-mpm-hs.h"
  40 #include "util-memcpy.h"
  41 #include "util-hash.h"
  42 #include "util-hash-lookup3.h"
  43 #include "util-hyperscan.h"
  44
  45 #ifdef BUILD_HYPERSCAN
  46
  47 #include <hs.h>
  48
  49 void SCHSInitCtx(MpmCtx *);
  50 void SCHSInitThreadCtx(MpmCtx *, MpmThreadCtx *);
  51 void SCHSDestroyCtx(MpmCtx *);
  52 void SCHSDestroyThreadCtx(MpmCtx *, MpmThreadCtx *);
  53 int SCHSAddPatternCI(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
  54                      uint32_t, SigIntId, uint8_t);
  55 int SCHSAddPatternCS(MpmCtx *, uint8_t *, uint16_t, uint16_t, uint16_t,
  56                      uint32_t, SigIntId, uint8_t);
  57 int SCHSPreparePatterns(MpmCtx *mpm_ctx);
  58 uint32_t SCHSSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
  59                     PrefilterRuleStore *pmq, const uint8_t *buf, const uint32_t buflen);
  60 void SCHSPrintInfo(MpmCtx *mpm_ctx);
  61 void SCHSPrintSearchStats(MpmThreadCtx *mpm_thread_ctx);
  62 void SCHSRegisterTests(void);
  63
  64 /* size of the hash table used to speed up pattern insertions initially */
  65 #define INIT_HASH_SIZE 65536
  66
  67 /* Initial size of the global database hash (used for de-duplication). */
  68 #define INIT_DB_HASH_SIZE 1000
  69
  70 /* Global prototype scratch, built incrementally as Hyperscan databases are
  71  * built and then cloned for each thread context. Access is serialised via
  72  * g_scratch_proto_mutex. */
  73 static hs_scratch_t *g_scratch_proto = NULL;
  74 static SCMutex g_scratch_proto_mutex = SCMUTEX_INITIALIZER;
  75
  76 /* Global hash table of Hyperscan databases, used for de-duplication. Access is
  77  * serialised via g_db_table_mutex. */
  78 static HashTable *g_db_table = NULL;
  79 static SCMutex g_db_table_mutex = SCMUTEX_INITIALIZER;
  80
  81 /**
  82  * \internal
  83  * \brief Wraps SCMalloc (which is a macro) so that it can be passed to
  84  * hs_set_allocator() for Hyperscan's use.
  85  */
  86 static void *SCHSMalloc(size_t size)
  87 {
  88     return SCMalloc(size);
  89 }
  90
  91 /**
  92  * \internal
  93  * \brief Wraps SCFree (which is a macro) so that it can be passed to
  94  * hs_set_allocator() for Hyperscan's use.
  95  */
  96 static void SCHSFree(void *ptr)
  97 {
  98     SCFree(ptr);
  99 }
 100
 101 /** \brief Register Suricata malloc/free with Hyperscan.
 102  *
 103  * Requests that Hyperscan use Suricata's allocator for allocation of
 104  * databases, scratch space, etc.
 105  */
 106 static void SCHSSetAllocators(void)
 107 {
 108     hs_error_t err = hs_set_allocator(SCHSMalloc, SCHSFree);
 109     if (err != HS_SUCCESS) {
 110         FatalError(SC_ERR_FATAL, "Failed to set Hyperscan allocator.");
 111     }
 112 }
 113
 114 /**
 115  * \internal
 116  * \brief Creates a hash of the pattern.  We use it for the hashing process
 117  *        during the initial pattern insertion time, to cull duplicate sigs.
 118  *
 119  * \param pat    Pointer to the pattern.
 120  * \param patlen Pattern length.
 121  *
 122  * \retval hash A 32 bit unsigned hash.
 123  */
 124 static inline uint32_t SCHSInitHashRaw(uint8_t *pat, uint16_t patlen)
 125 {
 126     uint32_t hash = patlen * pat[0];
 127     if (patlen > 1)
 128         hash += pat[1];
 129
 130     return (hash % INIT_HASH_SIZE);
 131 }
 132
 133 /**
 134  * \internal
 135  * \brief Looks up a pattern.  We use it for the hashing process during
 136  *        the initial pattern insertion time, to cull duplicate sigs.
 137  *
 138  * \param ctx    Pointer to the HS ctx.
 139  * \param pat    Pointer to the pattern.
 140  * \param patlen Pattern length.
 141  * \param flags  Flags.  We don't need this.
 142  *
 143  * \retval hash A 32 bit unsigned hash.
 144  */
 145 static inline SCHSPattern *SCHSInitHashLookup(SCHSCtx *ctx, uint8_t *pat,
 146                                               uint16_t patlen, uint16_t offset,
 147                                               uint16_t depth, char flags,
 148                                               uint32_t pid)
 149 {
 150     uint32_t hash = SCHSInitHashRaw(pat, patlen);
 151
 152     if (ctx->init_hash == NULL) {
 153         return NULL;
 154     }
 155
 156     SCHSPattern *t = ctx->init_hash[hash];
 157     for (; t != NULL; t = t->next) {
 158         /* Since Hyperscan uses offset/depth, we must distinguish between
 159          * patterns with the same ID but different offset/depth here. */
 160         if (t->id == pid && t->offset == offset && t->depth == depth) {
 161             BUG_ON(t->len != patlen);
 162             BUG_ON(SCMemcmp(t->original_pat, pat, patlen) != 0);
 163             return t;
 164         }
 165     }
 166
 167     return NULL;
 168 }
 169
 170 /**
 171  * \internal
 172  * \brief Allocates a new pattern instance.
 173  *
 174  * \param mpm_ctx Pointer to the mpm context.
 175  *
 176  * \retval p Pointer to the newly created pattern.
 177  */
 178 static inline SCHSPattern *SCHSAllocPattern(MpmCtx *mpm_ctx)
 179 {
 180     SCHSPattern *p = SCMalloc(sizeof(SCHSPattern));
 181     if (unlikely(p == NULL)) {
 182         exit(EXIT_FAILURE);
 183     }
 184     memset(p, 0, sizeof(SCHSPattern));
 185
 186     mpm_ctx->memory_cnt++;
 187     mpm_ctx->memory_size += sizeof(SCHSPattern);
 188
 189     return p;
 190 }
 191
 192 /**
 193  * \internal
 194  * \brief Used to free SCHSPattern instances.
 195  *
 196  * \param mpm_ctx Pointer to the mpm context.
 197  * \param p       Pointer to the SCHSPattern instance to be freed.
 198  * \param free    Free the above pointer or not.
 199  */
 200 static inline void SCHSFreePattern(MpmCtx *mpm_ctx, SCHSPattern *p)
 201 {
 202     if (p != NULL && p->original_pat != NULL) {
 203         SCFree(p->original_pat);
 204         mpm_ctx->memory_cnt--;
 205         mpm_ctx->memory_size -= p->len;
 206     }
 207
 208     if (p != NULL && p->sids != NULL) {
 209         SCFree(p->sids);
 210     }
 211
 212     if (p != NULL) {
 213         SCFree(p);
 214         mpm_ctx->memory_cnt--;
 215         mpm_ctx->memory_size -= sizeof(SCHSPattern);
 216     }
 217 }
 218
 219 static inline uint32_t SCHSInitHash(SCHSPattern *p)
 220 {
 221     uint32_t hash = p->len * p->original_pat[0];
 222     if (p->len > 1)
 223         hash += p->original_pat[1];
 224
 225     return (hash % INIT_HASH_SIZE);
 226 }
 227
 228 static inline int SCHSInitHashAdd(SCHSCtx *ctx, SCHSPattern *p)
 229 {
 230     uint32_t hash = SCHSInitHash(p);
 231
 232     if (ctx->init_hash == NULL) {
 233         return 0;
 234     }
 235
 236     if (ctx->init_hash[hash] == NULL) {
 237         ctx->init_hash[hash] = p;
 238         return 0;
 239     }
 240
 241     SCHSPattern *tt = NULL;
 242     SCHSPattern *t = ctx->init_hash[hash];
 243
 244     /* get the list tail */
 245     do {
 246         tt = t;
 247         t = t->next;
 248     } while (t != NULL);
 249
 250     tt->next = p;
 251
 252     return 0;
 253 }
 254
 255 /**
 256  * \internal
 257  * \brief Add a pattern to the mpm-hs context.
 258  *
 259  * \param mpm_ctx Mpm context.
 260  * \param pat     Pointer to the pattern.
 261  * \param patlen  Length of the pattern.
 262  * \param pid     Pattern id
 263  * \param sid     Signature id (internal id).
 264  * \param flags   Pattern's MPM_PATTERN_* flags.
 265  *
 266  * \retval  0 On success.
 267  * \retval -1 On failure.
 268  */
 269 static int SCHSAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
 270                           uint16_t offset, uint16_t depth, uint32_t pid,
 271                           SigIntId sid, uint8_t flags)
 272 {
 273     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 274
 275     if (offset != 0) {
 276         flags |= MPM_PATTERN_FLAG_OFFSET;
 277     }
 278     if (depth != 0) {
 279         flags |= MPM_PATTERN_FLAG_DEPTH;
 280     }
 281
 282     if (patlen == 0) {
 283         SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0");
 284         return 0;
 285     }
 286
 287     /* check if we have already inserted this pattern */
 288     SCHSPattern *p =
 289         SCHSInitHashLookup(ctx, pat, patlen, offset, depth, flags, pid);
 290     if (p == NULL) {
 291         SCLogDebug("Allocing new pattern");
 292
 293         /* p will never be NULL */
 294         p = SCHSAllocPattern(mpm_ctx);
 295
 296         p->len = patlen;
 297         p->flags = flags;
 298         p->id = pid;
 299
 300         p->offset = offset;
 301         p->depth = depth;
 302
 303         p->original_pat = SCMalloc(patlen);
 304         if (p->original_pat == NULL)
 305             goto error;
 306         mpm_ctx->memory_cnt++;
 307         mpm_ctx->memory_size += patlen;
 308         memcpy(p->original_pat, pat, patlen);
 309
 310         /* put in the pattern hash */
 311         SCHSInitHashAdd(ctx, p);
 312
 313         mpm_ctx->pattern_cnt++;
 314
 315         if (!(mpm_ctx->flags & MPMCTX_FLAGS_NODEPTH)) {
 316             if (depth) {
 317                 mpm_ctx->maxdepth = MAX(mpm_ctx->maxdepth, depth);
 318                 SCLogDebug("%p: depth %u max %u", mpm_ctx, depth, mpm_ctx->maxdepth);
 319             } else {
 320                 mpm_ctx->flags |= MPMCTX_FLAGS_NODEPTH;
 321                 mpm_ctx->maxdepth = 0;
 322                 SCLogDebug("%p: alas, no depth for us", mpm_ctx);
 323             }
 324         }
 325
 326         if (mpm_ctx->maxlen < patlen)
 327             mpm_ctx->maxlen = patlen;
 328
 329         if (mpm_ctx->minlen == 0) {
 330             mpm_ctx->minlen = patlen;
 331         } else {
 332             if (mpm_ctx->minlen > patlen)
 333                 mpm_ctx->minlen = patlen;
 334         }
 335
 336         p->sids_size = 1;
 337         p->sids = SCMalloc(p->sids_size * sizeof(SigIntId));
 338         BUG_ON(p->sids == NULL);
 339         p->sids[0] = sid;
 340     } else {
 341         /* TODO figure out how we can be called multiple times for the same CTX with the same sid */
 342
 343         int found = 0;
 344         uint32_t x = 0;
 345         for (x = 0; x < p->sids_size; x++) {
 346             if (p->sids[x] == sid) {
 347                 found = 1;
 348                 break;
 349             }
 350         }
 351         if (!found) {
 352             SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1)));
 353             BUG_ON(sids == NULL);
 354             p->sids = sids;
 355             p->sids[p->sids_size] = sid;
 356             p->sids_size++;
 357         }
 358     }
 359
 360     return 0;
 361
 362 error:
 363     SCHSFreePattern(mpm_ctx, p);
 364     return -1;
 365 }
 366
 367 /**
 368  * \brief Pattern database information used only as input to the Hyperscan
 369  * compiler.
 370  */
 371 typedef struct SCHSCompileData_ {
 372     unsigned int *ids;
 373     unsigned int *flags;
 374     char **expressions;
 375     hs_expr_ext_t **ext;
 376     unsigned int pattern_cnt;
 377 } SCHSCompileData;
 378
 379 static SCHSCompileData *SCHSAllocCompileData(unsigned int pattern_cnt)
 380 {
 381     SCHSCompileData *cd = SCMalloc(pattern_cnt * sizeof(SCHSCompileData));
 382     if (cd == NULL) {
 383         goto error;
 384     }
 385     memset(cd, 0, pattern_cnt * sizeof(SCHSCompileData));
 386
 387     cd->pattern_cnt = pattern_cnt;
 388
 389     cd->ids = SCMalloc(pattern_cnt * sizeof(unsigned int));
 390     if (cd->ids == NULL) {
 391         goto error;
 392     }
 393     memset(cd->ids, 0, pattern_cnt * sizeof(unsigned int));
 394
 395     cd->flags = SCMalloc(pattern_cnt * sizeof(unsigned int));
 396     if (cd->flags == NULL) {
 397         goto error;
 398     }
 399     memset(cd->flags, 0, pattern_cnt * sizeof(unsigned int));
 400
 401     cd->expressions = SCMalloc(pattern_cnt * sizeof(char *));
 402     if (cd->expressions == NULL) {
 403         goto error;
 404     }
 405     memset(cd->expressions, 0, pattern_cnt * sizeof(char *));
 406
 407     cd->ext = SCMalloc(pattern_cnt * sizeof(hs_expr_ext_t *));
 408     if (cd->ext == NULL) {
 409         goto error;
 410     }
 411     memset(cd->ext, 0, pattern_cnt * sizeof(hs_expr_ext_t *));
 412
 413     return cd;
 414
 415 error:
 416     SCLogDebug("SCHSCompileData alloc failed");
 417     if (cd) {
 418         SCFree(cd->ids);
 419         SCFree(cd->flags);
 420         SCFree(cd->expressions);
 421         SCFree(cd->ext);
 422         SCFree(cd);
 423     }
 424     return NULL;
 425 }
 426
 427 static void SCHSFreeCompileData(SCHSCompileData *cd)
 428 {
 429     if (cd == NULL) {
 430         return;
 431     }
 432
 433     SCFree(cd->ids);
 434     SCFree(cd->flags);
 435     if (cd->expressions) {
 436         for (unsigned int i = 0; i < cd->pattern_cnt; i++) {
 437             SCFree(cd->expressions[i]);
 438         }
 439         SCFree(cd->expressions);
 440     }
 441     if (cd->ext) {
 442         for (unsigned int i = 0; i < cd->pattern_cnt; i++) {
 443             SCFree(cd->ext[i]);
 444         }
 445         SCFree(cd->ext);
 446     }
 447     SCFree(cd);
 448 }
 449
 450 typedef struct PatternDatabase_ {
 451     SCHSPattern **parray;
 452     hs_database_t *hs_db;
 453     uint32_t pattern_cnt;
 454
 455     /* Reference count: number of MPM contexts using this pattern database. */
 456     uint32_t ref_cnt;
 457 } PatternDatabase;
 458
 459 static uint32_t SCHSPatternHash(const SCHSPattern *p, uint32_t hash)
 460 {
 461     BUG_ON(p->original_pat == NULL);
 462     BUG_ON(p->sids == NULL);
 463
 464     hash = hashlittle_safe(&p->len, sizeof(p->len), hash);
 465     hash = hashlittle_safe(&p->flags, sizeof(p->flags), hash);
 466     hash = hashlittle_safe(p->original_pat, p->len, hash);
 467     hash = hashlittle_safe(&p->id, sizeof(p->id), hash);
 468     hash = hashlittle_safe(&p->offset, sizeof(p->offset), hash);
 469     hash = hashlittle_safe(&p->depth, sizeof(p->depth), hash);
 470     hash = hashlittle_safe(&p->sids_size, sizeof(p->sids_size), hash);
 471     hash = hashlittle_safe(p->sids, p->sids_size * sizeof(SigIntId), hash);
 472     return hash;
 473 }
 474
 475 static char SCHSPatternCompare(const SCHSPattern *p1, const SCHSPattern *p2)
 476 {
 477     if ((p1->len != p2->len) || (p1->flags != p2->flags) ||
 478         (p1->id != p2->id) || (p1->offset != p2->offset) ||
 479         (p1->depth != p2->depth) || (p1->sids_size != p2->sids_size)) {
 480         return 0;
 481     }
 482
 483     if (SCMemcmp(p1->original_pat, p2->original_pat, p1->len) != 0) {
 484         return 0;
 485     }
 486
 487     if (SCMemcmp(p1->sids, p2->sids, p1->sids_size * sizeof(p1->sids[0])) !=
 488         0) {
 489         return 0;
 490     }
 491
 492     return 1;
 493 }
 494
 495 static uint32_t PatternDatabaseHash(HashTable *ht, void *data, uint16_t len)
 496 {
 497     const PatternDatabase *pd = data;
 498     uint32_t hash = 0;
 499     hash = hashword(&pd->pattern_cnt, 1, hash);
 500
 501     for (uint32_t i = 0; i < pd->pattern_cnt; i++) {
 502         hash = SCHSPatternHash(pd->parray[i], hash);
 503     }
 504
 505     hash %= ht->array_size;
 506     return hash;
 507 }
 508
 509 static char PatternDatabaseCompare(void *data1, uint16_t len1, void *data2,
 510                                    uint16_t len2)
 511 {
 512     const PatternDatabase *pd1 = data1;
 513     const PatternDatabase *pd2 = data2;
 514
 515     if (pd1->pattern_cnt != pd2->pattern_cnt) {
 516         return 0;
 517     }
 518
 519     for (uint32_t i = 0; i < pd1->pattern_cnt; i++) {
 520         if (SCHSPatternCompare(pd1->parray[i], pd2->parray[i]) == 0) {
 521             return 0;
 522         }
 523     }
 524
 525     return 1;
 526 }
 527
 528 static void PatternDatabaseFree(PatternDatabase *pd)
 529 {
 530     BUG_ON(pd->ref_cnt != 0);
 531
 532     if (pd->parray != NULL) {
 533         for (uint32_t i = 0; i < pd->pattern_cnt; i++) {
 534             SCHSPattern *p = pd->parray[i];
 535             if (p != NULL) {
 536                 SCFree(p->original_pat);
 537                 SCFree(p->sids);
 538                 SCFree(p);
 539             }
 540         }
 541         SCFree(pd->parray);
 542     }
 543
 544     hs_free_database(pd->hs_db);
 545
 546     SCFree(pd);
 547 }
 548
 549 static void PatternDatabaseTableFree(void *data)
 550 {
 551     /* Stub function handed to hash table; actual freeing of PatternDatabase
 552      * structures is done in MPM destruction when the ref_cnt drops to zero. */
 553 }
 554
 555 static PatternDatabase *PatternDatabaseAlloc(uint32_t pattern_cnt)
 556 {
 557     PatternDatabase *pd = SCMalloc(sizeof(PatternDatabase));
 558     if (pd == NULL) {
 559         return NULL;
 560     }
 561     memset(pd, 0, sizeof(PatternDatabase));
 562     pd->pattern_cnt = pattern_cnt;
 563     pd->ref_cnt = 0;
 564     pd->hs_db = NULL;
 565
 566     /* alloc the pattern array */
 567     pd->parray =
 568         (SCHSPattern **)SCMalloc(pd->pattern_cnt * sizeof(SCHSPattern *));
 569     if (pd->parray == NULL) {
 570         SCFree(pd);
 571         return NULL;
 572     }
 573     memset(pd->parray, 0, pd->pattern_cnt * sizeof(SCHSPattern *));
 574
 575     return pd;
 576 }
 577
 578 /**
 579  * \brief Process the patterns added to the mpm, and create the internal tables.
 580  *
 581  * \param mpm_ctx Pointer to the mpm context.
 582  */
 583 int SCHSPreparePatterns(MpmCtx *mpm_ctx)
 584 {
 585     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 586
 587     if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) {
 588         SCLogDebug("no patterns supplied to this mpm_ctx");
 589         return 0;
 590     }
 591
 592     hs_error_t err;
 593     hs_compile_error_t *compile_err = NULL;
 594     SCHSCompileData *cd = NULL;
 595     PatternDatabase *pd = NULL;
 596
 597     cd = SCHSAllocCompileData(mpm_ctx->pattern_cnt);
 598     if (cd == NULL) {
 599         goto error;
 600     }
 601
 602     pd = PatternDatabaseAlloc(mpm_ctx->pattern_cnt);
 603     if (pd == NULL) {
 604         goto error;
 605     }
 606
 607     /* populate the pattern array with the patterns in the hash */
 608     for (uint32_t i = 0, p = 0; i < INIT_HASH_SIZE; i++) {
 609         SCHSPattern *node = ctx->init_hash[i], *nnode = NULL;
 610         while (node != NULL) {
 611             nnode = node->next;
 612             node->next = NULL;
 613             pd->parray[p++] = node;
 614             node = nnode;
 615         }
 616     }
 617
 618     /* we no longer need the hash, so free its memory */
 619     SCFree(ctx->init_hash);
 620     ctx->init_hash = NULL;
 621
 622     /* Serialise whole database compilation as a relatively easy way to ensure
 623      * dedupe is safe. */
 624     SCMutexLock(&g_db_table_mutex);
 625
 626     /* Init global pattern database hash if necessary. */
 627     if (g_db_table == NULL) {
 628         g_db_table = HashTableInit(INIT_DB_HASH_SIZE, PatternDatabaseHash,
 629                                    PatternDatabaseCompare,
 630                                    PatternDatabaseTableFree);
 631         if (g_db_table == NULL) {
 632             SCMutexUnlock(&g_db_table_mutex);
 633             goto error;
 634         }
 635     }
 636
 637     /* Check global hash table to see if we've seen this pattern database
 638      * before, and reuse the Hyperscan database if so. */
 639     PatternDatabase *pd_cached = HashTableLookup(g_db_table, pd, 1);
 640
 641     if (pd_cached != NULL) {
 642         SCLogDebug("Reusing cached database %p with %" PRIu32
 643                    " patterns (ref_cnt=%" PRIu32 ")",
 644                    pd_cached->hs_db, pd_cached->pattern_cnt,
 645                    pd_cached->ref_cnt);
 646         pd_cached->ref_cnt++;
 647         ctx->pattern_db = pd_cached;
 648         SCMutexUnlock(&g_db_table_mutex);
 649         PatternDatabaseFree(pd);
 650         SCHSFreeCompileData(cd);
 651         return 0;
 652     }
 653
 654     BUG_ON(ctx->pattern_db != NULL); /* already built? */
 655
 656     for (uint32_t i = 0; i < pd->pattern_cnt; i++) {
 657         const SCHSPattern *p = pd->parray[i];
 658
 659         cd->ids[i] = i;
 660         cd->flags[i] = HS_FLAG_SINGLEMATCH;
 661         if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
 662             cd->flags[i] |= HS_FLAG_CASELESS;
 663         }
 664
 665         cd->expressions[i] = HSRenderPattern(p->original_pat, p->len);
 666
 667         if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) {
 668             cd->ext[i] = SCMalloc(sizeof(hs_expr_ext_t));
 669             if (cd->ext[i] == NULL) {
 670                 SCMutexUnlock(&g_db_table_mutex);
 671                 goto error;
 672             }
 673             memset(cd->ext[i], 0, sizeof(hs_expr_ext_t));
 674
 675             if (p->flags & MPM_PATTERN_FLAG_OFFSET) {
 676                 cd->ext[i]->flags |= HS_EXT_FLAG_MIN_OFFSET;
 677                 cd->ext[i]->min_offset = p->offset + p->len;
 678             }
 679             if (p->flags & MPM_PATTERN_FLAG_DEPTH) {
 680                 cd->ext[i]->flags |= HS_EXT_FLAG_MAX_OFFSET;
 681                 cd->ext[i]->max_offset = p->offset + p->depth;
 682             }
 683         }
 684     }
 685
 686     BUG_ON(mpm_ctx->pattern_cnt == 0);
 687
 688     err = hs_compile_ext_multi((const char *const *)cd->expressions, cd->flags,
 689                                cd->ids, (const hs_expr_ext_t *const *)cd->ext,
 690                                cd->pattern_cnt, HS_MODE_BLOCK, NULL, &pd->hs_db,
 691                                &compile_err);
 692
 693     if (err != HS_SUCCESS) {
 694         SCLogError(SC_ERR_FATAL, "failed to compile hyperscan database");
 695         if (compile_err) {
 696             SCLogError(SC_ERR_FATAL, "compile error: %s", compile_err->message);
 697         }
 698         hs_free_compile_error(compile_err);
 699         SCMutexUnlock(&g_db_table_mutex);
 700         goto error;
 701     }
 702
 703     ctx->pattern_db = pd;
 704
 705     SCMutexLock(&g_scratch_proto_mutex);
 706     err = hs_alloc_scratch(pd->hs_db, &g_scratch_proto);
 707     SCMutexUnlock(&g_scratch_proto_mutex);
 708     if (err != HS_SUCCESS) {
 709         SCLogError(SC_ERR_FATAL, "failed to allocate scratch");
 710         SCMutexUnlock(&g_db_table_mutex);
 711         goto error;
 712     }
 713
 714     err = hs_database_size(pd->hs_db, &ctx->hs_db_size);
 715     if (err != HS_SUCCESS) {
 716         SCLogError(SC_ERR_FATAL, "failed to query database size");
 717         SCMutexUnlock(&g_db_table_mutex);
 718         goto error;
 719     }
 720
 721     mpm_ctx->memory_cnt++;
 722     mpm_ctx->memory_size += ctx->hs_db_size;
 723
 724     SCLogDebug("Built %" PRIu32 " patterns into a database of size %" PRIuMAX
 725                " bytes", mpm_ctx->pattern_cnt, (uintmax_t)ctx->hs_db_size);
 726
 727     /* Cache this database globally for later. */
 728     pd->ref_cnt = 1;
 729     int r = HashTableAdd(g_db_table, pd, 1);
 730     SCMutexUnlock(&g_db_table_mutex);
 731     if (r < 0)
 732         goto error;
 733
 734     SCHSFreeCompileData(cd);
 735     return 0;
 736
 737 error:
 738     if (pd) {
 739         PatternDatabaseFree(pd);
 740     }
 741     if (cd) {
 742         SCHSFreeCompileData(cd);
 743     }
 744     return -1;
 745 }
 746
 747 /**
 748  * \brief Init the mpm thread context.
 749  *
 750  * \param mpm_ctx        Pointer to the mpm context.
 751  * \param mpm_thread_ctx Pointer to the mpm thread context.
 752  */
 753 void SCHSInitThreadCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx)
 754 {
 755     memset(mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
 756
 757     SCHSThreadCtx *ctx = SCMalloc(sizeof(SCHSThreadCtx));
 758     if (ctx == NULL) {
 759         exit(EXIT_FAILURE);
 760     }
 761     mpm_thread_ctx->ctx = ctx;
 762
 763     memset(ctx, 0, sizeof(SCHSThreadCtx));
 764     mpm_thread_ctx->memory_cnt++;
 765     mpm_thread_ctx->memory_size += sizeof(SCHSThreadCtx);
 766
 767     ctx->scratch = NULL;
 768     ctx->scratch_size = 0;
 769
 770     SCMutexLock(&g_scratch_proto_mutex);
 771
 772     if (g_scratch_proto == NULL) {
 773         /* There is no scratch prototype: this means that we have not compiled
 774          * any Hyperscan databases. */
 775         SCMutexUnlock(&g_scratch_proto_mutex);
 776         SCLogDebug("No scratch space prototype");
 777         return;
 778     }
 779
 780     hs_error_t err = hs_clone_scratch(g_scratch_proto,
 781                                       (hs_scratch_t **)&ctx->scratch);
 782
 783     SCMutexUnlock(&g_scratch_proto_mutex);
 784
 785     if (err != HS_SUCCESS) {
 786         FatalError(SC_ERR_FATAL, "Unable to clone scratch prototype");
 787     }
 788
 789     err = hs_scratch_size(ctx->scratch, &ctx->scratch_size);
 790     if (err != HS_SUCCESS) {
 791         FatalError(SC_ERR_FATAL, "Unable to query scratch size");
 792     }
 793
 794     mpm_thread_ctx->memory_cnt++;
 795     mpm_thread_ctx->memory_size += ctx->scratch_size;
 796 }
 797
 798 /**
 799  * \brief Initialize the HS context.
 800  *
 801  * \param mpm_ctx       Mpm context.
 802  */
 803 void SCHSInitCtx(MpmCtx *mpm_ctx)
 804 {
 805     if (mpm_ctx->ctx != NULL)
 806         return;
 807
 808     mpm_ctx->ctx = SCMalloc(sizeof(SCHSCtx));
 809     if (mpm_ctx->ctx == NULL) {
 810         exit(EXIT_FAILURE);
 811     }
 812     memset(mpm_ctx->ctx, 0, sizeof(SCHSCtx));
 813
 814     mpm_ctx->memory_cnt++;
 815     mpm_ctx->memory_size += sizeof(SCHSCtx);
 816
 817     /* initialize the hash we use to speed up pattern insertions */
 818     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 819     ctx->init_hash = SCMalloc(sizeof(SCHSPattern *) * INIT_HASH_SIZE);
 820     if (ctx->init_hash == NULL) {
 821         exit(EXIT_FAILURE);
 822     }
 823     memset(ctx->init_hash, 0, sizeof(SCHSPattern *) * INIT_HASH_SIZE);
 824 }
 825
 826 /**
 827  * \brief Destroy the mpm thread context.
 828  *
 829  * \param mpm_ctx        Pointer to the mpm context.
 830  * \param mpm_thread_ctx Pointer to the mpm thread context.
 831  */
 832 void SCHSDestroyThreadCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx)
 833 {
 834     SCHSPrintSearchStats(mpm_thread_ctx);
 835
 836     if (mpm_thread_ctx->ctx != NULL) {
 837         SCHSThreadCtx *thr_ctx = (SCHSThreadCtx *)mpm_thread_ctx->ctx;
 838
 839         if (thr_ctx->scratch != NULL) {
 840             hs_free_scratch(thr_ctx->scratch);
 841             mpm_thread_ctx->memory_cnt--;
 842             mpm_thread_ctx->memory_size -= thr_ctx->scratch_size;
 843         }
 844
 845         SCFree(mpm_thread_ctx->ctx);
 846         mpm_thread_ctx->ctx = NULL;
 847         mpm_thread_ctx->memory_cnt--;
 848         mpm_thread_ctx->memory_size -= sizeof(SCHSThreadCtx);
 849     }
 850 }
 851
 852 /**
 853  * \brief Destroy the mpm context.
 854  *
 855  * \param mpm_ctx Pointer to the mpm context.
 856  */
 857 void SCHSDestroyCtx(MpmCtx *mpm_ctx)
 858 {
 859     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 860     if (ctx == NULL)
 861         return;
 862
 863     if (ctx->init_hash != NULL) {
 864         SCFree(ctx->init_hash);
 865         ctx->init_hash = NULL;
 866         mpm_ctx->memory_cnt--;
 867         mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(SCHSPattern *));
 868     }
 869
 870     /* Decrement pattern database ref count, and delete it entirely if the
 871      * count has dropped to zero. */
 872     SCMutexLock(&g_db_table_mutex);
 873     PatternDatabase *pd = ctx->pattern_db;
 874     if (pd) {
 875         BUG_ON(pd->ref_cnt == 0);
 876         pd->ref_cnt--;
 877         if (pd->ref_cnt == 0) {
 878             HashTableRemove(g_db_table, pd, 1);
 879             PatternDatabaseFree(pd);
 880         }
 881     }
 882     SCMutexUnlock(&g_db_table_mutex);
 883
 884     SCFree(mpm_ctx->ctx);
 885     mpm_ctx->memory_cnt--;
 886     mpm_ctx->memory_size -= sizeof(SCHSCtx);
 887 }
 888
 889 typedef struct SCHSCallbackCtx_ {
 890     SCHSCtx *ctx;
 891     void *pmq;
 892     uint32_t match_count;
 893 } SCHSCallbackCtx;
 894
 895 /* Hyperscan MPM match event handler */
 896 static int SCHSMatchEvent(unsigned int id, unsigned long long from,
 897                           unsigned long long to, unsigned int flags,
 898                           void *ctx)
 899 {
 900     SCHSCallbackCtx *cctx = ctx;
 901     PrefilterRuleStore *pmq = cctx->pmq;
 902     const PatternDatabase *pd = cctx->ctx->pattern_db;
 903     const SCHSPattern *pat = pd->parray[id];
 904
 905     SCLogDebug("Hyperscan Match %" PRIu32 ": id=%" PRIu32 " @ %" PRIuMAX
 906                " (pat id=%" PRIu32 ")",
 907                cctx->match_count, (uint32_t)id, (uintmax_t)to, pat->id);
 908
 909     PrefilterAddSids(pmq, pat->sids, pat->sids_size);
 910
 911     cctx->match_count++;
 912     return 0;
 913 }
 914
 915 /**
 916  * \brief The Hyperscan search function.
 917  *
 918  * \param mpm_ctx        Pointer to the mpm context.
 919  * \param mpm_thread_ctx Pointer to the mpm thread context.
 920  * \param pmq            Pointer to the Pattern Matcher Queue to hold
 921  *                       search matches.
 922  * \param buf            Buffer to be searched.
 923  * \param buflen         Buffer length.
 924  *
 925  * \retval matches Match count.
 926  */
 927 uint32_t SCHSSearch(const MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
 928                     PrefilterRuleStore *pmq, const uint8_t *buf, const uint32_t buflen)
 929 {
 930     uint32_t ret = 0;
 931     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
 932     SCHSThreadCtx *hs_thread_ctx = (SCHSThreadCtx *)(mpm_thread_ctx->ctx);
 933     const PatternDatabase *pd = ctx->pattern_db;
 934
 935     if (unlikely(buflen == 0)) {
 936         return 0;
 937     }
 938
 939     SCHSCallbackCtx cctx = {.ctx = ctx, .pmq = pmq, .match_count = 0};
 940
 941     /* scratch should have been cloned from g_scratch_proto at thread init. */
 942     hs_scratch_t *scratch = hs_thread_ctx->scratch;
 943     BUG_ON(pd->hs_db == NULL);
 944     BUG_ON(scratch == NULL);
 945
 946     hs_error_t err = hs_scan(pd->hs_db, (const char *)buf, buflen, 0, scratch,
 947                              SCHSMatchEvent, &cctx);
 948     if (err != HS_SUCCESS) {
 949         /* An error value (other than HS_SCAN_TERMINATED) from hs_scan()
 950          * indicates that it was passed an invalid database or scratch region,
 951          * which is not something we can recover from at scan time. */
 952         SCLogError(SC_ERR_FATAL, "Hyperscan returned error %d", err);
 953         exit(EXIT_FAILURE);
 954     } else {
 955         ret = cctx.match_count;
 956     }
 957
 958     return ret;
 959 }
 960
 961 /**
 962  * \brief Add a case insensitive pattern.  Although we have different calls for
 963  *        adding case sensitive and insensitive patterns, we make a single call
 964  *        for either case.  No special treatment for either case.
 965  *
 966  * \param mpm_ctx Pointer to the mpm context.
 967  * \param pat     The pattern to add.
 968  * \param patlen  The pattern length.
 969  * \param offset  The pattern offset.
 970  * \param depth   The pattern depth.
 971  * \param pid     The pattern id.
 972  * \param sid     The pattern signature id.
 973  * \param flags   Flags associated with this pattern.
 974  *
 975  * \retval  0 On success.
 976  * \retval -1 On failure.
 977  */
 978 int SCHSAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
 979                      uint16_t offset, uint16_t depth, uint32_t pid,
 980                      SigIntId sid, uint8_t flags)
 981 {
 982     flags |= MPM_PATTERN_FLAG_NOCASE;
 983     return SCHSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
 984 }
 985
 986 /**
 987  * \brief Add a case sensitive pattern.  Although we have different calls for
 988  *        adding case sensitive and insensitive patterns, we make a single call
 989  *        for either case.  No special treatment for either case.
 990  *
 991  * \param mpm_ctx Pointer to the mpm context.
 992  * \param pat     The pattern to add.
 993  * \param patlen  The pattern length.
 994  * \param offset  The pattern offset.
 995  * \param depth   The pattern depth.
 996  * \param pid     The pattern id.
 997  * \param sid     The pattern signature id.
 998  * \param flags   Flags associated with this pattern.
 999  *
1000  * \retval  0 On success.
1001  * \retval -1 On failure.
1002  */
1003 int SCHSAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
1004                      uint16_t offset, uint16_t depth, uint32_t pid,
1005                      SigIntId sid, uint8_t flags)
1006 {
1007     return SCHSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
1008 }
1009
1010 void SCHSPrintSearchStats(MpmThreadCtx *mpm_thread_ctx)
1011 {
1012     return;
1013 }
1014
1015 void SCHSPrintInfo(MpmCtx *mpm_ctx)
1016 {
1017     SCHSCtx *ctx = (SCHSCtx *)mpm_ctx->ctx;
1018
1019     printf("MPM HS Information:\n");
1020     printf("Memory allocs:   %" PRIu32 "\n", mpm_ctx->memory_cnt);
1021     printf("Memory alloced:  %" PRIu32 "\n", mpm_ctx->memory_size);
1022     printf(" Sizeof:\n");
1023     printf("  MpmCtx         %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx));
1024     printf("  SCHSCtx:       %" PRIuMAX "\n", (uintmax_t)sizeof(SCHSCtx));
1025     printf("  SCHSPattern    %" PRIuMAX "\n", (uintmax_t)sizeof(SCHSPattern));
1026     printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
1027     printf("Smallest:        %" PRIu32 "\n", mpm_ctx->minlen);
1028     printf("Largest:         %" PRIu32 "\n", mpm_ctx->maxlen);
1029     printf("\n");
1030
1031     if (ctx) {
1032         PatternDatabase *pd = ctx->pattern_db;
1033         char *db_info = NULL;
1034         if (hs_database_info(pd->hs_db, &db_info) == HS_SUCCESS) {
1035             printf("HS Database Info: %s\n", db_info);
1036             SCFree(db_info);
1037         }
1038         printf("HS Database Size: %" PRIuMAX " bytes\n",
1039                (uintmax_t)ctx->hs_db_size);
1040     }
1041
1042     printf("\n");
1043 }
1044
1045 /************************** Mpm Registration ***************************/
1046
1047 /**
1048  * \brief Register the Hyperscan MPM.
1049  */
1050 void MpmHSRegister(void)
1051 {
1052     mpm_table[MPM_HS].name = "hs";
1053     mpm_table[MPM_HS].InitCtx = SCHSInitCtx;
1054     mpm_table[MPM_HS].InitThreadCtx = SCHSInitThreadCtx;
1055     mpm_table[MPM_HS].DestroyCtx = SCHSDestroyCtx;
1056     mpm_table[MPM_HS].DestroyThreadCtx = SCHSDestroyThreadCtx;
1057     mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS;
1058     mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI;
1059     mpm_table[MPM_HS].Prepare = SCHSPreparePatterns;
1060     mpm_table[MPM_HS].Search = SCHSSearch;
1061     mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo;
1062     mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats;
1063     mpm_table[MPM_HS].RegisterUnittests = SCHSRegisterTests;
1064
1065     /* Set Hyperscan memory allocators */
1066     SCHSSetAllocators();
1067 }
1068
1069 /**
1070  * \brief Clean up global memory used by all Hyperscan MPM instances.
1071  *
1072  * Currently, this is just the global scratch prototype.
1073  */
1074 void MpmHSGlobalCleanup(void)
1075 {
1076     SCMutexLock(&g_scratch_proto_mutex);
1077     if (g_scratch_proto) {
1078         SCLogPerf("Cleaning up Hyperscan global scratch");
1079         hs_free_scratch(g_scratch_proto);
1080         g_scratch_proto = NULL;
1081     }
1082     SCMutexUnlock(&g_scratch_proto_mutex);
1083
1084     SCMutexLock(&g_db_table_mutex);
1085     if (g_db_table != NULL) {
1086         SCLogPerf("Clearing Hyperscan database cache");
1087         HashTableFree(g_db_table);
1088         g_db_table = NULL;
1089     }
1090     SCMutexUnlock(&g_db_table_mutex);
1091 }
1092
1093 /*************************************Unittests********************************/
1094
1095 #ifdef UNITTESTS
1096
1097 static int SCHSTest01(void)
1098 {
1099     int result = 0;
1100     MpmCtx mpm_ctx;
1101     MpmThreadCtx mpm_thread_ctx;
1102     PrefilterRuleStore pmq;
1103
1104     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1105     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1106     MpmInitCtx(&mpm_ctx, MPM_HS);
1107
1108     /* 1 match */
1109     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1110     PmqSetup(&pmq);
1111
1112     SCHSPreparePatterns(&mpm_ctx);
1113     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1114
1115     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
1116
1117     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1118                               strlen(buf));
1119
1120     if (cnt == 1)
1121         result = 1;
1122     else
1123         printf("1 != %" PRIu32 " ", cnt);
1124
1125     SCHSDestroyCtx(&mpm_ctx);
1126     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1127     PmqFree(&pmq);
1128     return result;
1129 }
1130
1131 static int SCHSTest02(void)
1132 {
1133     int result = 0;
1134     MpmCtx mpm_ctx;
1135     MpmThreadCtx mpm_thread_ctx;
1136     PrefilterRuleStore pmq;
1137
1138     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1139     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1140     MpmInitCtx(&mpm_ctx, MPM_HS);
1141
1142     /* 1 match */
1143     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abce", 4, 0, 0, 0, 0, 0);
1144     PmqSetup(&pmq);
1145
1146     SCHSPreparePatterns(&mpm_ctx);
1147     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1148
1149     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
1150     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1151                               strlen(buf));
1152
1153     if (cnt == 0)
1154         result = 1;
1155     else
1156         printf("0 != %" PRIu32 " ", cnt);
1157
1158     SCHSDestroyCtx(&mpm_ctx);
1159     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1160     PmqFree(&pmq);
1161     return result;
1162 }
1163
1164 static int SCHSTest03(void)
1165 {
1166     int result = 0;
1167     MpmCtx mpm_ctx;
1168     MpmThreadCtx mpm_thread_ctx;
1169     PrefilterRuleStore pmq;
1170
1171     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1172     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1173     MpmInitCtx(&mpm_ctx, MPM_HS);
1174
1175     /* 1 match */
1176     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1177     /* 1 match */
1178     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"bcde", 4, 0, 0, 1, 0, 0);
1179     /* 1 match */
1180     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghj", 4, 0, 0, 2, 0, 0);
1181     PmqSetup(&pmq);
1182
1183     SCHSPreparePatterns(&mpm_ctx);
1184     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1185
1186     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
1187     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1188                               strlen(buf));
1189
1190     if (cnt == 3)
1191         result = 1;
1192     else
1193         printf("3 != %" PRIu32 " ", cnt);
1194
1195     SCHSDestroyCtx(&mpm_ctx);
1196     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1197     PmqFree(&pmq);
1198     return result;
1199 }
1200
1201 static int SCHSTest04(void)
1202 {
1203     int result = 0;
1204     MpmCtx mpm_ctx;
1205     MpmThreadCtx mpm_thread_ctx;
1206     PrefilterRuleStore pmq;
1207
1208     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1209     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1210     MpmInitCtx(&mpm_ctx, MPM_HS);
1211
1212     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1213     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"bcdegh", 6, 0, 0, 1, 0, 0);
1214     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"fghjxyz", 7, 0, 0, 2, 0, 0);
1215     PmqSetup(&pmq);
1216
1217     SCHSPreparePatterns(&mpm_ctx);
1218     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1219
1220     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
1221     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1222                               strlen(buf));
1223
1224     if (cnt == 1)
1225         result = 1;
1226     else
1227         printf("1 != %" PRIu32 " ", cnt);
1228
1229     SCHSDestroyCtx(&mpm_ctx);
1230     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1231     PmqFree(&pmq);
1232     return result;
1233 }
1234
1235 static int SCHSTest05(void)
1236 {
1237     int result = 0;
1238     MpmCtx mpm_ctx;
1239     MpmThreadCtx mpm_thread_ctx;
1240     PrefilterRuleStore pmq;
1241
1242     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1243     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1244     MpmInitCtx(&mpm_ctx, MPM_HS);
1245
1246     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"ABCD", 4, 0, 0, 0, 0, 0);
1247     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"bCdEfG", 6, 0, 0, 1, 0, 0);
1248     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghJikl", 7, 0, 0, 2, 0, 0);
1249     PmqSetup(&pmq);
1250
1251     SCHSPreparePatterns(&mpm_ctx);
1252     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1253
1254     const char *buf = "abcdefghjiklmnopqrstuvwxyz";
1255     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1256                               strlen(buf));
1257
1258     if (cnt == 3)
1259         result = 1;
1260     else
1261         printf("3 != %" PRIu32 " ", cnt);
1262
1263     SCHSDestroyCtx(&mpm_ctx);
1264     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1265     PmqFree(&pmq);
1266     return result;
1267 }
1268
1269 static int SCHSTest06(void)
1270 {
1271     int result = 0;
1272     MpmCtx mpm_ctx;
1273     MpmThreadCtx mpm_thread_ctx;
1274     PrefilterRuleStore pmq;
1275
1276     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1277     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1278     MpmInitCtx(&mpm_ctx, MPM_HS);
1279
1280     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1281     PmqSetup(&pmq);
1282
1283     SCHSPreparePatterns(&mpm_ctx);
1284     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1285
1286     const char *buf = "abcd";
1287     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1288                               strlen(buf));
1289
1290     if (cnt == 1)
1291         result = 1;
1292     else
1293         printf("1 != %" PRIu32 " ", cnt);
1294
1295     SCHSDestroyCtx(&mpm_ctx);
1296     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1297     PmqFree(&pmq);
1298     return result;
1299 }
1300
1301 static int SCHSTest07(void)
1302 {
1303     int result = 0;
1304     MpmCtx mpm_ctx;
1305     MpmThreadCtx mpm_thread_ctx;
1306     PrefilterRuleStore pmq;
1307
1308     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1309     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1310     MpmInitCtx(&mpm_ctx, MPM_HS);
1311
1312     /* should match 30 times */
1313     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"A", 1, 0, 0, 0, 0, 0);
1314     /* should match 29 times */
1315     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 1, 0, 0);
1316     /* should match 28 times */
1317     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AAA", 3, 0, 0, 2, 0, 0);
1318     /* 26 */
1319     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AAAAA", 5, 0, 0, 3, 0, 0);
1320     /* 21 */
1321     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AAAAAAAAAA", 10, 0, 0, 4, 0, 0);
1322     /* 1 */
1323     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30,
1324                     0, 0, 5, 0, 0);
1325     PmqSetup(&pmq);
1326
1327     SCHSPreparePatterns(&mpm_ctx);
1328     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1329
1330     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
1331     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1332                               strlen(buf));
1333
1334     if (cnt == 6)
1335         result = 1;
1336     else
1337         printf("6 != %" PRIu32 " ", cnt);
1338
1339     SCHSDestroyCtx(&mpm_ctx);
1340     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1341     PmqFree(&pmq);
1342     return result;
1343 }
1344
1345 static int SCHSTest08(void)
1346 {
1347     int result = 0;
1348     MpmCtx mpm_ctx;
1349     MpmThreadCtx mpm_thread_ctx;
1350     PrefilterRuleStore pmq;
1351
1352     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1353     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1354     MpmInitCtx(&mpm_ctx, MPM_HS);
1355
1356     /* 1 match */
1357     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1358     PmqSetup(&pmq);
1359
1360     SCHSPreparePatterns(&mpm_ctx);
1361     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1362
1363     uint32_t cnt =
1364         SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"a", 1);
1365
1366     if (cnt == 0)
1367         result = 1;
1368     else
1369         printf("0 != %" PRIu32 " ", cnt);
1370
1371     SCHSDestroyCtx(&mpm_ctx);
1372     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1373     PmqFree(&pmq);
1374     return result;
1375 }
1376
1377 static int SCHSTest09(void)
1378 {
1379     int result = 0;
1380     MpmCtx mpm_ctx;
1381     MpmThreadCtx mpm_thread_ctx;
1382     PrefilterRuleStore pmq;
1383
1384     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1385     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1386     MpmInitCtx(&mpm_ctx, MPM_HS);
1387
1388     /* 1 match */
1389     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ab", 2, 0, 0, 0, 0, 0);
1390     PmqSetup(&pmq);
1391
1392     SCHSPreparePatterns(&mpm_ctx);
1393     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1394
1395     uint32_t cnt =
1396         SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"ab", 2);
1397
1398     if (cnt == 1)
1399         result = 1;
1400     else
1401         printf("1 != %" PRIu32 " ", cnt);
1402
1403     SCHSDestroyCtx(&mpm_ctx);
1404     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1405     PmqFree(&pmq);
1406     return result;
1407 }
1408
1409 static int SCHSTest10(void)
1410 {
1411     int result = 0;
1412     MpmCtx mpm_ctx;
1413     MpmThreadCtx mpm_thread_ctx;
1414     PrefilterRuleStore pmq;
1415
1416     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1417     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1418     MpmInitCtx(&mpm_ctx, MPM_HS);
1419
1420     /* 1 match */
1421     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcdefgh", 8, 0, 0, 0, 0, 0);
1422     PmqSetup(&pmq);
1423
1424     SCHSPreparePatterns(&mpm_ctx);
1425     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1426
1427     const char *buf = "01234567890123456789012345678901234567890123456789"
1428                 "01234567890123456789012345678901234567890123456789"
1429                 "abcdefgh"
1430                 "01234567890123456789012345678901234567890123456789"
1431                 "01234567890123456789012345678901234567890123456789";
1432     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1433                               strlen(buf));
1434
1435     if (cnt == 1)
1436         result = 1;
1437     else
1438         printf("1 != %" PRIu32 " ", cnt);
1439
1440     SCHSDestroyCtx(&mpm_ctx);
1441     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1442     PmqFree(&pmq);
1443     return result;
1444 }
1445
1446 static int SCHSTest11(void)
1447 {
1448     int result = 0;
1449     MpmCtx mpm_ctx;
1450     MpmThreadCtx mpm_thread_ctx;
1451     PrefilterRuleStore pmq;
1452
1453     memset(&mpm_ctx, 0, sizeof(MpmCtx));
1454     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1455     MpmInitCtx(&mpm_ctx, MPM_HS);
1456
1457     if (MpmAddPatternCS(&mpm_ctx, (uint8_t *)"he", 2, 0, 0, 1, 0, 0) == -1)
1458         goto end;
1459     if (MpmAddPatternCS(&mpm_ctx, (uint8_t *)"she", 3, 0, 0, 2, 0, 0) == -1)
1460         goto end;
1461     if (MpmAddPatternCS(&mpm_ctx, (uint8_t *)"his", 3, 0, 0, 3, 0, 0) == -1)
1462         goto end;
1463     if (MpmAddPatternCS(&mpm_ctx, (uint8_t *)"hers", 4, 0, 0, 4, 0, 0) == -1)
1464         goto end;
1465     PmqSetup(&pmq);
1466
1467     if (SCHSPreparePatterns(&mpm_ctx) == -1)
1468         goto end;
1469
1470     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1471
1472     result = 1;
1473
1474     const char *buf = "he";
1475     result &= (SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1476                           strlen(buf)) == 1);
1477     buf = "she";
1478     result &= (SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1479                           strlen(buf)) == 2);
1480     buf = "his";
1481     result &= (SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1482                           strlen(buf)) == 1);
1483     buf = "hers";
1484     result &= (SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1485                           strlen(buf)) == 2);
1486
1487 end:
1488     SCHSDestroyCtx(&mpm_ctx);
1489     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1490     PmqFree(&pmq);
1491     return result;
1492 }
1493
1494 static int SCHSTest12(void)
1495 {
1496     int result = 0;
1497     MpmCtx mpm_ctx;
1498     MpmThreadCtx mpm_thread_ctx;
1499     PrefilterRuleStore pmq;
1500
1501     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1502     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1503     MpmInitCtx(&mpm_ctx, MPM_HS);
1504
1505     /* 1 match */
1506     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"wxyz", 4, 0, 0, 0, 0, 0);
1507     /* 1 match */
1508     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"vwxyz", 5, 0, 0, 1, 0, 0);
1509     PmqSetup(&pmq);
1510
1511     SCHSPreparePatterns(&mpm_ctx);
1512     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1513
1514     const char *buf = "abcdefghijklmnopqrstuvwxyz";
1515     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1516                               strlen(buf));
1517
1518     if (cnt == 2)
1519         result = 1;
1520     else
1521         printf("2 != %" PRIu32 " ", cnt);
1522
1523     SCHSDestroyCtx(&mpm_ctx);
1524     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1525     PmqFree(&pmq);
1526     return result;
1527 }
1528
1529 static int SCHSTest13(void)
1530 {
1531     int result = 0;
1532     MpmCtx mpm_ctx;
1533     MpmThreadCtx mpm_thread_ctx;
1534     PrefilterRuleStore pmq;
1535
1536     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1537     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1538     MpmInitCtx(&mpm_ctx, MPM_HS);
1539
1540     /* 1 match */
1541     const char *pat = "abcdefghijklmnopqrstuvwxyzABCD";
1542     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1543     PmqSetup(&pmq);
1544
1545     SCHSPreparePatterns(&mpm_ctx);
1546     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1547
1548     const char *buf = "abcdefghijklmnopqrstuvwxyzABCD";
1549     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1550                               strlen(buf));
1551
1552     if (cnt == 1)
1553         result = 1;
1554     else
1555         printf("1 != %" PRIu32 " ", cnt);
1556
1557     SCHSDestroyCtx(&mpm_ctx);
1558     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1559     PmqFree(&pmq);
1560     return result;
1561 }
1562
1563 static int SCHSTest14(void)
1564 {
1565     int result = 0;
1566     MpmCtx mpm_ctx;
1567     MpmThreadCtx mpm_thread_ctx;
1568     PrefilterRuleStore pmq;
1569
1570     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1571     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1572     MpmInitCtx(&mpm_ctx, MPM_HS);
1573
1574     /* 1 match */
1575     const char *pat = "abcdefghijklmnopqrstuvwxyzABCDE";
1576     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1577     PmqSetup(&pmq);
1578
1579     SCHSPreparePatterns(&mpm_ctx);
1580     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1581
1582     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDE";
1583     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1584                               strlen(buf));
1585
1586     if (cnt == 1)
1587         result = 1;
1588     else
1589         printf("1 != %" PRIu32 " ", cnt);
1590
1591     SCHSDestroyCtx(&mpm_ctx);
1592     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1593     PmqFree(&pmq);
1594     return result;
1595 }
1596
1597 static int SCHSTest15(void)
1598 {
1599     int result = 0;
1600     MpmCtx mpm_ctx;
1601     MpmThreadCtx mpm_thread_ctx;
1602     PrefilterRuleStore pmq;
1603
1604     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1605     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1606     MpmInitCtx(&mpm_ctx, MPM_HS);
1607
1608     /* 1 match */
1609     const char *pat = "abcdefghijklmnopqrstuvwxyzABCDEF";
1610     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1611     PmqSetup(&pmq);
1612
1613     SCHSPreparePatterns(&mpm_ctx);
1614     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1615
1616     const char *buf = "abcdefghijklmnopqrstuvwxyzABCDEF";
1617     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1618                               strlen(buf));
1619
1620     if (cnt == 1)
1621         result = 1;
1622     else
1623         printf("1 != %" PRIu32 " ", cnt);
1624
1625     SCHSDestroyCtx(&mpm_ctx);
1626     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1627     PmqFree(&pmq);
1628     return result;
1629 }
1630
1631 static int SCHSTest16(void)
1632 {
1633     int result = 0;
1634     MpmCtx mpm_ctx;
1635     MpmThreadCtx mpm_thread_ctx;
1636     PrefilterRuleStore pmq;
1637
1638     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1639     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1640     MpmInitCtx(&mpm_ctx, MPM_HS);
1641
1642     /* 1 match */
1643     const char *pat = "abcdefghijklmnopqrstuvwxyzABC";
1644     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1645     PmqSetup(&pmq);
1646
1647     SCHSPreparePatterns(&mpm_ctx);
1648     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1649
1650     const char *buf = "abcdefghijklmnopqrstuvwxyzABC";
1651     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1652                               strlen(buf));
1653
1654     if (cnt == 1)
1655         result = 1;
1656     else
1657         printf("1 != %" PRIu32 " ", cnt);
1658
1659     SCHSDestroyCtx(&mpm_ctx);
1660     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1661     PmqFree(&pmq);
1662     return result;
1663 }
1664
1665 static int SCHSTest17(void)
1666 {
1667     int result = 0;
1668     MpmCtx mpm_ctx;
1669     MpmThreadCtx mpm_thread_ctx;
1670     PrefilterRuleStore pmq;
1671
1672     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1673     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1674     MpmInitCtx(&mpm_ctx, MPM_HS);
1675
1676     /* 1 match */
1677     const char *pat = "abcdefghijklmnopqrstuvwxyzAB";
1678     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1679     PmqSetup(&pmq);
1680
1681     SCHSPreparePatterns(&mpm_ctx);
1682     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1683
1684     const char *buf = "abcdefghijklmnopqrstuvwxyzAB";
1685     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1686                               strlen(buf));
1687
1688     if (cnt == 1)
1689         result = 1;
1690     else
1691         printf("1 != %" PRIu32 " ", cnt);
1692
1693     SCHSDestroyCtx(&mpm_ctx);
1694     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1695     PmqFree(&pmq);
1696     return result;
1697 }
1698
1699 static int SCHSTest18(void)
1700 {
1701     int result = 0;
1702     MpmCtx mpm_ctx;
1703     MpmThreadCtx mpm_thread_ctx;
1704     PrefilterRuleStore pmq;
1705
1706     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1707     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1708     MpmInitCtx(&mpm_ctx, MPM_HS);
1709
1710     /* 1 match */
1711     const char *pat = "abcde"
1712                 "fghij"
1713                 "klmno"
1714                 "pqrst"
1715                 "uvwxy"
1716                 "z";
1717     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1718     PmqSetup(&pmq);
1719
1720     SCHSPreparePatterns(&mpm_ctx);
1721     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1722
1723     const char *buf = "abcde"
1724                 "fghij"
1725                 "klmno"
1726                 "pqrst"
1727                 "uvwxy"
1728                 "z";
1729     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1730                               strlen(buf));
1731
1732     if (cnt == 1)
1733         result = 1;
1734     else
1735         printf("1 != %" PRIu32 " ", cnt);
1736
1737     SCHSDestroyCtx(&mpm_ctx);
1738     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1739     PmqFree(&pmq);
1740     return result;
1741 }
1742
1743 static int SCHSTest19(void)
1744 {
1745     int result = 0;
1746     MpmCtx mpm_ctx;
1747     MpmThreadCtx mpm_thread_ctx;
1748     PrefilterRuleStore pmq;
1749
1750     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1751     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1752     MpmInitCtx(&mpm_ctx, MPM_HS);
1753
1754     /* 1 */
1755     const char *pat = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
1756     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1757     PmqSetup(&pmq);
1758
1759     SCHSPreparePatterns(&mpm_ctx);
1760     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1761
1762     const char *buf = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
1763     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1764                               strlen(buf));
1765
1766     if (cnt == 1)
1767         result = 1;
1768     else
1769         printf("1 != %" PRIu32 " ", cnt);
1770
1771     SCHSDestroyCtx(&mpm_ctx);
1772     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1773     PmqFree(&pmq);
1774     return result;
1775 }
1776
1777 static int SCHSTest20(void)
1778 {
1779     int result = 0;
1780     MpmCtx mpm_ctx;
1781     MpmThreadCtx mpm_thread_ctx;
1782     PrefilterRuleStore pmq;
1783
1784     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1785     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1786     MpmInitCtx(&mpm_ctx, MPM_HS);
1787
1788     /* 1 */
1789     const char *pat = "AAAAA"
1790                 "AAAAA"
1791                 "AAAAA"
1792                 "AAAAA"
1793                 "AAAAA"
1794                 "AAAAA"
1795                 "AA";
1796     MpmAddPatternCS(&mpm_ctx, (uint8_t *)pat, strlen(pat), 0, 0, 0, 0, 0);
1797     PmqSetup(&pmq);
1798
1799     SCHSPreparePatterns(&mpm_ctx);
1800     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1801
1802     const char *buf = "AAAAA"
1803                 "AAAAA"
1804                 "AAAAA"
1805                 "AAAAA"
1806                 "AAAAA"
1807                 "AAAAA"
1808                 "AA";
1809     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1810                               strlen(buf));
1811
1812     if (cnt == 1)
1813         result = 1;
1814     else
1815         printf("1 != %" PRIu32 " ", cnt);
1816
1817     SCHSDestroyCtx(&mpm_ctx);
1818     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1819     PmqFree(&pmq);
1820     return result;
1821 }
1822
1823 static int SCHSTest21(void)
1824 {
1825     int result = 0;
1826     MpmCtx mpm_ctx;
1827     MpmThreadCtx mpm_thread_ctx;
1828     PrefilterRuleStore pmq;
1829
1830     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1831     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1832     MpmInitCtx(&mpm_ctx, MPM_HS);
1833
1834     /* 1 */
1835     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
1836     PmqSetup(&pmq);
1837
1838     SCHSPreparePatterns(&mpm_ctx);
1839     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1840
1841     uint32_t cnt =
1842         SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"AA", 2);
1843
1844     if (cnt == 1)
1845         result = 1;
1846     else
1847         printf("1 != %" PRIu32 " ", cnt);
1848
1849     SCHSDestroyCtx(&mpm_ctx);
1850     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1851     PmqFree(&pmq);
1852     return result;
1853 }
1854
1855 static int SCHSTest22(void)
1856 {
1857     int result = 0;
1858     MpmCtx mpm_ctx;
1859     MpmThreadCtx mpm_thread_ctx;
1860     PrefilterRuleStore pmq;
1861
1862     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1863     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1864     MpmInitCtx(&mpm_ctx, MPM_HS);
1865
1866     /* 1 match */
1867     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcd", 4, 0, 0, 0, 0, 0);
1868     /* 1 match */
1869     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"abcde", 5, 0, 0, 1, 0, 0);
1870     PmqSetup(&pmq);
1871
1872     SCHSPreparePatterns(&mpm_ctx);
1873     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1874
1875     const char *buf = "abcdefghijklmnopqrstuvwxyz";
1876     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1877                               strlen(buf));
1878
1879     if (cnt == 2)
1880         result = 1;
1881     else
1882         printf("2 != %" PRIu32 " ", cnt);
1883
1884     SCHSDestroyCtx(&mpm_ctx);
1885     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1886     PmqFree(&pmq);
1887     return result;
1888 }
1889
1890 static int SCHSTest23(void)
1891 {
1892     int result = 0;
1893     MpmCtx mpm_ctx;
1894     MpmThreadCtx mpm_thread_ctx;
1895     PrefilterRuleStore pmq;
1896
1897     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1898     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1899     MpmInitCtx(&mpm_ctx, MPM_HS);
1900
1901     /* 1 */
1902     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
1903     PmqSetup(&pmq);
1904
1905     SCHSPreparePatterns(&mpm_ctx);
1906     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1907
1908     uint32_t cnt =
1909         SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2);
1910
1911     if (cnt == 0)
1912         result = 1;
1913     else
1914         printf("1 != %" PRIu32 " ", cnt);
1915
1916     SCHSDestroyCtx(&mpm_ctx);
1917     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1918     PmqFree(&pmq);
1919     return result;
1920 }
1921
1922 static int SCHSTest24(void)
1923 {
1924     int result = 0;
1925     MpmCtx mpm_ctx;
1926     MpmThreadCtx mpm_thread_ctx;
1927     PrefilterRuleStore pmq;
1928
1929     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1930     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1931     MpmInitCtx(&mpm_ctx, MPM_HS);
1932
1933     /* 1 */
1934     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"AA", 2, 0, 0, 0, 0, 0);
1935     PmqSetup(&pmq);
1936
1937     SCHSPreparePatterns(&mpm_ctx);
1938     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1939
1940     uint32_t cnt =
1941         SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)"aa", 2);
1942
1943     if (cnt == 1)
1944         result = 1;
1945     else
1946         printf("1 != %" PRIu32 " ", cnt);
1947
1948     SCHSDestroyCtx(&mpm_ctx);
1949     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1950     PmqFree(&pmq);
1951     return result;
1952 }
1953
1954 static int SCHSTest25(void)
1955 {
1956     int result = 0;
1957     MpmCtx mpm_ctx;
1958     MpmThreadCtx mpm_thread_ctx;
1959     PrefilterRuleStore pmq;
1960
1961     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1962     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1963     MpmInitCtx(&mpm_ctx, MPM_HS);
1964
1965     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"ABCD", 4, 0, 0, 0, 0, 0);
1966     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"bCdEfG", 6, 0, 0, 1, 0, 0);
1967     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"fghiJkl", 7, 0, 0, 2, 0, 0);
1968     PmqSetup(&pmq);
1969
1970     SCHSPreparePatterns(&mpm_ctx);
1971     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1972
1973     const char *buf = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1974     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
1975                               strlen(buf));
1976
1977     if (cnt == 3)
1978         result = 1;
1979     else
1980         printf("3 != %" PRIu32 " ", cnt);
1981
1982     SCHSDestroyCtx(&mpm_ctx);
1983     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
1984     PmqFree(&pmq);
1985     return result;
1986 }
1987
1988 static int SCHSTest26(void)
1989 {
1990     int result = 0;
1991     MpmCtx mpm_ctx;
1992     MpmThreadCtx mpm_thread_ctx;
1993     PrefilterRuleStore pmq;
1994
1995     memset(&mpm_ctx, 0x00, sizeof(MpmCtx));
1996     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
1997     MpmInitCtx(&mpm_ctx, MPM_HS);
1998
1999     MpmAddPatternCI(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 0, 0, 0);
2000     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"Works", 5, 0, 0, 1, 0, 0);
2001     PmqSetup(&pmq);
2002
2003     SCHSPreparePatterns(&mpm_ctx);
2004     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2005
2006     const char *buf = "works";
2007     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
2008                               strlen(buf));
2009
2010     if (cnt == 1)
2011         result = 1;
2012     else
2013         printf("3 != %" PRIu32 " ", cnt);
2014
2015     SCHSDestroyCtx(&mpm_ctx);
2016     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2017     PmqFree(&pmq);
2018     return result;
2019 }
2020
2021 static int SCHSTest27(void)
2022 {
2023     int result = 0;
2024     MpmCtx mpm_ctx;
2025     MpmThreadCtx mpm_thread_ctx;
2026     PrefilterRuleStore pmq;
2027
2028     memset(&mpm_ctx, 0, sizeof(MpmCtx));
2029     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
2030     MpmInitCtx(&mpm_ctx, MPM_HS);
2031
2032     /* 0 match */
2033     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"ONE", 3, 0, 0, 0, 0, 0);
2034     PmqSetup(&pmq);
2035
2036     SCHSPreparePatterns(&mpm_ctx);
2037     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2038
2039     const char *buf = "tone";
2040     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
2041                               strlen(buf));
2042
2043     if (cnt == 0)
2044         result = 1;
2045     else
2046         printf("0 != %" PRIu32 " ", cnt);
2047
2048     SCHSDestroyCtx(&mpm_ctx);
2049     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2050     PmqFree(&pmq);
2051     return result;
2052 }
2053
2054 static int SCHSTest28(void)
2055 {
2056     int result = 0;
2057     MpmCtx mpm_ctx;
2058     MpmThreadCtx mpm_thread_ctx;
2059     PrefilterRuleStore pmq;
2060
2061     memset(&mpm_ctx, 0, sizeof(MpmCtx));
2062     memset(&mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
2063     MpmInitCtx(&mpm_ctx, MPM_HS);
2064
2065     /* 0 match */
2066     MpmAddPatternCS(&mpm_ctx, (uint8_t *)"one", 3, 0, 0, 0, 0, 0);
2067     PmqSetup(&pmq);
2068
2069     SCHSPreparePatterns(&mpm_ctx);
2070     SCHSInitThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2071
2072     const char *buf = "tONE";
2073     uint32_t cnt = SCHSSearch(&mpm_ctx, &mpm_thread_ctx, &pmq, (uint8_t *)buf,
2074                               strlen(buf));
2075
2076     if (cnt == 0)
2077         result = 1;
2078     else
2079         printf("0 != %" PRIu32 " ", cnt);
2080
2081     SCHSDestroyCtx(&mpm_ctx);
2082     SCHSDestroyThreadCtx(&mpm_ctx, &mpm_thread_ctx);
2083     PmqFree(&pmq);
2084     return result;
2085 }
2086
2087 static int SCHSTest29(void)
2088 {
2089     uint8_t *buf = (uint8_t *)"onetwothreefourfivesixseveneightnine";
2090     uint16_t buflen = strlen((char *)buf);
2091     Packet *p = NULL;
2092     ThreadVars th_v;
2093     DetectEngineThreadCtx *det_ctx = NULL;
2094     int result = 0;
2095
2096     memset(&th_v, 0, sizeof(th_v));
2097     p = UTHBuildPacket(buf, buflen, IPPROTO_TCP);
2098
2099     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
2100     if (de_ctx == NULL)
2101         goto end;
2102     de_ctx->mpm_matcher = MPM_HS;
2103
2104     de_ctx->flags |= DE_QUIET;
2105
2106     de_ctx->sig_list = SigInit(
2107         de_ctx, "alert tcp any any -> any any "
2108                 "(content:\"onetwothreefourfivesixseveneightnine\"; sid:1;)");
2109     if (de_ctx->sig_list == NULL)
2110         goto end;
2111     de_ctx->sig_list->next =
2112         SigInit(de_ctx, "alert tcp any any -> any any "
2113                         "(content:\"onetwothreefourfivesixseveneightnine\"; "
2114                         "fast_pattern:3,3; sid:2;)");
2115     if (de_ctx->sig_list->next == NULL)
2116         goto end;
2117
2118     SigGroupBuild(de_ctx);
2119     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
2120
2121     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
2122     if (PacketAlertCheck(p, 1) != 1) {
2123         printf("if (PacketAlertCheck(p, 1) != 1) failure\n");
2124         goto end;
2125     }
2126     if (PacketAlertCheck(p, 2) != 1) {
2127         printf("if (PacketAlertCheck(p, 1) != 2) failure\n");
2128         goto end;
2129     }
2130
2131     result = 1;
2132 end:
2133     if (de_ctx != NULL) {
2134         SigGroupCleanup(de_ctx);
2135         SigCleanSignatures(de_ctx);
2136
2137         DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
2138         DetectEngineCtxFree(de_ctx);
2139     }
2140
2141     UTHFreePackets(&p, 1);
2142     return result;
2143 }
2144
2145 #endif /* UNITTESTS */
2146
2147 void SCHSRegisterTests(void)
2148 {
2149 #ifdef UNITTESTS
2150     UtRegisterTest("SCHSTest01", SCHSTest01);
2151     UtRegisterTest("SCHSTest02", SCHSTest02);
2152     UtRegisterTest("SCHSTest03", SCHSTest03);
2153     UtRegisterTest("SCHSTest04", SCHSTest04);
2154     UtRegisterTest("SCHSTest05", SCHSTest05);
2155     UtRegisterTest("SCHSTest06", SCHSTest06);
2156     UtRegisterTest("SCHSTest07", SCHSTest07);
2157     UtRegisterTest("SCHSTest08", SCHSTest08);
2158     UtRegisterTest("SCHSTest09", SCHSTest09);
2159     UtRegisterTest("SCHSTest10", SCHSTest10);
2160     UtRegisterTest("SCHSTest11", SCHSTest11);
2161     UtRegisterTest("SCHSTest12", SCHSTest12);
2162     UtRegisterTest("SCHSTest13", SCHSTest13);
2163     UtRegisterTest("SCHSTest14", SCHSTest14);
2164     UtRegisterTest("SCHSTest15", SCHSTest15);
2165     UtRegisterTest("SCHSTest16", SCHSTest16);
2166     UtRegisterTest("SCHSTest17", SCHSTest17);
2167     UtRegisterTest("SCHSTest18", SCHSTest18);
2168     UtRegisterTest("SCHSTest19", SCHSTest19);
2169     UtRegisterTest("SCHSTest20", SCHSTest20);
2170     UtRegisterTest("SCHSTest21", SCHSTest21);
2171     UtRegisterTest("SCHSTest22", SCHSTest22);
2172     UtRegisterTest("SCHSTest23", SCHSTest23);
2173     UtRegisterTest("SCHSTest24", SCHSTest24);
2174     UtRegisterTest("SCHSTest25", SCHSTest25);
2175     UtRegisterTest("SCHSTest26", SCHSTest26);
2176     UtRegisterTest("SCHSTest27", SCHSTest27);
2177     UtRegisterTest("SCHSTest28", SCHSTest28);
2178     UtRegisterTest("SCHSTest29", SCHSTest29);
2179 #endif
2180
2181     return;
2182 }
2183
2184 #endif /* BUILD_HYPERSCAN */