Guard against hash table mask overflows

To do this we add bounds checking to mkmask. We also allochashtable
inline function that makes sure the mask+1 does not overflow on
a 32bit system.

diff --git a/ext/pool_fileconflicts.c b/ext/pool_fileconflicts.c
index 2fd3d5404cc9e54becc779c1d88ae7c1ab5a5e37..6e9074d8a1e058ada4efa78502fdaa9e9b7749b8 100644 (file)
--- a/ext/pool_fileconflicts.c
+++ b/ext/pool_fileconflicts.c
@@ -8,6 +8,7 @@
 #include <stdio.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <stdlib.h>
 
 #include "pool.h"
 #include "repo.h"
@@ -79,23 +80,23 @@ growhash(Hashtable map, Hashval *mapnp)
   Hashtable m;
   Id hx, qx;
 
-  m = solv_calloc(newn + 1, 2 * sizeof(Id));
+  m = allochashtable(newn, 2);
   for (i = 0; i <= mapn; i++)
     {
-      hx = map[2 * i];
+      hx = map[2 * (size_t)i];
       if (!hx)
        continue;
       h = hx & newn;
       hh = HASHCHAIN_START;
       for (;;)
        {
-         qx = m[2 * h];
+         qx = m[2 * (size_t)h];
          if (!qx)
            break;
          h = HASHCHAIN_NEXT(h, hh, newn);
        }
-      m[2 * h] = hx;
-      m[2 * h + 1] = map[2 * i + 1];
+      m[2 * (size_t)h] = hx;
+      m[2 * (size_t)h + 1] = map[2 * (size_t)i + 1];
     }
   solv_free(map);
   *mapnp = newn;
@@ -121,7 +122,7 @@ finddirs_cb(void *cbdatav, const char *fn, struct filelistinfo *info)
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->dirmap[2 * h];
+      qx = cbdata->dirmap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == dhx)
@@ -133,21 +134,21 @@ finddirs_cb(void *cbdatav, const char *fn, struct filelistinfo *info)
       /* a miss */
       if (!cbdata->create)
        return;
-      cbdata->dirmap[2 * h] = dhx;
-      cbdata->dirmap[2 * h + 1] = idx;
-      if (++cbdata->dirmapused * 2 > cbdata->dirmapn)
+      cbdata->dirmap[2 * (size_t)h] = dhx;
+      cbdata->dirmap[2 * (size_t)h + 1] = idx;
+      if (++cbdata->dirmapused > cbdata->dirmapn / 2)
        cbdata->dirmap = growhash(cbdata->dirmap, &cbdata->dirmapn);
       return;
     }
   /* we saw this dir before */
-  oidx = cbdata->dirmap[2 * h + 1];
+  oidx = cbdata->dirmap[2 * (size_t)h + 1];
   if (oidx == idx)
     return;
   /* found a conflict, this dir may be used in multiple packages */
   if (oidx != -1)
     {
       MAPSET(&cbdata->idxmap, oidx);
-      cbdata->dirmap[2 * h + 1] = -1;  /* mark as "multiple packages" */
+      cbdata->dirmap[2 * (size_t)h + 1] = -1;  /* mark as "multiple packages" */
       cbdata->dirconflicts++;
     }
   MAPSET(&cbdata->idxmap, idx);
@@ -164,11 +165,11 @@ isindirmap(struct cbdata *cbdata, Id dhx)
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->dirmap[2 * h];
+      qx = cbdata->dirmap[2 * (size_t)h];
       if (!qx)
        return 0;
       if (qx == dhx)
-       return cbdata->dirmap[2 * h + 1] == -1 ? 1 : 0;
+       return cbdata->dirmap[2 * (size_t)h + 1] == -1 ? 1 : 0;
       h = HASHCHAIN_NEXT(h, hh, cbdata->dirmapn);
     }
 }
@@ -212,7 +213,7 @@ findfileconflicts_cb(void *cbdatav, const char *fn, struct filelistinfo *info)
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->cflmap[2 * h];
+      qx = cbdata->cflmap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == hx)
@@ -224,14 +225,14 @@ findfileconflicts_cb(void *cbdatav, const char *fn, struct filelistinfo *info)
       /* a miss */
       if (!cbdata->create)
        return;
-      cbdata->cflmap[2 * h] = hx;
-      cbdata->cflmap[2 * h + 1] = (isdir ? ~idx : idx);
-      if (++cbdata->cflmapused * 2 > cbdata->cflmapn)
+      cbdata->cflmap[2 * (size_t)h] = hx;
+      cbdata->cflmap[2 * (size_t)h + 1] = (isdir ? ~idx : idx);
+      if (++cbdata->cflmapused > cbdata->cflmapn / 2)
        cbdata->cflmap = growhash(cbdata->cflmap, &cbdata->cflmapn);
       return;
     }
   /* we have seen this hx before */
-  oidx = cbdata->cflmap[2 * h + 1];
+  oidx = cbdata->cflmap[2 * (size_t)h + 1];
   if (oidx < 0)
     {
       int i;
@@ -243,7 +244,7 @@ findfileconflicts_cb(void *cbdatav, const char *fn, struct filelistinfo *info)
        }
       oidx = ~oidx;
       /* now have file, had directories before. */
-      cbdata->cflmap[2 * h + 1] = oidx;        /* make it a file */
+      cbdata->cflmap[2 * (size_t)h + 1] = oidx;        /* make it a file */
       /* dump all delayed directory hits for hx */
       for (i = 0; i < cbdata->lookat_dir.count; i += 2)
        if (cbdata->lookat_dir.elements[i] == hx)
@@ -291,7 +292,7 @@ findfileconflicts_basename_cb(void *cbdatav, const char *fn, struct filelistinfo
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->cflmap[2 * h];
+      qx = cbdata->cflmap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == hx)
@@ -303,13 +304,13 @@ findfileconflicts_basename_cb(void *cbdatav, const char *fn, struct filelistinfo
       /* a miss */
       if (!cbdata->create)
        return;
-      cbdata->cflmap[2 * h] = hx;
-      cbdata->cflmap[2 * h + 1] = (isdir ? -idx - 2 : idx);
-      if (++cbdata->cflmapused * 2 > cbdata->cflmapn)
+      cbdata->cflmap[2 * (size_t)h] = hx;
+      cbdata->cflmap[2 * (size_t)h + 1] = (isdir ? -idx - 2 : idx);
+      if (++cbdata->cflmapused > cbdata->cflmapn / 2)
        cbdata->cflmap = growhash(cbdata->cflmap, &cbdata->cflmapn);
       return;
     }
-  oidx = cbdata->cflmap[2 * h + 1];
+  oidx = cbdata->cflmap[2 * (size_t)h + 1];
   if (oidx < -1)
     {
       int i;
@@ -321,7 +322,7 @@ findfileconflicts_basename_cb(void *cbdatav, const char *fn, struct filelistinfo
        }
       oidx = -idx - 2;
       /* now have file, had directories before. */
-      cbdata->cflmap[2 * h + 1] = oidx;        /* make it a file */
+      cbdata->cflmap[2 * (size_t)h + 1] = oidx;        /* make it a file */
       /* dump all delayed directory hits for hx */
       for (i = 0; i < cbdata->lookat_dir.count; i += 2)
        if (cbdata->lookat_dir.elements[i] == hx)
@@ -333,7 +334,7 @@ findfileconflicts_basename_cb(void *cbdatav, const char *fn, struct filelistinfo
     MAPSET(&cbdata->idxmap, oidx);
   MAPSET(&cbdata->idxmap, idx);
   if (oidx != -1)
-    cbdata->cflmap[2 * h + 1] = -1;
+    cbdata->cflmap[2 * (size_t)h + 1] = -1;
 }
 
 static inline Id
@@ -373,12 +374,12 @@ unifywithstat(struct cbdata *cbdata, Id diroff, int dirl)
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->statmap[2 * h];
+      qx = cbdata->statmap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == hx)
        {
-         Id off = cbdata->statmap[2 * h + 1];
+         Id off = cbdata->statmap[2 * (size_t)h + 1];
          char *dp = (char *)cbdata->filesspace + cbdata->norq.elements[off];
          if (!memcmp(dp, statdata, 16))
            return cbdata->norq.elements[off + 1];
@@ -389,9 +390,9 @@ unifywithstat(struct cbdata *cbdata, Id diroff, int dirl)
   nspaceoff = addfilesspace(cbdata, 16);
   memcpy(cbdata->filesspace + nspaceoff, statdata, 16);
   queue_push2(&cbdata->norq, nspaceoff, nspaceoff);
-  cbdata->statmap[2 * h] = hx;
-  cbdata->statmap[2 * h + 1] = cbdata->norq.count - 2;
-  if (++cbdata->statmapused * 2 > cbdata->statmapn)
+  cbdata->statmap[2 * (size_t)h] = hx;
+  cbdata->statmap[2 * (size_t)h + 1] = cbdata->norq.count - 2;
+  if (++cbdata->statmapused > cbdata->statmapn / 2)
     cbdata->statmap = growhash(cbdata->statmap, &cbdata->statmapn);
   return nspaceoff;
 }
@@ -529,12 +530,12 @@ normalizedir(struct cbdata *cbdata, const char *dir, int dirl, Id hx, int create
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->normap[2 * h];
+      qx = cbdata->normap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == hx)
        {
-         Id off = cbdata->normap[2 * h + 1];
+         Id off = cbdata->normap[2 * (size_t)h + 1];
          char *dp = (char *)cbdata->filesspace + cbdata->norq.elements[off];
          if (!strncmp(dp, dir, dirl) && dp[dirl] == 0)
            return cbdata->norq.elements[off + 1];
@@ -558,9 +559,9 @@ normalizedir(struct cbdata *cbdata, const char *dir, int dirl, Id hx, int create
   cbdata->filesspace[nspaceoff + dirl] = 0;
   mycnt = cbdata->norq.count;
   queue_push2(&cbdata->norq, nspaceoff, -1);   /* -1: in progress */
-  cbdata->normap[2 * h] = hx;
-  cbdata->normap[2 * h + 1] = mycnt;
-  if (++cbdata->normapused * 2 > cbdata->normapn)
+  cbdata->normap[2 * (size_t)h] = hx;
+  cbdata->normap[2 * (size_t)h + 1] = mycnt;
+  if (++cbdata->normapused > cbdata->normapn / 2)
     cbdata->normap = growhash(cbdata->normap, &cbdata->normapn);
   /* unify */
   if (cbdata->usestat)
@@ -604,14 +605,14 @@ findfileconflicts_alias_cb(void *cbdatav, const char *fn, struct filelistinfo *i
   hh = HASHCHAIN_START;
   for (;;)
     {
-      qx = cbdata->cflmap[2 * h];
+      qx = cbdata->cflmap[2 * (size_t)h];
       if (!qx)
        break;
       if (qx == hx)
        break;
       h = HASHCHAIN_NEXT(h, hh, cbdata->cflmapn);
     }
-  if (!qx || cbdata->cflmap[2 * h + 1] != -1)
+  if (!qx || cbdata->cflmap[2 * (size_t)h + 1] != -1)
     return;
   /* found entry marked as "multiple", recored as conflict candidate */
   if (!cbdata->lastdirhash)
@@ -815,7 +816,7 @@ precheck_solvable_files(struct cbdata *cbdata, Pool *pool, Id p)
       hh = HASHCHAIN_START;
       for (;;)
        {
-         qx = cbdata->cflmap[2 * h];
+         qx = cbdata->cflmap[2 * (size_t)h];
          if (!qx)
            break;
          if (qx == hx)
@@ -891,6 +892,8 @@ pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, in
 
   if (cutoff <= 0)
     cutoff = pkgs->count;
+  if (cutoff >= 0x4000000)
+    abort();   /* sorry, this overflows our hashes */
 
   /* avarage file list size: 200 files per package */
   /* avarage dir count: 20 dirs per package */
@@ -900,7 +903,7 @@ pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, in
     {
       hdrfetches = 0;
       cbdata.dirmapn = mkmask((cutoff + 3) * 16);
-      cbdata.dirmap = solv_calloc(cbdata.dirmapn + 1, 2 * sizeof(Id));
+      cbdata.dirmap = allochashtable(cbdata.dirmapn, 2);
       cbdata.create = 1;
       idxmapset = 0;
       for (i = 0; i < pkgs->count; i++)
@@ -937,7 +940,7 @@ pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, in
   /* second pass: scan files in the directories found above */
   now = solv_timems(0);
   cbdata.cflmapn = mkmask((cutoff + 3) * 32);
-  cbdata.cflmap = solv_calloc(cbdata.cflmapn + 1, 2 * sizeof(Id));
+  cbdata.cflmap = allochashtable(cbdata.cflmapn, 2);
   cbdata.create = 1;
   hdrfetches = 0;
   for (i = 0; i < pkgs->count; i++)
@@ -978,11 +981,11 @@ pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, in
       now = solv_timems(0);
       addfilesspace(&cbdata, 1);       /* make sure the first offset is not zero */
       cbdata.normapn = mkmask((cutoff + 3) * 4);
-      cbdata.normap = solv_calloc(cbdata.normapn + 1, 2 * sizeof(Id));
+      cbdata.normap = allochashtable(cbdata.normapn, 2);
       if (cbdata.usestat)
        {
          cbdata.statmapn = cbdata.normapn;
-         cbdata.statmap = solv_calloc(cbdata.statmapn + 1, 2 * sizeof(Id));
+         cbdata.statmap = allochashtable(cbdata.statmapn, 2);
        }
       cbdata.create = 0;
       hdrfetches = 0;
@@ -1088,10 +1091,10 @@ pool_findfileconflicts(Pool *pool, Queue *pkgs, int cutoff, Queue *conflicts, in
       cbdata.fetchmapn = mkmask(cbdata.lookat.count + 3);
       if (cbdata.fetchmapn < 4095)
         cbdata.fetchmapn = 4095;
-      cbdata.fetchmap = solv_calloc(cbdata.fetchmapn + 1, sizeof(Id));
+      cbdata.fetchmap = allochashtable(cbdata.fetchmapn, 1);
       if (cbdata.aliases)
        {
-         cbdata.fetchdirmapn = ((cbdata.fetchmapn + 1) / 16) - 1;
+         cbdata.fetchdirmapn = (((size_t)cbdata.fetchmapn + 1) / 16) - 1;
          map_init(&cbdata.fetchdirmap, cbdata.fetchdirmapn + 1);
        }
     }

diff --git a/ext/repo_arch.c b/ext/repo_arch.c
index 1bea691cbcd471a23f0a09034f16ca9ab50d4976..5ef5e465ff94664fe23256a7718db4a819e1a9cd 100644 (file)
--- a/ext/repo_arch.c
+++ b/ext/repo_arch.c
@@ -203,7 +203,7 @@ static Hashtable
 joinhash_init(Repo *repo, Hashval *hmp)
 {
   Hashval hm = mkmask(repo->nsolvables);
-  Hashtable ht = solv_calloc(hm + 1, sizeof(*ht));
+  Hashtable ht = allochashtable(hm, 1);
   Hashval h, hh;
   Solvable *s;
   int i;

diff --git a/ext/repo_mdk.c b/ext/repo_mdk.c
index 9097e6fa7b950ba9c51a89cdea63691a2b2c0f1c..9ecd8551c602f9e7ce715a9f3606910ea764befa 100644 (file)
--- a/ext/repo_mdk.c
+++ b/ext/repo_mdk.c
@@ -273,7 +273,7 @@ static Hashtable
 joinhash_init(Repo *repo, Hashval *hmp)
 {
   Hashval hm = mkmask(repo->nsolvables);
-  Hashtable ht = solv_calloc(hm + 1, sizeof(*ht));
+  Hashtable ht = allochashtable(hm, 1);
   Hashval h, hh;
   Solvable *s;
   int i;

diff --git a/ext/repo_susetags.c b/ext/repo_susetags.c
index 77e4a57eacbb7a9fcb7e82d153f64f562bca65ff..057f2f8aaadb5e900fb8ff067b6dc0cb473e7e69 100644 (file)
--- a/ext/repo_susetags.c
+++ b/ext/repo_susetags.c
@@ -287,7 +287,7 @@ static Hashtable
 joinhash_init(Repo *repo, Hashval *hmp)
 {
   Hashval hm = mkmask(repo->nsolvables);
-  Hashtable ht = solv_calloc(hm + 1, sizeof(*ht));
+  Hashtable ht = allochashtable(hm, 1);
   Hashval h, hh;
   Solvable *s;
   int i;

diff --git a/ext/solv_xmlparser.c b/ext/solv_xmlparser.c
index 8c5592636eda554c5de124c90fa9634d0a65cd2b..385a23d21aeec06214736e1fc56b083417a3a6d1 100644 (file)
--- a/ext/solv_xmlparser.c
+++ b/ext/solv_xmlparser.c
@@ -203,7 +203,7 @@ solv_xmlparser_init(struct solv_xmlparser *xmlp,
   xmlp->elements = elements;
   xmlp->nelements = nelements;
   hm = mkmask(nelements);
-  elementhelper = solv_calloc(hm + 1, sizeof(Id));
+  elementhelper = allochashtable(hm, 1);
   for (i = 0; i < nelements; i++)
     {
       h = hash_state_name(elements[i].fromstate, elements[i].element) & hm;

diff --git a/src/dirpool.c b/src/dirpool.c
index 8d49e029cd06f44e675c8504045e3c6aec9861d3..c9bd978fa393522aa1685ac06ab3cbb49603b70c 100644 (file)
--- a/src/dirpool.c
+++ b/src/dirpool.c
@@ -118,7 +118,7 @@ dirpool_resize_hash(Dirpool *dp, int numnew)
     return;
   dp->dirhashmask = hm;
   solv_free(dp->dirhashtbl);
-  ht = dp->dirhashtbl = (Hashtable)solv_calloc(hm + 1, sizeof(Id));
+  ht = dp->dirhashtbl = allochashtable(hm, 1);
   for (i = 2; i < dp->ndirs; i++)
     {
       if (dp->dirs[i] <= 0)

diff --git a/src/hash.h b/src/hash.h
index 4f595bbb17e9417142c4e2e97be271e7767d16f3..118dcd4db30b089c00144b07154da9c6621e4361 100644 (file)
--- a/src/hash.h
+++ b/src/hash.h
@@ -14,6 +14,7 @@
 #define LIBSOLV_HASH_H
 
 #include "pooltypes.h"
+#include "util.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -80,12 +81,22 @@ relhash(Id name, Id evr, int flags)
 static inline Hashval
 mkmask(unsigned int num)
 {
+  if (num >= 0x3ffffffe)
+    return num >= 0x80000000 ? 0 : 0xffffffff;
   num = num * 2 + 3;
   while (num & (num - 1))
     num &= num - 1;
   return num * 2 - 1;
 }
 
+static inline Hashtable
+allochashtable(Hashval mask, size_t size)
+{
+  if (mask == 0 && ((size_t)mask + 1) == 0)
+    solv_oom((size_t)mask, size * sizeof(Id));
+  return solv_calloc((size_t)mask + 1, size * sizeof(Id));
+}
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/src/poolid.c b/src/poolid.c
index e20b461efe01990b97d6a9d6b2b21ecd47fcdce7..4dedf910efa9d48b96cd150187107800aa32bbe7 100644 (file)
--- a/src/poolid.c
+++ b/src/poolid.c
@@ -93,7 +93,7 @@ pool_resize_rels_hash(Pool *pool, int numnew)
   /* realloc hash table */
   pool->relhashmask = hashmask;
   solv_free(pool->relhashtbl);
-  pool->relhashtbl = hashtbl = solv_calloc(hashmask + 1, sizeof(Id));
+  pool->relhashtbl = hashtbl = allochashtable(hashmask, 1);
 
   /* rehash all rels into new hashtable */
   for (i = 1, rd = pool->rels + i; i < pool->nrels; i++, rd++)

diff --git a/src/repo.c b/src/repo.c
index 6f6b85bfea3ddf8434a9b3523ed1b45cf2a47e80..625eb2f1c0109ad8d904c0d09d8e2f764f6b9775 100644 (file)
--- a/src/repo.c
+++ b/src/repo.c
@@ -399,9 +399,11 @@ repo_addid_dep_hash(Repo *repo, Offset olddeps, Id id, Id marker, int size)
       if ((Hashval)size * 2 > repo->lastidhash_mask)
        {
          repo->lastidhash_mask = mkmask(size < REPO_ADDID_DEP_HASHMIN ? REPO_ADDID_DEP_HASHMIN : size);
-         repo->lastidhash = solv_realloc2(repo->lastidhash, repo->lastidhash_mask + 1, sizeof(Id));
+         solv_free(repo->lastidhash);
+         repo->lastidhash = allochashtable(repo->lastidhash_mask, 1);
        }
-      memset(repo->lastidhash, 0, (repo->lastidhash_mask + 1) * sizeof(Id));
+      else
+        memset(repo->lastidhash, 0, ((size_t)repo->lastidhash_mask + 1) * sizeof(Id));
       for (oidp = repo->idarraydata + olddeps; (oid = *oidp) != 0; oidp++)
        {
          h = oid & repo->lastidhash_mask;

diff --git a/src/strpool.c b/src/strpool.c
index 8d7c1212ece60f5d3deefaa2fbfc3f04d0163184..23df7f1260618d74ce0393713f6159241d5e0145 100644 (file)
--- a/src/strpool.c
+++ b/src/strpool.c
@@ -90,7 +90,7 @@ stringpool_resize_hash(Stringpool *ss, int numnew)
   /* realloc hash table */
   ss->stringhashmask = hashmask;
   solv_free(ss->stringhashtbl);
-  ss->stringhashtbl = hashtbl = (Hashtable)solv_calloc(hashmask + 1, sizeof(Id));
+  ss->stringhashtbl = hashtbl = allochashtable(hashmask, 1);
   
   /* rehash all strings into new hashtable */
   for (i = 1; i < ss->nstrings; i++)