pdns/cachecleaner.hh

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #pragma once
  23
  24 #include <mutex>
  25 #include "lock.hh"
  26
  27 // this function can clean any cache that has a getTTD() method on its entries, a preRemoval() method and a 'sequence' index as its second index
  28 // the ritual is that the oldest entries are in *front* of the sequence collection, so on a hit, move an item to the end
  29 // on a miss, move it to the beginning
  30 template <typename S, typename C, typename T> void pruneCollection(C& container, T& collection, unsigned int maxCached, unsigned int scanFraction=1000)
  31 {
  32   time_t now=time(0);
  33   unsigned int toTrim=0;
  34
  35   unsigned int cacheSize=collection.size();
  36
  37   if(cacheSize > maxCached) {
  38     toTrim = cacheSize - maxCached;
  39   }
  40
  41 //  cout<<"Need to trim "<<toTrim<<" from cache to meet target!\n";
  42
  43   typedef typename T::template index<S>::type sequence_t;
  44   sequence_t& sidx=collection.template get<S>();
  45
  46   unsigned int tried=0, lookAt, erased=0;
  47
  48   // two modes - if toTrim is 0, just look through 1/scanFraction of all records
  49   // and nuke everything that is expired
  50   // otherwise, scan first 5*toTrim records, and stop once we've nuked enough
  51   if(toTrim)
  52     lookAt=5*toTrim;
  53   else
  54     lookAt=cacheSize/scanFraction;
  55
  56   typename sequence_t::iterator iter=sidx.begin(), eiter;
  57   for(; iter != sidx.end() && tried < lookAt ; ++tried) {
  58     if(iter->getTTD() < now) {
  59       container.preRemoval(*iter);
  60       iter = sidx.erase(iter);
  61       erased++;
  62     }
  63     else
  64       ++iter;
  65
  66     if(toTrim && erased >= toTrim)
  67       break;
  68   }
  69
  70   //cout<<"erased "<<erased<<" records based on ttd\n";
  71
  72   if(erased >= toTrim) // done
  73     return;
  74
  75   toTrim -= erased;
  76
  77   //if(toTrim)
  78     // cout<<"Still have "<<toTrim - erased<<" entries left to erase to meet target\n";
  79
  80   eiter=iter=sidx.begin();
  81   std::advance(eiter, toTrim);
  82   // just lob it off from the beginning
  83   for (auto i = iter; ; ) {
  84     if (i == eiter) {
  85       break;
  86     }
  87
  88     container.preRemoval(*i);
  89     sidx.erase(i++);
  90   }
  91 }
  92
  93 // note: this expects iterator from first index
  94 template <typename S, typename T> void moveCacheItemToFrontOrBack(T& collection, typename T::iterator& iter, bool front)
  95 {
  96   typedef typename T::template index<S>::type sequence_t;
  97   sequence_t& sidx=collection.template get<S>();
  98   typename sequence_t::iterator si=collection.template project<S>(iter);
  99   if(front)
 100     sidx.relocate(sidx.begin(), si); // at the beginning of the delete queue
 101   else
 102     sidx.relocate(sidx.end(), si);  // back
 103 }
 104
 105 template <typename S, typename T> void moveCacheItemToFront(T& collection, typename T::iterator& iter)
 106 {
 107   moveCacheItemToFrontOrBack<S>(collection, iter, true);
 108 }
 109
 110 template <typename S, typename T> void moveCacheItemToBack(T& collection, typename T::iterator& iter)
 111 {
 112   moveCacheItemToFrontOrBack<S>(collection, iter, false);
 113 }
 114
 115 template <typename S, typename T> uint64_t pruneLockedCollectionsVector(vector<T>& maps, uint64_t maxCached, uint64_t cacheSize)
 116 {
 117   time_t now = time(nullptr);
 118   uint64_t totErased = 0;
 119   uint64_t toTrim = 0;
 120   uint64_t lookAt = 0;
 121
 122   // two modes - if toTrim is 0, just look through 10%  of the cache and nuke everything that is expired
 123   // otherwise, scan first 5*toTrim records, and stop once we've nuked enough
 124   if (maxCached && cacheSize > maxCached) {
 125     toTrim = cacheSize - maxCached;
 126     lookAt = 5 * toTrim;
 127   } else {
 128     lookAt = cacheSize / 10;
 129   }
 130
 131   for(auto& mc : maps) {
 132     WriteLock wl(&mc.d_mut);
 133     auto& sidx = boost::multi_index::get<S>(mc.d_map);
 134     uint64_t erased = 0, lookedAt = 0;
 135     for(auto i = sidx.begin(); i != sidx.end(); lookedAt++) {
 136       if (i->getTTD() < now) {
 137         i = sidx.erase(i);
 138         erased++;
 139       } else {
 140         ++i;
 141       }
 142
 143       if(toTrim && erased > toTrim / maps.size())
 144         break;
 145
 146       if(lookedAt > lookAt / maps.size())
 147         break;
 148     }
 149     totErased += erased;
 150   }
 151
 152   return totErased;
 153 }
 154
 155 template <typename S, typename C, typename T> uint64_t pruneMutexCollectionsVector(C& container, vector<T>& maps, uint64_t maxCached, uint64_t cacheSize)
 156 {
 157   time_t now = time(nullptr);
 158   uint64_t totErased = 0;
 159   uint64_t toTrim = 0;
 160   uint64_t lookAt = 0;
 161
 162   // two modes - if toTrim is 0, just look through 10%  of the cache and nuke everything that is expired
 163   // otherwise, scan first 5*toTrim records, and stop once we've nuked enough
 164   if (cacheSize > maxCached) {
 165     toTrim = cacheSize - maxCached;
 166     lookAt = 5 * toTrim;
 167   } else {
 168     lookAt = cacheSize / 10;
 169   }
 170
 171   for (auto& mc : maps) {
 172     const std::lock_guard<std::mutex> lock(mc.mutex);
 173     auto& sidx = boost::multi_index::get<S>(mc.d_map);
 174     uint64_t erased = 0, lookedAt = 0;
 175     for (auto i = sidx.begin(); i != sidx.end(); lookedAt++) {
 176       if (i->getTTD() < now) {
 177         container.preRemoval(*i);
 178         i = sidx.erase(i);
 179         erased++;
 180       } else {
 181         ++i;
 182       }
 183
 184       if (toTrim && erased >= toTrim / maps.size())
 185         break;
 186
 187       if (lookedAt > lookAt / maps.size())
 188         break;
 189     }
 190     totErased += erased;
 191     if (toTrim && totErased >= toTrim)
 192       break;
 193   }
 194
 195   if (totErased >= toTrim) { // done
 196     return totErased;
 197   }
 198
 199   toTrim -= totErased;
 200
 201   // XXXX kinda desperate method
 202   while (toTrim > 0) {
 203     for (auto& mc : maps) {
 204       const std::lock_guard<std::mutex> lock(mc.mutex);
 205       auto& sidx = boost::multi_index::get<S>(mc.d_map);
 206       auto i = sidx.begin();
 207       container.preRemoval(*i);
 208       i = sidx.erase(i);
 209       totErased++;
 210       toTrim--;
 211       if (toTrim == 0)
 212         break;
 213     }
 214   }
 215   return totErased;
 216 }
 217
 218 template <typename T> uint64_t purgeLockedCollectionsVector(vector<T>& maps)
 219 {
 220   uint64_t delcount=0;
 221
 222   for(auto& mc : maps) {
 223     WriteLock wl(&mc.d_mut);
 224     delcount += mc.d_map.size();
 225     mc.d_map.clear();
 226   }
 227
 228   return delcount;
 229 }
 230
 231 template <typename N, typename T> uint64_t purgeLockedCollectionsVector(vector<T>& maps, const std::string& match)
 232 {
 233   uint64_t delcount=0;
 234   string prefix(match);
 235   prefix.resize(prefix.size()-1);
 236   DNSName dprefix(prefix);
 237   for(auto& mc : maps) {
 238     WriteLock wl(&mc.d_mut);
 239     auto& idx = boost::multi_index::get<N>(mc.d_map);
 240     auto iter = idx.lower_bound(dprefix);
 241     auto start = iter;
 242
 243     for(; iter != idx.end(); ++iter) {
 244       if(!iter->qname.isPartOf(dprefix)) {
 245         break;
 246       }
 247       delcount++;
 248     }
 249     idx.erase(start, iter);
 250   }
 251
 252   return delcount;
 253 }
 254
 255 template <typename N, typename T> uint64_t purgeExactLockedCollection(T& mc, const DNSName& qname)
 256 {
 257   uint64_t delcount=0;
 258   WriteLock wl(&mc.d_mut);
 259   auto& idx = boost::multi_index::get<N>(mc.d_map);
 260   auto range = idx.equal_range(qname);
 261   if(range.first != range.second) {
 262     delcount += distance(range.first, range.second);
 263     idx.erase(range.first, range.second);
 264   }
 265
 266   return delcount;
 267 }
 268
 269 template<typename S, typename Index>
 270 std::pair<typename Index::iterator,bool>
 271 lruReplacingInsert(Index& i,const typename Index::value_type& x)
 272 {
 273   std::pair<typename Index::iterator,bool> res = i.insert(x);
 274   if (!res.second) {
 275     moveCacheItemToBack<S>(i, res.first);
 276     res.second = i.replace(res.first, x);
 277   }
 278   return res;
 279 }