src/refresh.cc

   1 /*
   2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 22    Refresh Calculation */
  10
  11 #ifndef USE_POSIX_REGEX
  12 #define USE_POSIX_REGEX     /* put before includes; always use POSIX */
  13 #endif
  14
  15 #include "squid.h"
  16 #include "HttpHdrCc.h"
  17 #include "HttpReply.h"
  18 #include "HttpRequest.h"
  19 #include "MemObject.h"
  20 #include "mgr/Registration.h"
  21 #include "RefreshPattern.h"
  22 #include "SquidConfig.h"
  23 #include "SquidTime.h"
  24 #include "Store.h"
  25 #include "util.h"
  26
  27 typedef enum {
  28     rcHTTP,
  29     rcICP,
  30 #if USE_HTCP
  31     rcHTCP,
  32 #endif
  33 #if USE_CACHE_DIGESTS
  34     rcCDigest,
  35 #endif
  36     rcStore,
  37     rcCount
  38 } refreshCountsEnum;
  39
  40 /**
  41  * Flags indicating which staleness algorithm has been applied.
  42  */
  43 typedef struct {
  44     bool expires;  ///< Expires: header absolute timestamp limit
  45     bool min;      ///< Heuristic minimum age limited
  46     bool lmfactor; ///< Last-Modified with heuristic determines limit
  47     bool max;      ///< Configured maximum age limit
  48 } stale_flags;
  49
  50 /*
  51  * This enumerated list assigns specific values, ala HTTP/FTP status
  52  * codes.  All Fresh codes are in the range 100-199 and all stale
  53  * codes are 200-299.  We might want to use these codes in logging,
  54  * so best to keep them consistent over time.
  55  */
  56 enum {
  57     FRESH_REQUEST_MAX_STALE_ALL = 100,
  58     FRESH_REQUEST_MAX_STALE_VALUE,
  59     FRESH_EXPIRES,
  60     FRESH_LMFACTOR_RULE,
  61     FRESH_MIN_RULE,
  62     FRESH_OVERRIDE_EXPIRES,
  63     FRESH_OVERRIDE_LASTMOD,
  64     STALE_MUST_REVALIDATE = 200,
  65     STALE_RELOAD_INTO_IMS,
  66     STALE_FORCED_RELOAD,
  67     STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
  68     STALE_EXPIRES,
  69     STALE_MAX_RULE,
  70     STALE_LMFACTOR_RULE,
  71     STALE_MAX_STALE,
  72     STALE_DEFAULT = 299
  73 };
  74
  75 static struct RefreshCounts {
  76     const char *proto;
  77     int total;
  78     int status[STALE_DEFAULT + 1];
  79 } refreshCounts[rcCount];
  80
  81 static const RefreshPattern *refreshUncompiledPattern(const char *);
  82 static OBJH refreshStats;
  83 static int refreshStaleness(const StoreEntry * entry, time_t check_time, const time_t age, const RefreshPattern * R, stale_flags * sf);
  84
  85 static RefreshPattern DefaultRefresh("<none>", 0);
  86
  87 /** Locate the first refresh_pattern rule that matches the given URL by regex.
  88  *
  89  * \return A pointer to the refresh_pattern parameters to use, or nullptr if there is no match.
  90  */
  91 const RefreshPattern *
  92 refreshLimits(const char *url)
  93 {
  94     for (auto R = Config.Refresh; R; R = R->next) {
  95         ++(R->stats.matchTests);
  96         if (R->pattern.match(url)) {
  97             ++(R->stats.matchCount);
  98             return R;
  99         }
 100     }
 101
 102     return nullptr;
 103 }
 104
 105 /** Locate the first refresh_pattern rule that has the given uncompiled regex.
 106  *
 107  * \note There is only one reference to this function, below. It always passes "." as the pattern.
 108  * This function is only ever called if there is no URI. Because a regex match is impossible, Squid
 109  * forces the "." rule to apply (if it exists)
 110  *
 111  * \return A pointer to the refresh_pattern parameters to use, or nullptr if there is no match.
 112  */
 113 static const RefreshPattern *
 114 refreshUncompiledPattern(const char *pat)
 115 {
 116     for (auto R = Config.Refresh; R; R = R->next) {
 117         if (0 == strcmp(R->pattern.c_str(), pat))
 118             return R;
 119     }
 120
 121     return nullptr;
 122 }
 123
 124 /**
 125  * Calculate how stale the response is (or will be at the check_time).
 126  *
 127  * We try the following ways until one gives a result:
 128  *
 129  * 1. response expiration time, if one was set
 130  * 2. age greater than configured maximum
 131  * 3. last-modified factor algorithm
 132  * 4. age less than configured minimum
 133  * 5. default (stale)
 134  *
 135  * \param entry       the StoreEntry being examined
 136  * \param check_time  the time (maybe future) at which we want to know whether $
 137  * \param age         the age of the entry at check_time
 138  * \param R           the refresh_pattern rule that matched this entry
 139  * \param sf          small struct to indicate reason for stale/fresh decision
 140  *
 141  * \retval -1  If the response is fresh.
 142  * \retval >0  The amount of staleness.
 143  * \retval 0   NOTE return value of 0 means the response is stale.
 144  */
 145 static int
 146 refreshStaleness(const StoreEntry * entry, time_t check_time, const time_t age, const RefreshPattern * R, stale_flags * sf)
 147 {
 148     // 1. If the cached object has an explicit expiration time, then we rely on this and
 149     //    completely ignore the Min, Percent and Max values in the refresh_pattern.
 150     if (entry->expires > -1) {
 151         sf->expires = true;
 152
 153         if (entry->expires > check_time) {
 154             debugs(22, 3, "FRESH: expires " << entry->expires <<
 155                    " >= check_time " << check_time << " ");
 156
 157             return -1;
 158         } else {
 159             debugs(22, 3, "STALE: expires " << entry->expires <<
 160                    " < check_time " << check_time << " ");
 161
 162             return (check_time - entry->expires);
 163         }
 164     }
 165
 166     debugs(22, 3, "No explicit expiry given, using heuristics to determine freshness");
 167
 168     // 2. If the entry is older than the maximum age in the refresh_pattern, it is STALE.
 169     if (age > R->max) {
 170         debugs(22, 3, "STALE: age " << age << " > max " << R->max << " ");
 171         sf->max = true;
 172         return (age - R->max);
 173     }
 174
 175     // 3. If there is a Last-Modified header, try the last-modified factor algorithm.
 176     const time_t lastmod_delta = entry->timestamp - entry->lastModified();
 177     if (lastmod_delta > 0) {
 178         /* stale_age is the age of the response when it became/becomes stale according to
 179          * the last-modified factor algorithm. It's how long we can consider the response
 180          * fresh from the time we cached it.
 181          */
 182         time_t stale_age = static_cast<time_t>(lastmod_delta * R->pct);
 183
 184         debugs(22,3, "Last modified " << lastmod_delta << " sec before we cached it, L-M factor " <<
 185                (100.0 * R->pct) << "% = " << stale_age << " sec freshness lifetime");
 186         sf->lmfactor = true;
 187
 188         if (age >= stale_age) {
 189             debugs(22, 3, "STALE: age " << age << " > stale_age " << stale_age);
 190             return (age - stale_age);
 191         } else {
 192             debugs(22, 3, "FRESH: age " << age << " <= stale_age " << stale_age);
 193             return -1;
 194         }
 195     }
 196
 197     // 4. If the entry is not as old as the minimum age in the refresh_pattern, it is FRESH.
 198     if (age < R->min) {
 199         debugs(22, 3, "FRESH: age (" << age << " sec) is less than configured minimum (" << R->min << " sec)");
 200         sf->min = true;
 201         return -1;
 202     }
 203
 204     // 5. default is stale, by the amount we missed the minimum by
 205     debugs(22, 3, "STALE: No explicit expiry, no last modified, and older than configured minimum.");
 206     return (age - R->min);
 207 }
 208
 209 /** Checks whether a store entry is fresh or stale, and why.
 210  *
 211  * This is where all aspects of request, response and squid configuration
 212  * meet to decide whether a response is cacheable or not:
 213  *
 214  * 1. Client request headers that affect cacheability, e.g.
 215  *  - Cache-Control: no-cache
 216  *  - Cache-Control: max-age=N
 217  *  - Cache-Control: max-stale[=N]
 218  *  - Pragma: no-cache
 219  *
 220  * 2. Server response headers that affect cacheability, e.g.
 221  *  - Age:
 222  *  - Cache-Control: proxy-revalidate
 223  *  - Cache-Control: must-revalidate
 224  *  - Cache-Control: no-cache
 225  *  - Cache-Control: max-age=N
 226  *  - Cache-Control: s-maxage=N
 227  *  - Date:
 228  *  - Expires:
 229  *  - Last-Modified:
 230  *
 231  * 3. Configuration options, e.g.
 232  *  - reload-into-ims (refresh_pattern)
 233  *  - ignore-reload (refresh_pattern)
 234  *  - refresh-ims (refresh_pattern)
 235  *  - override-lastmod (refresh_pattern)
 236  *  - override-expire (refresh_pattern)
 237  *  - reload_into_ims (global option)
 238  *  - refresh_all_ims (global option)
 239  *
 240  * \returns a status code (from enum above):
 241  *  - FRESH_REQUEST_MAX_STALE_ALL
 242  *  - FRESH_REQUEST_MAX_STALE_VALUE
 243  *  - FRESH_EXPIRES
 244  *  - FRESH_LMFACTOR_RULE
 245  *  - FRESH_MIN_RULE
 246  *  - FRESH_OVERRIDE_EXPIRES
 247  *  - FRESH_OVERRIDE_LASTMOD
 248  *  - STALE_MUST_REVALIDATE
 249  *  - STALE_RELOAD_INTO_IMS
 250  *  - STALE_FORCED_RELOAD
 251  *  - STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE
 252  *  - STALE_EXPIRES
 253  *  - STALE_MAX_RULE
 254  *  - STALE_LMFACTOR_RULE
 255  *  - STALE_MAX_STALE
 256  *  - STALE_DEFAULT
 257  *
 258  * \note request may be NULL (e.g. for cache digests build)
 259  *
 260  * \note the store entry being examined is not necessarily cached (e.g. if
 261  *       this response is being evaluated for the first time)
 262  */
 263 static int
 264 refreshCheck(const StoreEntry * entry, HttpRequest * request, time_t delta)
 265 {
 266     time_t age = 0;
 267     time_t check_time = squid_curtime + delta;
 268     int staleness;
 269     stale_flags sf;
 270
 271     // get the URL of this entry, if there is one
 272     static const SBuf nilUri("<none>");
 273     SBuf uri = nilUri;
 274     if (entry->mem_obj)
 275         uri = entry->mem_obj->storeId();
 276     else if (request)
 277         uri = request->effectiveRequestUri();
 278
 279     debugs(22, 3, "checking freshness of URI: " << uri);
 280
 281     // age is not necessarily the age now, but the age at the given check_time
 282     if (check_time > entry->timestamp)
 283         age = check_time - entry->timestamp;
 284
 285     // XXX: what to do when age < 0 or counter overflow?
 286     assert(age >= 0);
 287
 288     /* We need a refresh rule. In order of preference:
 289      *
 290      *   1. the rule that matches this URI by regex
 291      *   2. the "." rule from the config file
 292      *   3. the default "." rule
 293      */
 294     // XXX: performance regression. c_str() reallocates
 295     const RefreshPattern *R = (uri != nilUri) ? refreshLimits(uri.c_str()) : refreshUncompiledPattern(".");
 296     if (NULL == R)
 297         R = &DefaultRefresh;
 298
 299     debugs(22, 3, "Matched '" << R->pattern.c_str() << " " <<
 300            (int) R->min << " " << (int) (100.0 * R->pct) << "%% " <<
 301            (int) R->max << "'");
 302
 303     debugs(22, 3, "\tage:\t" << age);
 304
 305     debugs(22, 3, "\tcheck_time:\t" << mkrfc1123(check_time));
 306
 307     debugs(22, 3, "\tentry->timestamp:\t" << mkrfc1123(entry->timestamp));
 308
 309     if (request && !request->flags.ignoreCc) {
 310         const HttpHdrCc *const cc = request->cache_control;
 311         int minFresh = -1;
 312         if (cc && cc->hasMinFresh(&minFresh)) {
 313             debugs(22, 3, "\tage + min-fresh:\t" << age << " + " <<
 314                    minFresh << " = " << age + minFresh);
 315             debugs(22, 3, "\tcheck_time + min-fresh:\t" << check_time << " + "
 316                    << minFresh << " = " <<
 317                    mkrfc1123(check_time + minFresh));
 318             age += minFresh;
 319             check_time += minFresh;
 320         }
 321     }
 322
 323     memset(&sf, '\0', sizeof(sf));
 324
 325     staleness = refreshStaleness(entry, check_time, age, R, &sf);
 326
 327     debugs(22, 3, "Staleness = " << staleness);
 328
 329     const auto reply = entry->hasFreshestReply(); // may be nil
 330
 331     // stale-if-error requires any failure be passed thru when its period is over.
 332     int staleIfError = -1;
 333     if (request && reply && reply->cache_control &&
 334             reply->cache_control->hasStaleIfError(&staleIfError) &&
 335             staleIfError < staleness) {
 336
 337         debugs(22, 3, "stale-if-error period expired. Will produce error if validation fails.");
 338         request->flags.failOnValidationError = true;
 339     }
 340
 341     /* If the origin server specified either of:
 342      *   Cache-Control: must-revalidate
 343      *   Cache-Control: proxy-revalidate
 344      * the spec says the response must always be revalidated if stale.
 345      */
 346     const bool revalidateAlways = EBIT_TEST(entry->flags, ENTRY_REVALIDATE_ALWAYS);
 347     if (revalidateAlways || (staleness > -1 &&
 348                              EBIT_TEST(entry->flags, ENTRY_REVALIDATE_STALE))) {
 349         debugs(22, 3, "YES: Must revalidate stale object (origin set " <<
 350                (revalidateAlways ? "no-cache or private" :
 351                 "must-revalidate, proxy-revalidate or s-maxage") << ")");
 352         if (request)
 353             request->flags.failOnValidationError = true;
 354         return STALE_MUST_REVALIDATE;
 355     }
 356
 357     /* request-specific checks */
 358     if (request && !request->flags.ignoreCc) {
 359         HttpHdrCc *cc = request->cache_control;
 360
 361         /* If the request is an IMS request, and squid is configured NOT to service this from cache
 362          * (either by 'refresh-ims' in the refresh pattern or 'refresh_all_ims on' globally)
 363          * then force a reload from the origin.
 364          */
 365         if (request->flags.ims && (R->flags.refresh_ims || Config.onoff.refresh_all_ims)) {
 366             // The client's no-cache header is changed into a IMS query
 367             debugs(22, 3, "YES: Client IMS request forcing revalidation of object (refresh-ims option)");
 368             return STALE_FORCED_RELOAD;
 369         }
 370
 371 #if USE_HTTP_VIOLATIONS
 372         /* Normally a client reload request ("Cache-Control: no-cache" or "Pragma: no-cache")
 373          * means we must treat this response as STALE and fetch a new one.
 374          *
 375          * However, some options exist to override this behaviour. For example, we might just
 376          * revalidate our existing response, or even just serve it up without revalidating it.
 377          *
 378          *     ---- Note on the meaning of nocache_hack -----
 379          *
 380          * The nocache_hack flag has a very specific and complex meaning:
 381          *
 382          * (a) this is a reload request ("Cache-Control: no-cache" or "Pragma: no-cache" header)
 383          * and (b) the configuration file either has at least one refresh_pattern with
 384          * ignore-reload or reload-into-ims (not necessarily the rule matching this request) or
 385          * the global reload_into_ims is set to on
 386          *
 387          * In other words: this is a client reload, and we might need to override
 388          * the default behaviour (but we might not).
 389          *
 390          * "nocache_hack" is a pretty deceptive name for such a complicated meaning.
 391          */
 392         if (request->flags.noCacheHack()) {
 393
 394             if (R->flags.ignore_reload) {
 395                 /* The client's no-cache header is ignored completely - we'll try to serve
 396                  * what we have (assuming it's still fresh, etc.)
 397                  */
 398                 debugs(22, 3, "MAYBE: Ignoring client reload request - trying to serve from cache (ignore-reload option)");
 399             } else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) {
 400                 /* The client's no-cache header is not honoured completely - we'll just try
 401                  * to revalidate our cached copy (IMS to origin) instead of fetching a new
 402                  * copy with an unconditional GET.
 403                  */
 404                 debugs(22, 3, "YES: Client reload request - cheating, only revalidating with origin (reload-into-ims option)");
 405                 return STALE_RELOAD_INTO_IMS;
 406             } else {
 407                 /* The client's no-cache header is honoured - we fetch a new copy from origin */
 408                 debugs(22, 3, "YES: Client reload request - fetching new copy from origin");
 409                 request->flags.noCache = true;
 410                 return STALE_FORCED_RELOAD;
 411             }
 412         }
 413 #endif
 414
 415         // Check the Cache-Control client request header
 416         if (NULL != cc) {
 417
 418             // max-age directive
 419             int maxAge = -1;
 420             if (cc->hasMaxAge(&maxAge)) {
 421
 422                 // RFC 8246: reply contains CC:immutable then ignore client CC:max-age=N
 423                 if (reply && reply->cache_control && reply->cache_control->hasImmutable()) {
 424                     debugs(22, 3, "MAYBE: Ignoring client CC:max-age=" << maxAge << " request - 'Cache-Control: immutable'");
 425
 426 #if USE_HTTP_VIOLATIONS
 427                     // Ignore of client "Cache-Control: max-age=0" header
 428                 } else if (R->flags.ignore_reload && maxAge == 0) {
 429                     debugs(22, 3, "MAYBE: Ignoring client reload request - trying to serve from cache (ignore-reload option)");
 430 #endif
 431
 432                     // Honour client "Cache-Control: max-age=x" header
 433                 } else if (age > maxAge || maxAge == 0) {
 434                     debugs(22, 3, "YES: Revalidating object - client 'Cache-Control: max-age=" << maxAge << "'");
 435                     return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
 436                 }
 437             }
 438
 439             // max-stale directive
 440             int maxStale = -1;
 441             if (cc->hasMaxStale(&maxStale) && staleness > -1) {
 442                 if (maxStale==HttpHdrCc::MAX_STALE_ANY) {
 443                     debugs(22, 3, "NO: Client accepts a stale response of any age - 'Cache-Control: max-stale'");
 444                     return FRESH_REQUEST_MAX_STALE_ALL;
 445                 } else if (staleness < maxStale) {
 446                     debugs(22, 3, "NO: Client accepts a stale response - 'Cache-Control: max-stale=" << maxStale << "'");
 447                     return FRESH_REQUEST_MAX_STALE_VALUE;
 448                 }
 449             }
 450         }
 451     }
 452
 453     // If the object is fresh, return the right FRESH_ code
 454     if (-1 == staleness) {
 455         debugs(22, 3, "Object isn't stale..");
 456         if (sf.expires) {
 457             debugs(22, 3, "returning FRESH_EXPIRES");
 458             return FRESH_EXPIRES;
 459         }
 460
 461         assert(!sf.max);
 462
 463         if (sf.lmfactor) {
 464             debugs(22, 3, "returning FRESH_LMFACTOR_RULE");
 465             return FRESH_LMFACTOR_RULE;
 466         }
 467
 468         assert(sf.min);
 469
 470         debugs(22, 3, "returning FRESH_MIN_RULE");
 471         return FRESH_MIN_RULE;
 472     }
 473
 474     /*
 475      * At this point the response is stale, unless one of
 476      * the override options kicks in.
 477      * NOTE: max-stale config blocks the overrides.
 478      */
 479     int max_stale = (R->max_stale >= 0 ? R->max_stale : Config.maxStale);
 480     if ( max_stale >= 0 && staleness > max_stale) {
 481         debugs(22, 3, "YES: refresh_pattern max-stale=N limit from squid.conf");
 482         if (request)
 483             request->flags.failOnValidationError = true;
 484         return STALE_MAX_STALE;
 485     }
 486
 487     if (sf.expires) {
 488 #if USE_HTTP_VIOLATIONS
 489
 490         if (R->flags.override_expire && age < R->min) {
 491             debugs(22, 3, "NO: Serving from cache - even though explicit expiry has passed, we enforce Min value (override-expire option)");
 492             return FRESH_OVERRIDE_EXPIRES;
 493         }
 494
 495 #endif
 496         return STALE_EXPIRES;
 497     }
 498
 499     if (sf.max)
 500         return STALE_MAX_RULE;
 501
 502     if (sf.lmfactor) {
 503 #if USE_HTTP_VIOLATIONS
 504         if (R->flags.override_lastmod && age < R->min) {
 505             debugs(22, 3, "NO: Serving from cache - even though L-M factor says the object is stale, we enforce Min value (override-lastmod option)");
 506             return FRESH_OVERRIDE_LASTMOD;
 507         }
 508 #endif
 509         debugs(22, 3, "YES: L-M factor says the object is stale'");
 510         return STALE_LMFACTOR_RULE;
 511     }
 512
 513     debugs(22, 3, "returning STALE_DEFAULT");
 514     return STALE_DEFAULT;
 515 }
 516
 517 /**
 518  * This is called by http.cc once it has received and parsed the origin server's
 519  * response headers. It uses the result as part of its algorithm to decide whether a
 520  * response should be cached.
 521  *
 522  * \retval true if the entry is cacheable, regardless of whether FRESH or STALE
 523  * \retval false if the entry is not cacheable
 524  *
 525  * TODO: this algorithm seems a bit odd and might not be quite right. Verify against HTTPbis.
 526  */
 527 bool
 528 refreshIsCachable(const StoreEntry * entry)
 529 {
 530     /*
 531      * Don't look at the request to avoid no-cache and other nuisances.
 532      * the object should have a mem_obj so the URL will be found there.
 533      * minimum_expiry_time seconds delta (defaults to 60 seconds), to
 534      * avoid objects which expire almost immediately, and which can't
 535      * be refreshed.
 536      */
 537     int reason = refreshCheck(entry, NULL, Config.minimum_expiry_time);
 538     ++ refreshCounts[rcStore].total;
 539     ++ refreshCounts[rcStore].status[reason];
 540
 541     if (reason < STALE_MUST_REVALIDATE)
 542         /* Does not need refresh. This is certainly cachable */
 543         return true;
 544
 545     if (entry->lastModified() < 0)
 546         /* We should know entry's modification time to do a refresh */
 547         return false;
 548
 549     if (entry->mem_obj == NULL)
 550         /* no mem_obj? */
 551         return true;
 552
 553     if (entry->mem_obj->baseReply().content_length == 0)
 554         /* No use refreshing (caching?) 0 byte objects */
 555         return false;
 556
 557     /* This seems to be refreshable. Cache it */
 558     return true;
 559 }
 560
 561 /// whether reply is stale if it is a hit
 562 static bool
 563 refreshIsStaleIfHit(const int reason)
 564 {
 565     switch (reason) {
 566     case FRESH_MIN_RULE:
 567     case FRESH_LMFACTOR_RULE:
 568     case FRESH_EXPIRES:
 569         return false;
 570     default:
 571         return true;
 572     }
 573 }
 574
 575 /**
 576  * Protocol-specific wrapper around refreshCheck() function.
 577  *
 578  * Note the reason for STALE/FRESH then return true/false respectively.
 579  *
 580  * \retval 1 if STALE
 581  * \retval 0 if FRESH
 582  */
 583 int
 584 refreshCheckHTTP(const StoreEntry * entry, HttpRequest * request)
 585 {
 586     int reason = refreshCheck(entry, request, 0);
 587     ++ refreshCounts[rcHTTP].total;
 588     ++ refreshCounts[rcHTTP].status[reason];
 589     request->flags.staleIfHit = refreshIsStaleIfHit(reason);
 590     // TODO: Treat collapsed responses as fresh but second-hand.
 591     return (Config.onoff.offline || reason < 200) ? 0 : 1;
 592 }
 593
 594 /// \see int refreshCheckHTTP(const StoreEntry * entry, HttpRequest * request)
 595 int
 596 refreshCheckICP(const StoreEntry * entry, HttpRequest * request)
 597 {
 598     int reason = refreshCheck(entry, request, 30);
 599     ++ refreshCounts[rcICP].total;
 600     ++ refreshCounts[rcICP].status[reason];
 601     return (reason < 200) ? 0 : 1;
 602 }
 603
 604 #if USE_HTCP
 605 /// \see int refreshCheckHTTP(const StoreEntry * entry, HttpRequest * request)
 606 int
 607 refreshCheckHTCP(const StoreEntry * entry, HttpRequest * request)
 608 {
 609     int reason = refreshCheck(entry, request, 10);
 610     ++ refreshCounts[rcHTCP].total;
 611     ++ refreshCounts[rcHTCP].status[reason];
 612     return (reason < 200) ? 0 : 1;
 613 }
 614
 615 #endif
 616
 617 #if USE_CACHE_DIGESTS
 618 /// \see int refreshCheckHTTP(const StoreEntry * entry, HttpRequest * request)
 619 int
 620 refreshCheckDigest(const StoreEntry * entry, time_t delta)
 621 {
 622     int reason = refreshCheck(entry,
 623                               entry->mem_obj ? entry->mem_obj->request.getRaw() : nullptr,
 624                               delta);
 625     ++ refreshCounts[rcCDigest].total;
 626     ++ refreshCounts[rcCDigest].status[reason];
 627     return (reason < 200) ? 0 : 1;
 628 }
 629 #endif
 630
 631 /**
 632  * Get the configured maximum caching time for objects with this URL
 633  * according to refresh_pattern.
 634  *
 635  * Used by http.cc when generating a upstream requests to ensure that
 636  * responses it is given are fresh enough to be worth caching.
 637  *
 638  * \retval pattern-max if there is a refresh_pattern matching the URL configured.
 639  * \retval REFRESH_DEFAULT_MAX if there are no explicit limits configured
 640  */
 641 time_t
 642 getMaxAge(const char *url)
 643 {
 644     const RefreshPattern *R;
 645     debugs(22, 3, "getMaxAge: '" << url << "'");
 646
 647     if ((R = refreshLimits(url)))
 648         return R->max;
 649     else
 650         return REFRESH_DEFAULT_MAX;
 651 }
 652
 653 static int
 654 refreshCountsStatsEntry(StoreEntry * sentry, struct RefreshCounts &rc, int code, const char *desc)
 655 {
 656     storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", rc.status[code], xpercent(rc.status[code], rc.total), desc);
 657     return rc.status[code];
 658 }
 659
 660 static void
 661 refreshCountsStats(StoreEntry * sentry, struct RefreshCounts &rc)
 662 {
 663     if (!rc.total)
 664         return;
 665
 666     storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc.proto);
 667     storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n");
 668
 669     int sum = 0;
 670     sum += refreshCountsStatsEntry(sentry, rc, FRESH_REQUEST_MAX_STALE_ALL, "Fresh: request max-stale wildcard");
 671     sum += refreshCountsStatsEntry(sentry, rc, FRESH_REQUEST_MAX_STALE_VALUE, "Fresh: request max-stale value");
 672     sum += refreshCountsStatsEntry(sentry, rc, FRESH_EXPIRES, "Fresh: expires time not reached");
 673     sum += refreshCountsStatsEntry(sentry, rc, FRESH_LMFACTOR_RULE, "Fresh: refresh_pattern last-mod factor percentage");
 674     sum += refreshCountsStatsEntry(sentry, rc, FRESH_MIN_RULE, "Fresh: refresh_pattern min value");
 675     sum += refreshCountsStatsEntry(sentry, rc, FRESH_OVERRIDE_EXPIRES, "Fresh: refresh_pattern override-expires");
 676     sum += refreshCountsStatsEntry(sentry, rc, FRESH_OVERRIDE_LASTMOD, "Fresh: refresh_pattern override-lastmod");
 677     sum += refreshCountsStatsEntry(sentry, rc, STALE_MUST_REVALIDATE, "Stale: response has must-revalidate");
 678     sum += refreshCountsStatsEntry(sentry, rc, STALE_RELOAD_INTO_IMS, "Stale: changed reload into IMS");
 679     sum += refreshCountsStatsEntry(sentry, rc, STALE_FORCED_RELOAD, "Stale: request has no-cache directive");
 680     sum += refreshCountsStatsEntry(sentry, rc, STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE, "Stale: age exceeds request max-age value");
 681     sum += refreshCountsStatsEntry(sentry, rc, STALE_EXPIRES, "Stale: expires time reached");
 682     sum += refreshCountsStatsEntry(sentry, rc, STALE_MAX_RULE, "Stale: refresh_pattern max age rule");
 683     sum += refreshCountsStatsEntry(sentry, rc, STALE_LMFACTOR_RULE, "Stale: refresh_pattern last-mod factor percentage");
 684     sum += refreshCountsStatsEntry(sentry, rc, STALE_DEFAULT, "Stale: by default");
 685     storeAppendPrintf(sentry, "\n");
 686 }
 687
 688 static void
 689 refreshStats(StoreEntry * sentry)
 690 {
 691     // display per-rule counts of usage and tests
 692     storeAppendPrintf(sentry, "\nRefresh pattern usage:\n\n");
 693     storeAppendPrintf(sentry, "  Used      \tChecks    \t%% Matches\tPattern\n");
 694     for (const RefreshPattern *R = Config.Refresh; R; R = R->next) {
 695         storeAppendPrintf(sentry, "  %10" PRIu64 "\t%10" PRIu64 "\t%6.2f\t%s%s\n",
 696                           R->stats.matchCount,
 697                           R->stats.matchTests,
 698                           xpercent(R->stats.matchCount, R->stats.matchTests),
 699                           (R->pattern.flags&REG_ICASE ? "-i " : ""),
 700                           R->pattern.c_str());
 701     }
 702
 703     int i;
 704     int total = 0;
 705
 706     /* get total usage count */
 707
 708     for (i = 0; i < rcCount; ++i)
 709         total += refreshCounts[i].total;
 710
 711     /* protocol usage histogram */
 712     storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n");
 713
 714     storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n");
 715
 716     for (i = 0; i < rcCount; ++i)
 717         storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n",
 718                           refreshCounts[i].proto,
 719                           refreshCounts[i].total,
 720                           xpercent(refreshCounts[i].total, total));
 721
 722     /* per protocol histograms */
 723     storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n");
 724
 725     for (i = 0; i < rcCount; ++i)
 726         refreshCountsStats(sentry, refreshCounts[i]);
 727 }
 728
 729 static void
 730 refreshRegisterWithCacheManager(void)
 731 {
 732     Mgr::RegisterAction("refresh", "Refresh Algorithm Statistics", refreshStats, 0, 1);
 733 }
 734
 735 void
 736 refreshInit(void)
 737 {
 738     memset(refreshCounts, 0, sizeof(refreshCounts));
 739     refreshCounts[rcHTTP].proto = "HTTP";
 740     refreshCounts[rcICP].proto = "ICP";
 741 #if USE_HTCP
 742
 743     refreshCounts[rcHTCP].proto = "HTCP";
 744 #endif
 745
 746     refreshCounts[rcStore].proto = "On Store";
 747 #if USE_CACHE_DIGESTS
 748
 749     refreshCounts[rcCDigest].proto = "Cache Digests";
 750 #endif
 751
 752     refreshRegisterWithCacheManager();
 753 }
 754