src/refresh.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 22    Refresh Calculation
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #ifndef USE_POSIX_REGEX
  37 #define USE_POSIX_REGEX         /* put before includes; always use POSIX */
  38 #endif
  39
  40 #include "squid.h"
  41 #include "CacheManager.h"
  42 #include "Store.h"
  43 #include "MemObject.h"
  44 #include "HttpRequest.h"
  45 #include "HttpReply.h"
  46 #include "SquidTime.h"
  47
  48 typedef enum {
  49     rcHTTP,
  50     rcICP,
  51 #if USE_HTCP
  52     rcHTCP,
  53 #endif
  54 #if USE_CACHE_DIGESTS
  55     rcCDigest,
  56 #endif
  57     rcStore,
  58     rcCount
  59 } refreshCountsEnum;
  60
  61 typedef struct {
  62     bool expires;
  63     bool min;
  64     bool lmfactor;
  65     bool max;
  66 } stale_flags;
  67
  68 /*
  69  * This enumerated list assigns specific values, ala HTTP/FTP status
  70  * codes.  All Fresh codes are in the range 100-199 and all stale
  71  * codes are 200-299.  We might want to use these codes in logging,
  72  * so best to keep them consistent over time.
  73  */
  74 enum {
  75     FRESH_REQUEST_MAX_STALE_ALL = 100,
  76     FRESH_REQUEST_MAX_STALE_VALUE,
  77     FRESH_EXPIRES,
  78     FRESH_LMFACTOR_RULE,
  79     FRESH_MIN_RULE,
  80     FRESH_OVERRIDE_EXPIRES,
  81     FRESH_OVERRIDE_LASTMOD,
  82     STALE_MUST_REVALIDATE = 200,
  83     STALE_RELOAD_INTO_IMS,
  84     STALE_FORCED_RELOAD,
  85     STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
  86     STALE_EXPIRES,
  87     STALE_MAX_RULE,
  88     STALE_LMFACTOR_RULE,
  89     STALE_DEFAULT = 299
  90 };
  91
  92 static struct RefreshCounts {
  93     const char *proto;
  94     int total;
  95     int status[STALE_DEFAULT + 1];
  96 }
  97
  98 refreshCounts[rcCount];
  99
 100 /*
 101  * Defaults:
 102  *      MIN     NONE
 103  *      PCT     20%
 104  *      MAX     3 days
 105  */
 106 #define REFRESH_DEFAULT_MIN     (time_t)0
 107 #define REFRESH_DEFAULT_PCT     0.20
 108 #define REFRESH_DEFAULT_MAX     (time_t)259200
 109
 110 static const refresh_t *refreshUncompiledPattern(const char *);
 111 static OBJH refreshStats;
 112 static int refreshStaleness(const StoreEntry *, time_t, time_t, const refresh_t *, stale_flags *);
 113
 114 static refresh_t DefaultRefresh;
 115
 116 const refresh_t *
 117 refreshLimits(const char *url)
 118 {
 119     const refresh_t *R;
 120
 121     for (R = Config.Refresh; R; R = R->next) {
 122         if (!regexec(&(R->compiled_pattern), url, 0, 0, 0))
 123             return R;
 124     }
 125
 126     return NULL;
 127 }
 128
 129 static const refresh_t *
 130 refreshUncompiledPattern(const char *pat)
 131 {
 132     const refresh_t *R;
 133
 134     for (R = Config.Refresh; R; R = R->next) {
 135         if (0 == strcmp(R->pattern, pat))
 136             return R;
 137     }
 138
 139     return NULL;
 140 }
 141
 142 /**
 143  * Calculate how stale the response is (or will be at the check_time).
 144  * Staleness calculation is based on the following: (1) response
 145  * expiration time, (2) age greater than configured maximum, (3)
 146  * last-modified factor, and (4) age less than configured minimum.
 147  *
 148  * \retval -1  If the response is fresh.
 149  * \retval >0  Otherwise return it's staleness.
 150  * \retval 0   NOTE return value of 0 means the response is stale.
 151  *
 152  * The 'stale_flags' structure is used to tell the calling function
 153  * _why_ this response is fresh or stale.  Its used, for example,
 154  * when the admin wants to override expiration and last-modified
 155  * times.
 156  */
 157 static int
 158 refreshStaleness(const StoreEntry * entry, time_t check_time, time_t age, const refresh_t * R, stale_flags * sf)
 159 {
 160     /** \par
 161      * Check for an explicit expiration time (Expires: header).
 162      */
 163
 164     if (entry->expires > -1) {
 165         sf->expires = true;
 166
 167         if (entry->expires > check_time) {
 168             debugs(22, 3, "FRESH: expires " << entry->expires <<
 169                    " >= check_time " << check_time << " ");
 170
 171             return -1;
 172         } else {
 173             debugs(22, 3, "STALE: expires " << entry->expires <<
 174                    " < check_time " << check_time << " ");
 175
 176             return (check_time - entry->expires);
 177         }
 178     }
 179
 180     assert(age >= 0);
 181     /** \par
 182      * Use local heuristics to determine staleness.  Start with the
 183      * max age from the refresh_pattern rule.
 184      */
 185
 186     if (age > R->max) {
 187         debugs(22, 3, "STALE: age " << age << " > max " << R->max << " ");
 188         sf->max = true;
 189         return (age - R->max);
 190     }
 191
 192     /** \par
 193      * Try the last-modified factor algorithm:  refresh_pattern n% percentage of Last-Modified: age.
 194      */
 195     if (entry->lastmod > -1 && entry->timestamp > entry->lastmod) {
 196         /*
 197          * stale_age is the Age of the response when it became/becomes
 198          * stale according to the last-modified factor algorithm.
 199          */
 200         time_t stale_age = static_cast<time_t>((entry->timestamp - entry->lastmod) * R->pct);
 201         sf->lmfactor = true;
 202
 203         if (age >= stale_age) {
 204             debugs(22, 3, "STALE: age " << age << " > stale_age " << stale_age);
 205             return (age - stale_age);
 206         } else {
 207             debugs(22, 3, "FRESH: age " << age << " <= stale_age " << stale_age);
 208             return -1;
 209         }
 210     }
 211
 212     /** \par
 213      * Finally, if all else fails;  staleness is determined by the refresh_pattern
 214      * configured minimum age.
 215      */
 216     if (age < R->min) {
 217         debugs(22, 3, "FRESH: age " << age << " < min " << R->min);
 218         sf->min = true;
 219         return -1;
 220     }
 221
 222     debugs(22, 3, "STALE: age " << age << " >= min " << R->min);
 223     return (age - R->min);
 224 }
 225
 226 /**
 227  * \retval 1 if the entry must be revalidated within delta seconds
 228  * \retval 0 otherwise
 229  *
 230  *  note: request maybe null (e.g. for cache digests build)
 231  */
 232 static int
 233 refreshCheck(const StoreEntry * entry, HttpRequest * request, time_t delta)
 234 {
 235     const refresh_t *R;
 236     const char *uri = NULL;
 237     time_t age = 0;
 238     time_t check_time = squid_curtime + delta;
 239     int staleness;
 240     stale_flags sf;
 241
 242     if (entry->mem_obj)
 243         uri = entry->mem_obj->url;
 244     else if (request)
 245         uri = urlCanonical(request);
 246
 247     debugs(22, 3, "refreshCheck: '" << (uri ? uri : "<none>") << "'");
 248
 249     assert(age >= 0);
 250     if (check_time > entry->timestamp)
 251         age = check_time - entry->timestamp;
 252
 253     assert(age >= 0);
 254     R = uri ? refreshLimits(uri) : refreshUncompiledPattern(".");
 255
 256     if (NULL == R)
 257         R = &DefaultRefresh;
 258
 259     memset(&sf, '\0', sizeof(sf));
 260
 261     assert(age >= 0);
 262     staleness = refreshStaleness(entry, check_time, age, R, &sf);
 263
 264     debugs(22, 3, "Staleness = " << staleness);
 265
 266     debugs(22, 3, "refreshCheck: Matched '" << R->pattern << " " <<
 267            (int) R->min << " " << (int) (100.0 * R->pct) << "%% " <<
 268            (int) R->max << "'");
 269
 270
 271     debugs(22, 3, "refreshCheck: age = " << age);
 272
 273     debugs(22, 3, "\tcheck_time:\t" << mkrfc1123(check_time));
 274
 275     debugs(22, 3, "\tentry->timestamp:\t" << mkrfc1123(entry->timestamp));
 276
 277     if (EBIT_TEST(entry->flags, ENTRY_REVALIDATE) && staleness > -1) {
 278         debugs(22, 3, "refreshCheck: YES: Must revalidate stale response");
 279         return STALE_MUST_REVALIDATE;
 280     }
 281
 282     /* request-specific checks */
 283     if (request) {
 284         HttpHdrCc *cc = request->cache_control;
 285
 286         if (request->flags.ims && (R->flags.refresh_ims || Config.onoff.refresh_all_ims)) {
 287             /* The clients no-cache header is changed into a IMS query */
 288             debugs(22, 3, "refreshCheck: YES: refresh-ims");
 289             return STALE_FORCED_RELOAD;
 290         }
 291
 292 #if HTTP_VIOLATIONS
 293
 294         if (!request->flags.nocache_hack) {
 295             (void) 0;
 296         } else if (R->flags.ignore_reload) {
 297             /* The clients no-cache header is ignored */
 298             debugs(22, 3, "refreshCheck: MAYBE: ignore-reload");
 299         } else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) {
 300             /* The clients no-cache header is changed into a IMS query */
 301             debugs(22, 3, "refreshCheck: YES: reload-into-ims");
 302             return STALE_RELOAD_INTO_IMS;
 303         } else {
 304             /* The clients no-cache header is not overridden on this request */
 305             debugs(22, 3, "refreshCheck: YES: client reload");
 306             request->flags.nocache = 1;
 307             return STALE_FORCED_RELOAD;
 308         }
 309
 310 #endif
 311         if (NULL != cc) {
 312             if (cc->max_age > -1) {
 313 #if HTTP_VIOLATIONS
 314                 if (R->flags.ignore_reload && cc->max_age == 0) {} else
 315 #endif
 316                 {
 317 #if 0
 318
 319                     if (cc->max_age == 0) {
 320                         debugs(22, 3, "refreshCheck: YES: client-max-age = 0");
 321                         return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
 322                     }
 323
 324 #endif
 325                     if (age > cc->max_age) {
 326                         debugs(22, 3, "refreshCheck: YES: age > client-max-age");
 327                         return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
 328                     }
 329                 }
 330             }
 331
 332             if (EBIT_TEST(cc->mask, CC_MAX_STALE) && staleness > -1) {
 333                 if (cc->max_stale < 0) {
 334                     /* max-stale directive without a value */
 335                     debugs(22, 3, "refreshCheck: NO: max-stale wildcard");
 336                     return FRESH_REQUEST_MAX_STALE_ALL;
 337                 } else if (staleness < cc->max_stale) {
 338                     debugs(22, 3, "refreshCheck: NO: staleness < max-stale");
 339                     return FRESH_REQUEST_MAX_STALE_VALUE;
 340                 }
 341             }
 342         }
 343     }
 344
 345     if (-1 == staleness) {
 346         debugs(22, 3, "refreshCheck: object isn't stale..");
 347         if (sf.expires) {
 348             debugs(22, 3, "refreshCheck: returning FRESH_EXPIRES");
 349             return FRESH_EXPIRES;
 350         }
 351
 352         assert(!sf.max);
 353
 354         if (sf.lmfactor) {
 355             debugs(22, 3, "refreshCheck: returning FRESH_LMFACTOR_RULE");
 356             return FRESH_LMFACTOR_RULE;
 357         }
 358
 359         assert(sf.min);
 360
 361         debugs(22, 3, "refreshCheck: returning FRESH_MIN_RULE");
 362         return FRESH_MIN_RULE;
 363     }
 364
 365     /*
 366      * At this point the response is stale, unless one of
 367      * the override options kicks in.
 368      */
 369     if (sf.expires) {
 370 #if HTTP_VIOLATIONS
 371
 372         if (R->flags.override_expire && age < R->min) {
 373             debugs(22, 3, "refreshCheck: NO: age < min && override-expire");
 374             return FRESH_OVERRIDE_EXPIRES;
 375         }
 376
 377 #endif
 378         return STALE_EXPIRES;
 379     }
 380
 381     if (sf.max)
 382         return STALE_MAX_RULE;
 383
 384     if (sf.lmfactor) {
 385 #if HTTP_VIOLATIONS
 386
 387         if (R->flags.override_lastmod && age < R->min) {
 388             debugs(22, 3, "refreshCheck: NO: age < min && override-lastmod");
 389             return FRESH_OVERRIDE_LASTMOD;
 390         }
 391
 392 #endif
 393         return STALE_LMFACTOR_RULE;
 394     }
 395
 396     debugs(22, 3, "refreshCheck: returning STALE_DEFAULT");
 397     return STALE_DEFAULT;
 398 }
 399
 400 int
 401 refreshIsCachable(const StoreEntry * entry)
 402 {
 403     /*
 404      * Don't look at the request to avoid no-cache and other nuisances.
 405      * the object should have a mem_obj so the URL will be found there.
 406      * minimum_expiry_time seconds delta (defaults to 60 seconds), to
 407      * avoid objects which expire almost immediately, and which can't
 408      * be refreshed.
 409      */
 410     int reason = refreshCheck(entry, NULL, Config.minimum_expiry_time);
 411     refreshCounts[rcStore].total++;
 412     refreshCounts[rcStore].status[reason]++;
 413
 414     if (reason < STALE_MUST_REVALIDATE)
 415         /* Does not need refresh. This is certainly cachable */
 416         return 1;
 417
 418     if (entry->lastmod < 0)
 419         /* Last modified is needed to do a refresh */
 420         return 0;
 421
 422     if (entry->mem_obj == NULL)
 423         /* no mem_obj? */
 424         return 1;
 425
 426     if (entry->getReply() == NULL)
 427         /* no reply? */
 428         return 1;
 429
 430     if (entry->getReply()->content_length == 0)
 431         /* No use refreshing (caching?) 0 byte objects */
 432         return 0;
 433
 434     /* This seems to be refreshable. Cache it */
 435     return 1;
 436 }
 437
 438 /* refreshCheck... functions below are protocol-specific wrappers around
 439  * refreshCheck() function above */
 440
 441 int
 442 refreshCheckHTTP(const StoreEntry * entry, HttpRequest * request)
 443 {
 444     int reason = refreshCheck(entry, request, 0);
 445     refreshCounts[rcHTTP].total++;
 446     refreshCounts[rcHTTP].status[reason]++;
 447     return (reason < 200) ? 0 : 1;
 448 }
 449
 450 int
 451 refreshCheckICP(const StoreEntry * entry, HttpRequest * request)
 452 {
 453     int reason = refreshCheck(entry, request, 30);
 454     refreshCounts[rcICP].total++;
 455     refreshCounts[rcICP].status[reason]++;
 456     return (reason < 200) ? 0 : 1;
 457 }
 458
 459 #if USE_HTCP
 460 int
 461 refreshCheckHTCP(const StoreEntry * entry, HttpRequest * request)
 462 {
 463     int reason = refreshCheck(entry, request, 10);
 464     refreshCounts[rcHTCP].total++;
 465     refreshCounts[rcHTCP].status[reason]++;
 466     return (reason < 200) ? 0 : 1;
 467 }
 468
 469 #endif
 470
 471 #if USE_CACHE_DIGESTS
 472 int
 473 refreshCheckDigest(const StoreEntry * entry, time_t delta)
 474 {
 475     int reason = refreshCheck(entry,
 476                               entry->mem_obj ? entry->mem_obj->request : NULL,
 477                               delta);
 478     refreshCounts[rcCDigest].total++;
 479     refreshCounts[rcCDigest].status[reason]++;
 480     return (reason < 200) ? 0 : 1;
 481 }
 482
 483 #endif
 484
 485 time_t
 486 getMaxAge(const char *url)
 487 {
 488     const refresh_t *R;
 489     debugs(22, 3, "getMaxAge: '" << url << "'");
 490
 491     if ((R = refreshLimits(url)))
 492         return R->max;
 493     else
 494         return REFRESH_DEFAULT_MAX;
 495 }
 496
 497 static void
 498
 499 refreshCountsStats(StoreEntry * sentry, struct RefreshCounts *rc)
 500 {
 501     int sum = 0;
 502     int tot = rc->total;
 503
 504     storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc->proto);
 505     storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n");
 506
 507 #define refreshCountsStatsEntry(code,desc) { \
 508         storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", \
 509             rc->status[code], xpercent(rc->status[code], tot), desc); \
 510     sum += rc->status[code]; \
 511 }
 512
 513     refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_ALL,
 514                             "Fresh: request max-stale wildcard");
 515     refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_VALUE,
 516                             "Fresh: request max-stale value");
 517     refreshCountsStatsEntry(FRESH_EXPIRES,
 518                             "Fresh: expires time not reached");
 519     refreshCountsStatsEntry(FRESH_LMFACTOR_RULE,
 520                             "Fresh: refresh_pattern last-mod factor percentage");
 521     refreshCountsStatsEntry(FRESH_MIN_RULE,
 522                             "Fresh: refresh_pattern min value");
 523     refreshCountsStatsEntry(FRESH_OVERRIDE_EXPIRES,
 524                             "Fresh: refresh_pattern override expires");
 525     refreshCountsStatsEntry(FRESH_OVERRIDE_LASTMOD,
 526                             "Fresh: refresh_pattern override lastmod");
 527     refreshCountsStatsEntry(STALE_MUST_REVALIDATE,
 528                             "Stale: response has must-revalidate");
 529     refreshCountsStatsEntry(STALE_RELOAD_INTO_IMS,
 530                             "Stale: changed reload into IMS");
 531     refreshCountsStatsEntry(STALE_FORCED_RELOAD,
 532                             "Stale: request has no-cache directive");
 533     refreshCountsStatsEntry(STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
 534                             "Stale: age exceeds request max-age value");
 535     refreshCountsStatsEntry(STALE_EXPIRES,
 536                             "Stale: expires time reached");
 537     refreshCountsStatsEntry(STALE_MAX_RULE,
 538                             "Stale: refresh_pattern max age rule");
 539     refreshCountsStatsEntry(STALE_LMFACTOR_RULE,
 540                             "Stale: refresh_pattern last-mod factor percentage");
 541     refreshCountsStatsEntry(STALE_DEFAULT,
 542                             "Stale: by default");
 543
 544     tot = sum;                  /* paranoid: "total" line shows 100% if we forgot nothing */
 545     storeAppendPrintf(sentry, "%6d\t%6.2f\tTOTAL\n",
 546                       rc->total, xpercent(rc->total, tot));
 547     \
 548     storeAppendPrintf(sentry, "\n");
 549 }
 550
 551 static void
 552 refreshStats(StoreEntry * sentry)
 553 {
 554     int i;
 555     int total = 0;
 556
 557     /* get total usage count */
 558
 559     for (i = 0; i < rcCount; ++i)
 560         total += refreshCounts[i].total;
 561
 562     /* protocol usage histogram */
 563     storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n");
 564
 565     storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n");
 566
 567     for (i = 0; i < rcCount; ++i)
 568         storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n",
 569                           refreshCounts[i].proto,
 570                           refreshCounts[i].total,
 571                           xpercent(refreshCounts[i].total, total));
 572
 573     /* per protocol histograms */
 574     storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n");
 575
 576     for (i = 0; i < rcCount; ++i)
 577         refreshCountsStats(sentry, &refreshCounts[i]);
 578 }
 579
 580 static void
 581 refreshRegisterWithCacheManager(void)
 582 {
 583     CacheManager::GetInstance()->
 584     registerAction("refresh", "Refresh Algorithm Statistics", refreshStats, 0, 1);
 585 }
 586
 587 void
 588 refreshInit(void)
 589 {
 590     memset(refreshCounts, 0, sizeof(refreshCounts));
 591     refreshCounts[rcHTTP].proto = "HTTP";
 592     refreshCounts[rcICP].proto = "ICP";
 593 #if USE_HTCP
 594
 595     refreshCounts[rcHTCP].proto = "HTCP";
 596 #endif
 597
 598     refreshCounts[rcStore].proto = "On Store";
 599 #if USE_CACHE_DIGESTS
 600
 601     refreshCounts[rcCDigest].proto = "Cache Digests";
 602 #endif
 603
 604     memset(&DefaultRefresh, '\0', sizeof(DefaultRefresh));
 605     DefaultRefresh.pattern = "<none>";
 606     DefaultRefresh.min = REFRESH_DEFAULT_MIN;
 607     DefaultRefresh.pct = REFRESH_DEFAULT_PCT;
 608     DefaultRefresh.max = REFRESH_DEFAULT_MAX;
 609
 610     refreshRegisterWithCacheManager();
 611 }