src/refresh.cc

   1
   2 /*
   3  * $Id: refresh.cc,v 1.58 2002/10/13 20:35:03 robertc Exp $
   4  *
   5  * DEBUG: section 22    Refresh Calculation
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #ifndef USE_POSIX_REGEX
  37 #define USE_POSIX_REGEX         /* put before includes; always use POSIX */
  38 #endif
  39
  40 #include "squid.h"
  41 #include "Store.h"
  42
  43 typedef enum {
  44     rcHTTP,
  45     rcICP,
  46 #if USE_HTCP
  47     rcHTCP,
  48 #endif
  49 #if USE_CACHE_DIGESTS
  50     rcCDigest,
  51 #endif
  52     rcStore,
  53     rcCount
  54 } refreshCountsEnum;
  55
  56 typedef struct {
  57     unsigned int expires:1;
  58     unsigned int min:1;
  59     unsigned int lmfactor:1;
  60     unsigned int max;
  61 } stale_flags;
  62
  63 /*
  64  * This enumerated list assigns specific values, ala HTTP/FTP status
  65  * codes.  All Fresh codes are in the range 100-199 and all stale
  66  * codes are 200-299.  We might want to use these codes in logging,
  67  * so best to keep them consistent over time.
  68  */
  69 enum {
  70     FRESH_REQUEST_MAX_STALE_ALL = 100,
  71     FRESH_REQUEST_MAX_STALE_VALUE,
  72     FRESH_EXPIRES,
  73     FRESH_LMFACTOR_RULE,
  74     FRESH_MIN_RULE,
  75     FRESH_OVERRIDE_EXPIRES,
  76     FRESH_OVERRIDE_LASTMOD,
  77     STALE_MUST_REVALIDATE = 200,
  78     STALE_RELOAD_INTO_IMS,
  79     STALE_FORCED_RELOAD,
  80     STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
  81     STALE_EXPIRES,
  82     STALE_MAX_RULE,
  83     STALE_LMFACTOR_RULE,
  84     STALE_DEFAULT = 299
  85 };
  86
  87 static struct RefreshCounts {
  88     const char *proto;
  89     int total;
  90     int status[STALE_DEFAULT + 1];
  91 } refreshCounts[rcCount];
  92
  93 /*
  94  * Defaults:
  95  *      MIN     NONE
  96  *      PCT     20%
  97  *      MAX     3 days
  98  */
  99 #define REFRESH_DEFAULT_MIN     (time_t)0
 100 #define REFRESH_DEFAULT_PCT     0.20
 101 #define REFRESH_DEFAULT_MAX     (time_t)259200
 102
 103 static const refresh_t *refreshLimits(const char *);
 104 static const refresh_t *refreshUncompiledPattern(const char *);
 105 static OBJH refreshStats;
 106 static int refreshStaleness(const StoreEntry *, time_t, time_t, const refresh_t *, stale_flags *);
 107
 108 static refresh_t DefaultRefresh;
 109
 110 static const refresh_t *
 111 refreshLimits(const char *url)
 112 {
 113     const refresh_t *R;
 114     for (R = Config.Refresh; R; R = R->next) {
 115         if (!regexec(&(R->compiled_pattern), url, 0, 0, 0))
 116             return R;
 117     }
 118     return NULL;
 119 }
 120
 121 static const refresh_t *
 122 refreshUncompiledPattern(const char *pat)
 123 {
 124     const refresh_t *R;
 125     for (R = Config.Refresh; R; R = R->next) {
 126         if (0 == strcmp(R->pattern, pat))
 127             return R;
 128     }
 129     return NULL;
 130 }
 131
 132 /*
 133  * Calculate how stale the response is (or will be at the check_time).
 134  * Staleness calculation is based on the following: (1) response
 135  * expiration time, (2) age greater than configured maximum, (3)
 136  * last-modified factor, and (4) age less than configured minimum.
 137  *
 138  * If the response is fresh, return -1.  Otherwise return its
 139  * staleness.  NOTE return value of 0 means the response is stale.
 140  *
 141  * The 'stale_flags' structure is used to tell the calling function
 142  * _why_ this response is fresh or stale.  Its used, for example,
 143  * when the admin wants to override expiration and last-modified
 144  * times.
 145  */
 146 static int
 147 refreshStaleness(const StoreEntry * entry, time_t check_time, time_t age, const refresh_t * R, stale_flags * sf)
 148 {
 149     /*
 150      * Check for an explicit expiration time.
 151      */
 152     if (entry->expires > -1) {
 153         sf->expires = 1;
 154         if (entry->expires > check_time) {
 155             debug(22, 3) ("FRESH: expires %d >= check_time %d \n",
 156                 (int) entry->expires, (int) check_time);
 157             return -1;
 158         } else {
 159             debug(22, 3) ("STALE: expires %d < check_time %d \n",
 160                 (int) entry->expires, (int) check_time);
 161             return (check_time - entry->expires);
 162         }
 163     }
 164     assert(age >= 0);
 165     /*
 166      * Use local heuristics to determine staleness.  Start with the
 167      * max age from the refresh_pattern rule.
 168      */
 169     if (age > R->max) {
 170         debug(22, 3) ("STALE: age %d > max %d \n", (int) age, (int) R->max);
 171         sf->max = 1;
 172         return (age - R->max);
 173     }
 174     /*
 175      * Try the last-modified factor algorithm.
 176      */
 177     if (entry->lastmod > -1 && entry->timestamp > entry->lastmod) {
 178         /*
 179          * stale_age is the Age of the response when it became/becomes
 180          * stale according to the last-modified factor algorithm.
 181          */
 182         time_t stale_age = static_cast<time_t>((entry->timestamp - entry->lastmod) * R->pct);
 183         sf->lmfactor = 1;
 184         if (age >= stale_age) {
 185             debug(22, 3) ("STALE: age %d > stale_age %d\n",
 186                 (int) age, (int) stale_age);
 187             return (age - stale_age);
 188         } else {
 189             debug(22, 3) ("FRESH: age %d <= stale_age %d\n",
 190                 (int) age, (int) stale_age);
 191             return -1;
 192         }
 193     }
 194     /*
 195      * If we are here, staleness is determined by the refresh_pattern
 196      * configured minimum age.
 197      */
 198     if (age <= R->min) {
 199         debug(22, 3) ("FRESH: age %d <= min %d\n", (int) age, (int) R->min);
 200         sf->min = 1;
 201         return -1;
 202     }
 203     debug(22, 3) ("STALE: age %d > min %d\n", (int) age, (int) R->min);
 204     return (age - R->min);
 205 }
 206
 207 /*  return 1 if the entry must be revalidated within delta seconds
 208  *         0 otherwise
 209  *
 210  *  note: request maybe null (e.g. for cache digests build)
 211  */
 212 static int
 213 refreshCheck(const StoreEntry * entry, request_t * request, time_t delta)
 214 {
 215     const refresh_t *R;
 216     const char *uri = NULL;
 217     time_t age = 0;
 218     time_t check_time = squid_curtime + delta;
 219     int staleness;
 220     stale_flags sf;
 221     if (entry->mem_obj)
 222         uri = entry->mem_obj->url;
 223     else if (request)
 224         uri = urlCanonical(request);
 225
 226     debug(22, 3) ("refreshCheck: '%s'\n", uri ? uri : "<none>");
 227
 228     if (check_time > entry->timestamp)
 229         age = check_time - entry->timestamp;
 230     R = uri ? refreshLimits(uri) : refreshUncompiledPattern(".");
 231     if (NULL == R)
 232         R = &DefaultRefresh;
 233     memset(&sf, '\0', sizeof(sf));
 234     staleness = refreshStaleness(entry, check_time, age, R, &sf);
 235     debug(22, 3) ("Staleness = %d\n", staleness);
 236
 237     debug(22, 3) ("refreshCheck: Matched '%s %d %d%% %d'\n",
 238         R->pattern, (int) R->min, (int) (100.0 * R->pct), (int) R->max);
 239     debug(22, 3) ("refreshCheck: age = %d\n", (int) age);
 240     debug(22, 3) ("\tcheck_time:\t%s\n", mkrfc1123(check_time));
 241     debug(22, 3) ("\tentry->timestamp:\t%s\n", mkrfc1123(entry->timestamp));
 242
 243     if (EBIT_TEST(entry->flags, ENTRY_REVALIDATE) && staleness > -1) {
 244         debug(22, 3) ("refreshCheck: YES: Must revalidate stale response\n");
 245         return STALE_MUST_REVALIDATE;
 246     }
 247     /* request-specific checks */
 248     if (request) {
 249         HttpHdrCc *cc = request->cache_control;
 250 #if HTTP_VIOLATIONS
 251         if (!request->flags.nocache_hack) {
 252             (void) 0;
 253         } else if (R->flags.ignore_reload) {
 254             /* The clients no-cache header is ignored */
 255             debug(22, 3) ("refreshCheck: MAYBE: ignore-reload\n");
 256         } else if (R->flags.reload_into_ims || Config.onoff.reload_into_ims) {
 257             /* The clients no-cache header is changed into a IMS query */
 258             debug(22, 3) ("refreshCheck: YES: reload-into-ims\n");
 259             return STALE_RELOAD_INTO_IMS;
 260         } else {
 261             /* The clients no-cache header is not overridden on this request */
 262             debug(22, 3) ("refreshCheck: YES: client reload\n");
 263             request->flags.nocache = 1;
 264             return STALE_FORCED_RELOAD;
 265         }
 266 #endif
 267         if (NULL != cc) {
 268             if (cc->max_age > -1) {
 269 #if HTTP_VIOLATIONS
 270                 if (R->flags.ignore_reload && cc->max_age == 0) {
 271                 } else
 272 #endif
 273                 if (age > cc->max_age) {
 274                     debug(22, 3) ("refreshCheck: YES: age > client-max-age\n");
 275                     return STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE;
 276                 }
 277             }
 278             if (EBIT_TEST(cc->mask, CC_MAX_STALE) && staleness > -1) {
 279                 if (cc->max_stale < 0) {
 280                     /* max-stale directive without a value */
 281                     debug(22, 3) ("refreshCheck: NO: max-stale wildcard\n");
 282                     return FRESH_REQUEST_MAX_STALE_ALL;
 283                 } else if (staleness < cc->max_stale) {
 284                     debug(22, 3) ("refreshCheck: NO: staleness < max-stale\n");
 285                     return FRESH_REQUEST_MAX_STALE_VALUE;
 286                 }
 287             }
 288         }
 289     }
 290     if (-1 == staleness) {
 291         if (sf.expires)
 292             return FRESH_EXPIRES;
 293         assert(!sf.max);
 294         if (sf.lmfactor)
 295             return FRESH_LMFACTOR_RULE;
 296         assert(sf.min);
 297         return FRESH_MIN_RULE;
 298     }
 299     /*
 300      * At this point the response is stale, unless one of
 301      * the override options kicks in.
 302      */
 303     if (sf.expires) {
 304 #if HTTP_VIOLATIONS
 305         if (R->flags.override_expire && age < R->min) {
 306             debug(22, 3) ("refreshCheck: NO: age < min && override-expire\n");
 307             return FRESH_OVERRIDE_EXPIRES;
 308         }
 309 #endif
 310         return STALE_EXPIRES;
 311     }
 312     if (sf.max)
 313         return STALE_MAX_RULE;
 314     if (sf.lmfactor) {
 315 #if HTTP_VIOLATIONS
 316         if (R->flags.override_lastmod && age < R->min) {
 317             debug(22, 3) ("refreshCheck: NO: age < min && override-lastmod\n");
 318             return FRESH_OVERRIDE_LASTMOD;
 319         }
 320 #endif
 321         return STALE_LMFACTOR_RULE;
 322     }
 323     return STALE_DEFAULT;
 324 }
 325
 326 int
 327 refreshIsCachable(const StoreEntry * entry)
 328 {
 329     /*
 330      * Don't look at the request to avoid no-cache and other nuisances.
 331      * the object should have a mem_obj so the URL will be found there.
 332      * 60 seconds delta, to avoid objects which expire almost
 333      * immediately, and which can't be refreshed.
 334      */
 335     int reason = refreshCheck(entry, NULL, 60);
 336     refreshCounts[rcStore].total++;
 337     refreshCounts[rcStore].status[reason]++;
 338     if (reason < 200)
 339         /* Does not need refresh. This is certainly cachable */
 340         return 1;
 341     if (entry->lastmod < 0)
 342         /* Last modified is needed to do a refresh */
 343         return 0;
 344     if (entry->mem_obj == NULL)
 345         /* no mem_obj? */
 346         return 1;
 347     if (entry->mem_obj->reply == NULL)
 348         /* no reply? */
 349         return 1;
 350     if (entry->mem_obj->reply->content_length == 0)
 351         /* No use refreshing (caching?) 0 byte objects */
 352         return 0;
 353     /* This seems to be refreshable. Cache it */
 354     return 1;
 355 }
 356
 357 /* refreshCheck... functions below are protocol-specific wrappers around
 358  * refreshCheck() function above */
 359
 360 int
 361 refreshCheckHTTP(const StoreEntry * entry, request_t * request)
 362 {
 363     int reason = refreshCheck(entry, request, 0);
 364     refreshCounts[rcHTTP].total++;
 365     refreshCounts[rcHTTP].status[reason]++;
 366     return (reason < 200) ? 0 : 1;
 367 }
 368
 369 int
 370 refreshCheckICP(const StoreEntry * entry, request_t * request)
 371 {
 372     int reason = refreshCheck(entry, request, 30);
 373     refreshCounts[rcICP].total++;
 374     refreshCounts[rcICP].status[reason]++;
 375     return (reason < 200) ? 0 : 1;
 376 }
 377
 378 #if USE_HTCP
 379 int
 380 refreshCheckHTCP(const StoreEntry * entry, request_t * request)
 381 {
 382     int reason = refreshCheck(entry, request, 10);
 383     refreshCounts[rcHTCP].total++;
 384     refreshCounts[rcHTCP].status[reason]++;
 385     return (reason < 200) ? 0 : 1;
 386 }
 387 #endif
 388
 389 #if USE_CACHE_DIGESTS
 390 int
 391 refreshCheckDigest(const StoreEntry * entry, time_t delta)
 392 {
 393     int reason = refreshCheck(entry,
 394         entry->mem_obj ? entry->mem_obj->request : NULL,
 395         delta);
 396     refreshCounts[rcCDigest].total++;
 397     refreshCounts[rcCDigest].status[reason]++;
 398     return (reason < 200) ? 0 : 1;
 399 }
 400 #endif
 401
 402 time_t
 403 getMaxAge(const char *url)
 404 {
 405     const refresh_t *R;
 406     debug(22, 3) ("getMaxAge: '%s'\n", url);
 407     if ((R = refreshLimits(url)))
 408         return R->max;
 409     else
 410         return REFRESH_DEFAULT_MAX;
 411 }
 412
 413 static void
 414 refreshCountsStats(StoreEntry * sentry, struct RefreshCounts *rc)
 415 {
 416     int sum = 0;
 417     int tot = rc->total;
 418
 419     storeAppendPrintf(sentry, "\n\n%s histogram:\n", rc->proto);
 420     storeAppendPrintf(sentry, "Count\t%%Total\tCategory\n");
 421
 422 #define refreshCountsStatsEntry(code,desc) { \
 423         storeAppendPrintf(sentry, "%6d\t%6.2f\t%s\n", \
 424             rc->status[code], xpercent(rc->status[code], tot), desc); \
 425     sum += rc->status[code]; \
 426 }
 427
 428     refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_ALL,
 429         "Fresh: request max-stale wildcard");
 430     refreshCountsStatsEntry(FRESH_REQUEST_MAX_STALE_VALUE,
 431         "Fresh: request max-stale value");
 432     refreshCountsStatsEntry(FRESH_EXPIRES,
 433         "Fresh: expires time not reached");
 434     refreshCountsStatsEntry(FRESH_LMFACTOR_RULE,
 435         "Fresh: refresh_pattern last-mod factor percentage");
 436     refreshCountsStatsEntry(FRESH_MIN_RULE,
 437         "Fresh: refresh_pattern min value");
 438     refreshCountsStatsEntry(FRESH_OVERRIDE_EXPIRES,
 439         "Fresh: refresh_pattern override expires");
 440     refreshCountsStatsEntry(FRESH_OVERRIDE_LASTMOD,
 441         "Fresh: refresh_pattern override lastmod");
 442     refreshCountsStatsEntry(STALE_MUST_REVALIDATE,
 443         "Stale: response has must-revalidate");
 444     refreshCountsStatsEntry(STALE_RELOAD_INTO_IMS,
 445         "Stale: changed reload into IMS");
 446     refreshCountsStatsEntry(STALE_FORCED_RELOAD,
 447         "Stale: request has no-cache directive");
 448     refreshCountsStatsEntry(STALE_EXCEEDS_REQUEST_MAX_AGE_VALUE,
 449         "Stale: age exceeds request max-age value");
 450     refreshCountsStatsEntry(STALE_EXPIRES,
 451         "Stale: expires time reached");
 452     refreshCountsStatsEntry(STALE_MAX_RULE,
 453         "Stale: refresh_pattern max age rule");
 454     refreshCountsStatsEntry(STALE_LMFACTOR_RULE,
 455         "Stale: refresh_pattern last-mod factor percentage");
 456     refreshCountsStatsEntry(STALE_DEFAULT,
 457         "Stale: by default");
 458
 459     tot = sum;                  /* paranoid: "total" line shows 100% if we forgot nothing */
 460     storeAppendPrintf(sentry, "%6d\t%6.2f\tTOTAL\n",
 461         rc->total, xpercent(rc->total, tot));
 462     \
 463         storeAppendPrintf(sentry, "\n");
 464 }
 465
 466 static void
 467 refreshStats(StoreEntry * sentry)
 468 {
 469     int i;
 470     int total = 0;
 471
 472     /* get total usage count */
 473     for (i = 0; i < rcCount; ++i)
 474         total += refreshCounts[i].total;
 475
 476     /* protocol usage histogram */
 477     storeAppendPrintf(sentry, "\nRefreshCheck calls per protocol\n\n");
 478     storeAppendPrintf(sentry, "Protocol\t#Calls\t%%Calls\n");
 479     for (i = 0; i < rcCount; ++i)
 480         storeAppendPrintf(sentry, "%10s\t%6d\t%6.2f\n",
 481             refreshCounts[i].proto,
 482             refreshCounts[i].total,
 483             xpercent(refreshCounts[i].total, total));
 484
 485     /* per protocol histograms */
 486     storeAppendPrintf(sentry, "\n\nRefreshCheck histograms for various protocols\n");
 487     for (i = 0; i < rcCount; ++i)
 488         refreshCountsStats(sentry, &refreshCounts[i]);
 489 }
 490
 491 void
 492 refreshInit(void)
 493 {
 494     memset(refreshCounts, 0, sizeof(refreshCounts));
 495     refreshCounts[rcHTTP].proto = "HTTP";
 496     refreshCounts[rcICP].proto = "ICP";
 497 #if USE_HTCP
 498     refreshCounts[rcHTCP].proto = "HTCP";
 499 #endif
 500     refreshCounts[rcStore].proto = "On Store";
 501 #if USE_CACHE_DIGESTS
 502     refreshCounts[rcCDigest].proto = "Cache Digests";
 503 #endif
 504     cachemgrRegister("refresh",
 505         "Refresh Algorithm Statistics",
 506         refreshStats,
 507         0,
 508         1);
 509     memset(&DefaultRefresh, '\0', sizeof(DefaultRefresh));
 510     DefaultRefresh.pattern = "<none>";
 511     DefaultRefresh.min = REFRESH_DEFAULT_MIN;
 512     DefaultRefresh.pct = REFRESH_DEFAULT_PCT;
 513     DefaultRefresh.max = REFRESH_DEFAULT_MAX;
 514 }