src/store_dir.cc

   1
   2 /*
   3  * $Id: store_dir.cc,v 1.83 1998/09/23 15:37:43 wessels Exp $
   4  *
   5  * DEBUG: section 47    Store Directory Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  Duane Wessels and the University of California San Diego.  Please
  16  *  see the COPYRIGHT file for full details.  Squid incorporates
  17  *  software developed and/or copyrighted by other sources.  Please see
  18  *  the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #if HAVE_STATVFS
  38 #if HAVE_SYS_STATVFS_H
  39 #include <sys/statvfs.h>
  40 #endif
  41 #endif
  42
  43 #define SWAP_DIR_SHIFT 24
  44 #define SWAP_FILE_MASK 0x00FFFFFF
  45 #define DefaultLevelOneDirs     16
  46 #define DefaultLevelTwoDirs     256
  47
  48 static char *storeSwapSubDir(int dirn, int subdirn);
  49 static int storeDirSelectSwapDir(void);
  50 static int storeVerifyDirectory(const char *path);
  51 static int storeCreateDirectory(const char *path, int);
  52 static void storeCreateSwapSubDirs(int j);
  53
  54 /* return full name to swapfile */
  55 char *
  56 storeSwapFullPath(int fn, char *fullpath)
  57 {
  58     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  59     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
  60     int filn = fn & SWAP_FILE_MASK;
  61     int L1 = Config.cacheSwap.swapDirs[dirn].l1;
  62     int L2 = Config.cacheSwap.swapDirs[dirn].l2;
  63     if (!fullpath)
  64         fullpath = fullfilename;
  65     fullpath[0] = '\0';
  66     snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
  67         Config.cacheSwap.swapDirs[dirn].path,
  68         ((filn / L2) / L2) % L1,
  69         (filn / L2) % L2,
  70         filn);
  71     return fullpath;
  72 }
  73
  74 static char *
  75 storeSwapSubDir(int dirn, int subdirn)
  76 {
  77     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  78     SwapDir *SD;
  79     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
  80     SD = &Config.cacheSwap.swapDirs[dirn];
  81     assert(0 <= subdirn && subdirn < SD->l1);
  82     snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X",
  83         Config.cacheSwap.swapDirs[dirn].path,
  84         subdirn);
  85     return fullfilename;
  86 }
  87
  88 char *
  89 storeSwapSubSubDir(int fn, char *fullpath)
  90 {
  91     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  92     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
  93     int filn = fn & SWAP_FILE_MASK;
  94     int L1 = Config.cacheSwap.swapDirs[dirn].l1;
  95     int L2 = Config.cacheSwap.swapDirs[dirn].l2;
  96     if (!fullpath)
  97         fullpath = fullfilename;
  98     fullpath[0] = '\0';
  99     snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
 100         Config.cacheSwap.swapDirs[dirn].path,
 101         ((filn / L2) / L2) % L1,
 102         (filn / L2) % L2);
 103     return fullpath;
 104 }
 105
 106 /*
 107  * Does swapfile number 'fn' belong in cachedir #F0,
 108  * level1 dir #F1, level2 dir #F2?
 109  *
 110  * This is called by storeDirClean(), but placed here because
 111  * the algorithm needs to match storeSwapSubSubDir().
 112  *
 113  * Don't check that (fn >> SWAP_DIR_SHIFT) == F0 because
 114  * 'fn' may not have the directory bits set.
 115  */
 116 int
 117 storeFilenoBelongsHere(int fn, int F0, int F1, int F2)
 118 {
 119     int D1, D2;
 120     int L1, L2;
 121     int filn = fn & SWAP_FILE_MASK;
 122     assert(F0 < Config.cacheSwap.n_configured);
 123     L1 = Config.cacheSwap.swapDirs[F0].l1;
 124     L2 = Config.cacheSwap.swapDirs[F0].l2;
 125     D1 = ((filn / L2) / L2) % L1;
 126     if (F1 != D1)
 127         return 0;
 128     D2 = (filn / L2) % L2;
 129     if (F2 != D2)
 130         return 0;
 131     return 1;
 132 }
 133
 134 static int
 135 storeCreateDirectory(const char *path, int should_exist)
 136 {
 137     int created = 0;
 138     struct stat st;
 139     getCurrentTime();
 140     if (0 == stat(path, &st)) {
 141         if (S_ISDIR(st.st_mode)) {
 142             debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
 143         } else {
 144             fatalf("Swap directory %s is not a directory.", path);
 145         }
 146     } else if (0 == mkdir(path, 0755)) {
 147         debug(20, should_exist ? 1 : 3) ("%s created\n", path);
 148         created = 1;
 149     } else {
 150         fatalf("Failed to make swap directory %s: %s",
 151             path, xstrerror());
 152     }
 153     return created;
 154 }
 155
 156 static int
 157 storeVerifyDirectory(const char *path)
 158 {
 159     struct stat sb;
 160     if (stat(path, &sb) < 0) {
 161         debug(20, 0) ("%s: %s\n", path, xstrerror());
 162         return -1;
 163     }
 164     if (S_ISDIR(sb.st_mode) == 0) {
 165         debug(20, 0) ("%s is not a directory\n", path);
 166         return -1;
 167     }
 168     return 0;
 169 }
 170
 171 /*
 172  * This function is called by storeInit().  If this returns < 0,
 173  * then Squid exits, complains about swap directories not
 174  * existing, and instructs the admin to run 'squid -z'
 175  */
 176 int
 177 storeVerifyCacheDirs(void)
 178 {
 179     int i;
 180     int j;
 181     const char *path;
 182     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 183         path = Config.cacheSwap.swapDirs[i].path;
 184         if (storeVerifyDirectory(path) < 0)
 185             return -1;
 186         for (j = 0; j < Config.cacheSwap.swapDirs[i].l1; j++) {
 187             path = storeSwapSubDir(i, j);
 188             if (storeVerifyDirectory(path) < 0)
 189                 return -1;
 190         }
 191     }
 192     return 0;
 193 }
 194
 195 void
 196 storeCreateSwapDirectories(void)
 197 {
 198     int i;
 199     const char *path = NULL;
 200     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 201         path = Config.cacheSwap.swapDirs[i].path;
 202         debug(47, 3) ("Creating swap space in %s\n", path);
 203         storeCreateDirectory(path, 0);
 204         storeCreateSwapSubDirs(i);
 205     }
 206 }
 207
 208 static void
 209 storeCreateSwapSubDirs(int j)
 210 {
 211     int i, k;
 212     int should_exist;
 213     SwapDir *SD = &Config.cacheSwap.swapDirs[j];
 214     LOCAL_ARRAY(char, name, MAXPATHLEN);
 215     for (i = 0; i < SD->l1; i++) {
 216         snprintf(name, MAXPATHLEN, "%s/%02X", SD->path, i);
 217         if (storeCreateDirectory(name, 0))
 218             should_exist = 0;
 219         else
 220             should_exist = 1;
 221         debug(47, 1) ("Making directories in %s\n", name);
 222         for (k = 0; k < SD->l2; k++) {
 223             snprintf(name, MAXPATHLEN, "%s/%02X/%02X", SD->path, i, k);
 224             storeCreateDirectory(name, should_exist);
 225         }
 226     }
 227 }
 228
 229 /*
 230  *Spread load across least 3/4 of the store directories
 231  */
 232 static int
 233 storeDirSelectSwapDir(void)
 234 {
 235     double least_used = 1.0;
 236     double high = (double) Config.Swap.highWaterMark / 100.0;
 237     double u;
 238     int dirn;
 239     int i, j;
 240     SwapDir *SD;
 241     static int nleast = 0;
 242     static int nconf = 0;
 243     static int *dirq = NULL;
 244     static double *diru = NULL;
 245     /*
 246      * Handle simplest case of a single swap directory immediately
 247      */
 248     if (Config.cacheSwap.n_configured == 1)
 249         return 0;
 250     /*
 251      * Initialise dirq on the first call or on change of number of dirs
 252      */
 253     if (nconf != Config.cacheSwap.n_configured) {
 254         nconf = Config.cacheSwap.n_configured;
 255         nleast = (nconf * 3) / 4;
 256         safe_free(dirq);
 257         dirq = (int *) xmalloc(sizeof(int) * nleast);
 258         safe_free(diru);
 259         diru = (double *) xmalloc(sizeof(double) * nconf);
 260         for (j = 0; j < nleast; j++)
 261             dirq[j] = -1;
 262     }
 263     /*
 264      * Scan for a non-negative dirn in the dirq array and return that one
 265      */
 266     dirn = -1;
 267     for (j = 0; j < nleast; j++) {
 268         dirn = dirq[j];
 269         if (dirn < 0)
 270             continue;
 271         dirq[j] = -1;
 272         break;
 273     }
 274     /*
 275      * If we found a valid dirn return it
 276      */
 277     if (dirn >= 0)
 278         return dirn;
 279     /*
 280      * Now for the real guts of the algorithm - building the dirq array
 281      */
 282     for (i = 0; i < nconf; i++) {
 283         diru[i] = 1.1;
 284         SD = &Config.cacheSwap.swapDirs[i];
 285         SD->flags.selected = 0;
 286         if (SD->read_only)
 287             continue;
 288         u = (double) SD->cur_size / SD->max_size;
 289         if (u > high)
 290             continue;
 291         diru[i] = u;
 292     }
 293     for (j = 0; j < nleast; j++) {
 294         dirq[j] = -1;
 295         least_used = 1.0;
 296         dirn = -1;
 297         for (i = 0; i < nconf; i++) {
 298             if (diru[i] < least_used) {
 299                 least_used = diru[i];
 300                 dirn = i;
 301             }
 302         }
 303         if (dirn < 0)
 304             break;
 305         dirq[j] = dirn;
 306         diru[dirn] = 1.1;
 307         /* set selected flag for debugging/cachemgr only */
 308         Config.cacheSwap.swapDirs[dirn].flags.selected = 1;
 309     }
 310     /*
 311      * Setup default return of 0 if no least found
 312      */
 313     if (dirq[0] < 0)
 314         dirq[0] = 0;
 315     dirn = dirq[0];
 316     dirq[0] = -1;
 317     return dirn;
 318 }
 319
 320 int
 321 storeDirValidFileno(int fn)
 322 {
 323     int dirn = fn >> SWAP_DIR_SHIFT;
 324     int filn = fn & SWAP_FILE_MASK;
 325     if (dirn > Config.cacheSwap.n_configured)
 326         return 0;
 327     if (dirn < 0)
 328         return 0;
 329     if (filn < 0)
 330         return 0;
 331     if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
 332         return 0;
 333     return 1;
 334 }
 335
 336 int
 337 storeDirMapBitTest(int fn)
 338 {
 339     int dirn = fn >> SWAP_DIR_SHIFT;
 340     int filn = fn & SWAP_FILE_MASK;
 341     return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn);
 342 }
 343
 344 void
 345 storeDirMapBitSet(int fn)
 346 {
 347     int dirn = fn >> SWAP_DIR_SHIFT;
 348     int filn = fn & SWAP_FILE_MASK;
 349     file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn);
 350 }
 351
 352 void
 353 storeDirMapBitReset(int fn)
 354 {
 355     int dirn = fn >> SWAP_DIR_SHIFT;
 356     int filn = fn & SWAP_FILE_MASK;
 357     file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn);
 358 }
 359
 360 int
 361 storeDirMapAllocate(void)
 362 {
 363     int dirn = storeDirSelectSwapDir();
 364     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 365     int filn = file_map_allocate(SD->map, SD->suggest);
 366     SD->suggest = filn + 1;
 367     return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK);
 368 }
 369
 370 char *
 371 storeSwapDir(int dirn)
 372 {
 373     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
 374     return Config.cacheSwap.swapDirs[dirn].path;
 375 }
 376
 377 int
 378 storeDirNumber(int swap_file_number)
 379 {
 380     return swap_file_number >> SWAP_DIR_SHIFT;
 381 }
 382
 383 int
 384 storeDirProperFileno(int dirn, int fn)
 385 {
 386     return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK);
 387 }
 388
 389 /*
 390  * An entry written to the swap log MUST have the following
 391  * properties.
 392  *   1.  It MUST be a public key.  It does no good to log
 393  *       a public ADD, change the key, then log a private
 394  *       DEL.  So we need to log a DEL before we change a
 395  *       key from public to private.
 396  *   2.  It MUST have a valid (> -1) swap_file_number.
 397  */
 398 void
 399 storeDirSwapLog(const StoreEntry * e, int op)
 400 {
 401     storeSwapLogData *s;
 402     int dirn;
 403     dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
 404     assert(dirn < Config.cacheSwap.n_configured);
 405     assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
 406     assert(e->swap_file_number >= 0);
 407     /*
 408      * icons and such; don't write them to the swap log
 409      */
 410     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 411         return;
 412     assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
 413     debug(20, 3) ("storeDirSwapLog: %s %s %08X\n",
 414         swap_log_op_str[op],
 415         storeKeyText(e->key),
 416         e->swap_file_number);
 417     s = xcalloc(1, sizeof(storeSwapLogData));
 418     s->op = (char) op;
 419     s->swap_file_number = e->swap_file_number;
 420     s->timestamp = e->timestamp;
 421     s->lastref = e->lastref;
 422     s->expires = e->expires;
 423     s->lastmod = e->lastmod;
 424     s->swap_file_sz = e->swap_file_sz;
 425     s->refcount = e->refcount;
 426     s->flags = e->flags;
 427     xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
 428     file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd,
 429         -1,
 430         s,
 431         sizeof(storeSwapLogData),
 432         NULL,
 433         NULL,
 434         xfree);
 435 }
 436
 437 char *
 438 storeDirSwapLogFile(int dirn, const char *ext)
 439 {
 440     LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
 441     LOCAL_ARRAY(char, digit, 32);
 442     if (Config.Log.swap) {
 443         xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64);
 444         strcat(path, ".");
 445         snprintf(digit, 32, "%02d", dirn);
 446         strncat(path, digit, 3);
 447     } else {
 448         xstrncpy(path, storeSwapDir(dirn), SQUID_MAXPATHLEN - 64);
 449         strcat(path, "/swap.state");
 450     }
 451     if (ext)
 452         strncat(path, ext, 16);
 453     return path;
 454 }
 455
 456 void
 457 storeDirOpenSwapLogs(void)
 458 {
 459     int i;
 460     char *path;
 461     int fd;
 462     SwapDir *SD;
 463     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 464         SD = &Config.cacheSwap.swapDirs[i];
 465         path = storeDirSwapLogFile(i, NULL);
 466         fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
 467         if (fd < 0) {
 468             debug(50, 1) ("%s: %s\n", path, xstrerror());
 469             fatal("storeDirOpenSwapLogs: Failed to open swap log.");
 470         }
 471         debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", i, fd);
 472         SD->swaplog_fd = fd;
 473     }
 474 }
 475
 476 void
 477 storeDirCloseSwapLogs(void)
 478 {
 479     int i;
 480     SwapDir *SD;
 481     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 482         SD = &Config.cacheSwap.swapDirs[i];
 483         if (SD->swaplog_fd < 0) /* not open */
 484             continue;
 485         file_close(SD->swaplog_fd);
 486         debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", i, SD->swaplog_fd);
 487         SD->swaplog_fd = -1;
 488     }
 489 }
 490
 491 FILE *
 492 storeDirOpenTmpSwapLog(int dirn, int *clean_flag, int *zero_flag)
 493 {
 494     char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
 495     char *clean_path = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
 496     char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
 497     struct stat log_sb;
 498     struct stat clean_sb;
 499     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 500     FILE *fp;
 501     int fd;
 502     if (stat(swaplog_path, &log_sb) < 0) {
 503         debug(47, 1) ("Cache Dir #%d: No log file\n", dirn);
 504         safe_free(swaplog_path);
 505         safe_free(clean_path);
 506         safe_free(new_path);
 507         return NULL;
 508     }
 509     *zero_flag = log_sb.st_size == 0 ? 1 : 0;
 510     /* close the existing write-only FD */
 511     if (SD->swaplog_fd >= 0)
 512         file_close(SD->swaplog_fd);
 513     /* open a write-only FD for the new log */
 514     fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL);
 515     if (fd < 0) {
 516         debug(50, 1) ("%s: %s\n", new_path, xstrerror());
 517         fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
 518     }
 519     SD->swaplog_fd = fd;
 520     /* open a read-only stream of the old log */
 521     fp = fopen(swaplog_path, "r");
 522     if (fp == NULL) {
 523         debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
 524         fatal("Failed to open swap log for reading");
 525     }
 526     memset(&clean_sb, '\0', sizeof(struct stat));
 527     if (stat(clean_path, &clean_sb) < 0)
 528         *clean_flag = 0;
 529     else if (clean_sb.st_mtime < log_sb.st_mtime)
 530         *clean_flag = 0;
 531     else
 532         *clean_flag = 1;
 533     safeunlink(clean_path, 1);
 534     safe_free(swaplog_path);
 535     safe_free(clean_path);
 536     safe_free(new_path);
 537     return fp;
 538 }
 539
 540 void
 541 storeDirCloseTmpSwapLog(int dirn)
 542 {
 543     char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
 544     char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
 545     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 546     int fd;
 547     file_close(SD->swaplog_fd);
 548     if (rename(new_path, swaplog_path) < 0) {
 549         debug(50, 0) ("%s,%s: %s\n", new_path, swaplog_path, xstrerror());
 550         fatal("storeDirCloseTmpSwapLog: rename failed");
 551     }
 552     fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
 553     if (fd < 0) {
 554         debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
 555         fatal("storeDirCloseTmpSwapLog: Failed to open swap log.");
 556     }
 557     safe_free(swaplog_path);
 558     safe_free(new_path);
 559     SD->swaplog_fd = fd;
 560     debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", dirn, fd);
 561 }
 562
 563 void
 564 storeDirUpdateSwapSize(int fn, size_t size, int sign)
 565 {
 566     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
 567     int k = ((size + 1023) >> 10) * sign;
 568     Config.cacheSwap.swapDirs[dirn].cur_size += k;
 569     store_swap_size += k;
 570     if (sign > 0)
 571         n_disk_objects++;
 572     else if (sign < 0)
 573         n_disk_objects--;
 574 }
 575
 576 void
 577 storeDirStats(StoreEntry * sentry)
 578 {
 579     int i;
 580     SwapDir *SD;
 581 #if HAVE_STATVFS
 582     struct statvfs sfs;
 583 #endif
 584     storeAppendPrintf(sentry, "Store Directory Statistics:\n");
 585     storeAppendPrintf(sentry, "Store Entries          : %d\n",
 586         memInUse(MEM_STOREENTRY));
 587     storeAppendPrintf(sentry, "Maximum Swap Size      : %8d KB\n",
 588         Config.Swap.maxSize);
 589     storeAppendPrintf(sentry, "Current Store Swap Size: %8d KB\n",
 590         store_swap_size);
 591     storeAppendPrintf(sentry, "Current Capacity       : %d%% used, %d%% free\n",
 592         percent((int) store_swap_size, (int) Config.Swap.maxSize),
 593         percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize));
 594     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 595         SD = &Config.cacheSwap.swapDirs[i];
 596         storeAppendPrintf(sentry, "\n");
 597         storeAppendPrintf(sentry, "Store Directory #%d: %s\n", i, SD->path);
 598         storeAppendPrintf(sentry, "First level subdirectories: %d\n", SD->l1);
 599         storeAppendPrintf(sentry, "Second level subdirectories: %d\n", SD->l2);
 600         storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
 601         storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
 602         storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
 603             100.0 * SD->cur_size / SD->max_size);
 604         storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
 605             SD->map->n_files_in_map, SD->map->max_n_files,
 606             percent(SD->map->n_files_in_map, SD->map->max_n_files));
 607 #if HAVE_STATVFS
 608 #define fsbtoblk(num, fsbs, bs) \
 609         (((fsbs) != 0 && (fsbs) < (bs)) ? \
 610                 (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
 611         if (!statvfs(SD->path, &sfs)) {
 612             storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
 613                 fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024),
 614                 fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024),
 615                 percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
 616             storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
 617                 sfs.f_files - sfs.f_ffree, sfs.f_files,
 618                 percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
 619         }
 620 #endif
 621         storeAppendPrintf(sentry, "Flags:");
 622         if (SD->flags.selected)
 623             storeAppendPrintf(sentry, " SELECTED");
 624         storeAppendPrintf(sentry, "\n");
 625     }
 626 }
 627
 628 int
 629 storeDirMapBitsInUse(void)
 630 {
 631     int i;
 632     int n = 0;
 633     for (i = 0; i < Config.cacheSwap.n_configured; i++)
 634         n += Config.cacheSwap.swapDirs[i].map->n_files_in_map;
 635     return n;
 636 }
 637
 638 /*
 639  *  storeDirWriteCleanLogs
 640  *
 641  *  Writes a "clean" swap log file from in-memory metadata.
 642  */
 643 #define CLEAN_BUF_SZ 16384
 644 int
 645 storeDirWriteCleanLogs(int reopen)
 646 {
 647     StoreEntry *e = NULL;
 648     int *fd;
 649     int n = 0;
 650     time_t start, stop, r;
 651     struct stat sb;
 652     char **cur;
 653     char **new;
 654     char **cln;
 655     int dirn;
 656     int N = Config.cacheSwap.n_configured;
 657     dlink_node *m;
 658     char **outbuf;
 659     off_t *outbufoffset;
 660     storeSwapLogData s;
 661     size_t ss = sizeof(storeSwapLogData);
 662     if (store_rebuilding) {
 663         debug(20, 1) ("Not currently OK to rewrite swap log.\n");
 664         debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n");
 665         return 0;
 666     }
 667     debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n");
 668     start = squid_curtime;
 669     fd = xcalloc(N, sizeof(int));
 670     cur = xcalloc(N, sizeof(char *));
 671     new = xcalloc(N, sizeof(char *));
 672     cln = xcalloc(N, sizeof(char *));
 673     for (dirn = 0; dirn < N; dirn++) {
 674         fd[dirn] = -1;
 675         cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL));
 676         new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean"));
 677         cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
 678         unlink(new[dirn]);
 679         unlink(cln[dirn]);
 680         fd[dirn] = file_open(new[dirn],
 681             O_WRONLY | O_CREAT | O_TRUNC,
 682             NULL,
 683             NULL,
 684             NULL);
 685         if (fd[dirn] < 0) {
 686             debug(50, 0) ("storeDirWriteCleanLogs: %s: %s\n", new[dirn], xstrerror());
 687             continue;
 688         }
 689         debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
 690             new[dirn], fd[dirn]);
 691 #if HAVE_FCHMOD
 692         if (stat(cur[dirn], &sb) == 0)
 693             fchmod(fd[dirn], sb.st_mode);
 694 #endif
 695     }
 696     outbuf = xcalloc(N, sizeof(char *));
 697     outbufoffset = xcalloc(N, sizeof(*outbufoffset));
 698     for (dirn = 0; dirn < N; dirn++) {
 699         outbuf[dirn] = xcalloc(CLEAN_BUF_SZ, 1);
 700         outbufoffset[dirn] = 0;
 701     }
 702     for (m = store_list.tail; m; m = m->prev) {
 703         e = m->data;
 704         if (e->swap_file_number < 0)
 705             continue;
 706         if (e->swap_status != SWAPOUT_DONE)
 707             continue;
 708         if (e->swap_file_sz <= 0)
 709             continue;
 710         if (EBIT_TEST(e->flags, RELEASE_REQUEST))
 711             continue;
 712         if (EBIT_TEST(e->flags, KEY_PRIVATE))
 713             continue;
 714         if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 715             continue;
 716         dirn = storeDirNumber(e->swap_file_number);
 717         assert(dirn < N);
 718         if (fd[dirn] < 0)
 719             continue;
 720         memset(&s, '\0', ss);
 721         s.op = (char) SWAP_LOG_ADD;
 722         s.swap_file_number = e->swap_file_number;
 723         s.timestamp = e->timestamp;
 724         s.lastref = e->lastref;
 725         s.expires = e->expires;
 726         s.lastmod = e->lastmod;
 727         s.swap_file_sz = e->swap_file_sz;
 728         s.refcount = e->refcount;
 729         s.flags = e->flags;
 730         xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
 731         xmemcpy(outbuf[dirn] + outbufoffset[dirn], &s, ss);
 732         outbufoffset[dirn] += ss;
 733         /* buffered write */
 734         if (outbufoffset[dirn] + ss > CLEAN_BUF_SZ) {
 735             if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
 736                 debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
 737                     new[dirn], xstrerror());
 738                 debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
 739                 file_close(fd[dirn]);
 740                 fd[dirn] = -1;
 741                 unlink(new[dirn]);
 742                 continue;
 743             }
 744             outbufoffset[dirn] = 0;
 745         }
 746         if ((++n & 0xFFFF) == 0) {
 747             getCurrentTime();
 748             debug(20, 1) ("  %7d entries written so far.\n", n);
 749         }
 750     }
 751     /* flush */
 752     for (dirn = 0; dirn < N; dirn++) {
 753         if (outbufoffset[dirn] == 0)
 754             continue;
 755         if (fd[dirn] < 0)
 756             continue;
 757         if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
 758             debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
 759                 new[dirn], xstrerror());
 760             debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
 761             file_close(fd[dirn]);
 762             fd[dirn] = -1;
 763             unlink(new[dirn]);
 764             continue;
 765         }
 766         safe_free(outbuf[dirn]);
 767     }
 768     safe_free(outbuf);
 769     safe_free(outbufoffset);
 770 #ifdef _SQUID_MSWIN_
 771     /*
 772      * You can't rename open files on Microsoft "operating systems"
 773      * so we close before renaming.
 774      */
 775     storeDirCloseSwapLogs();
 776 #endif
 777     /* rename */
 778     for (dirn = 0; dirn < N; dirn++) {
 779         if (fd[dirn] < 0)
 780             continue;
 781         if (rename(new[dirn], cur[dirn]) < 0) {
 782             debug(50, 0) ("storeDirWriteCleanLogs: rename failed: %s, %s -> %s\n",
 783                 xstrerror(), new[dirn], cur[dirn]);
 784         }
 785     }
 786 #ifndef _SQUID_MSWIN_
 787     storeDirCloseSwapLogs();
 788 #endif
 789     if (reopen)
 790         storeDirOpenSwapLogs();
 791     stop = squid_curtime;
 792     r = stop - start;
 793     debug(20, 1) ("  Finished.  Wrote %d entries.\n", n);
 794     debug(20, 1) ("  Took %d seconds (%6.1f entries/sec).\n",
 795         r > 0 ? (int) r : 0,
 796         (double) n / (r > 0 ? r : 1));
 797     /* touch a timestamp file if we're not still validating */
 798     if (!store_rebuilding) {
 799         for (dirn = 0; dirn < N; dirn++) {
 800             if (fd[dirn] < 0)
 801                 continue;
 802             file_close(file_open(cln[dirn],
 803                     O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL));
 804         }
 805     }
 806     /* close */
 807     for (dirn = 0; dirn < N; dirn++) {
 808         safe_free(cur[dirn]);
 809         safe_free(new[dirn]);
 810         safe_free(cln[dirn]);
 811         if (fd[dirn] < 0)
 812             continue;
 813         file_close(fd[dirn]);
 814         fd[dirn] = -1;
 815     }
 816     safe_free(cur);
 817     safe_free(new);
 818     safe_free(cln);
 819     safe_free(fd);
 820     return n;
 821 }
 822 #undef CLEAN_BUF_SZ
 823
 824 void
 825 storeDirConfigure(void)
 826 {
 827     SwapDir *SD;
 828     int n;
 829     int i;
 830     fileMap *fm;
 831     Config.Swap.maxSize = 0;
 832     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 833         SD = &Config.cacheSwap.swapDirs[i];;
 834         Config.Swap.maxSize += SD->max_size;
 835         n = 2 * SD->max_size / Config.Store.avgObjectSize;
 836         if (NULL == SD->map) {
 837             /* first time */
 838             SD->map = file_map_create(n);
 839         } else if (n > SD->map->max_n_files) {
 840             /* it grew, need to expand */
 841             fm = file_map_create(n);
 842             filemapCopy(SD->map, fm);
 843             filemapFreeMemory(SD->map);
 844             SD->map = fm;
 845         }
 846         /* else it shrunk, and we leave the old one in place */
 847     }
 848 }
 849
 850 void
 851 storeDirDiskFull(int fn)
 852 {
 853     int dirn = fn >> SWAP_DIR_SHIFT;
 854     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 855     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
 856     SD->max_size = SD->cur_size;
 857     debug(20, 1) ("WARNING: Shrinking cache_dir #%d to %d KB\n",
 858         dirn, SD->cur_size);
 859 }