src/store_dir.cc

   1
   2 /*
   3  * $Id: store_dir.cc,v 1.81 1998/09/22 18:55:01 wessels Exp $
   4  *
   5  * DEBUG: section 47    Store Directory Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  Duane Wessels and the University of California San Diego.  Please
  16  *  see the COPYRIGHT file for full details.  Squid incorporates
  17  *  software developed and/or copyrighted by other sources.  Please see
  18  *  the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #if HAVE_STATVFS
  38 #if HAVE_SYS_STATVFS_H
  39 #include <sys/statvfs.h>
  40 #endif
  41 #endif
  42
  43 #define SWAP_DIR_SHIFT 24
  44 #define SWAP_FILE_MASK 0x00FFFFFF
  45 #define DefaultLevelOneDirs     16
  46 #define DefaultLevelTwoDirs     256
  47
  48 static char *storeSwapSubDir(int dirn, int subdirn);
  49 static int storeDirSelectSwapDir(void);
  50 static int storeVerifyDirectory(const char *path);
  51 static int storeCreateDirectory(const char *path, int);
  52 static void storeCreateSwapSubDirs(int j);
  53
  54 /* return full name to swapfile */
  55 char *
  56 storeSwapFullPath(int fn, char *fullpath)
  57 {
  58     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  59     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
  60     int filn = fn & SWAP_FILE_MASK;
  61     int L1 = Config.cacheSwap.swapDirs[dirn].l1;
  62     int L2 = Config.cacheSwap.swapDirs[dirn].l2;
  63     if (!fullpath)
  64         fullpath = fullfilename;
  65     fullpath[0] = '\0';
  66     snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
  67         Config.cacheSwap.swapDirs[dirn].path,
  68         ((filn / L2) / L2) % L1,
  69         (filn / L2) % L2,
  70         filn);
  71     return fullpath;
  72 }
  73
  74 static char *
  75 storeSwapSubDir(int dirn, int subdirn)
  76 {
  77     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  78     SwapDir *SD;
  79     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
  80     SD = &Config.cacheSwap.swapDirs[dirn];
  81     assert(0 <= subdirn && subdirn < SD->l1);
  82     snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X",
  83         Config.cacheSwap.swapDirs[dirn].path,
  84         subdirn);
  85     return fullfilename;
  86 }
  87
  88 char *
  89 storeSwapSubSubDir(int fn, char *fullpath)
  90 {
  91     LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
  92     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
  93     int filn = fn & SWAP_FILE_MASK;
  94     int L1 = Config.cacheSwap.swapDirs[dirn].l1;
  95     int L2 = Config.cacheSwap.swapDirs[dirn].l2;
  96     if (!fullpath)
  97         fullpath = fullfilename;
  98     fullpath[0] = '\0';
  99     snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
 100         Config.cacheSwap.swapDirs[dirn].path,
 101         ((filn / L2) / L2) % L1,
 102         (filn / L2) % L2);
 103     return fullpath;
 104 }
 105
 106 /*
 107  * Does swapfile number 'fn' belong in cachedir #F0,
 108  * level1 dir #F1, level2 dir #F2?
 109  *
 110  * This is called by storeDirClean(), but placed here because
 111  * the algorithm needs to match storeSwapSubSubDir().
 112  *
 113  * Don't check that (fn >> SWAP_DIR_SHIFT) == F0 because
 114  * 'fn' may not have the directory bits set.
 115  */
 116 int
 117 storeFilenoBelongsHere(int fn, int F0, int F1, int F2)
 118 {
 119     int D1, D2;
 120     int L1, L2;
 121     int filn = fn & SWAP_FILE_MASK;
 122     assert(F0 < Config.cacheSwap.n_configured);
 123     L1 = Config.cacheSwap.swapDirs[F0].l1;
 124     L2 = Config.cacheSwap.swapDirs[F0].l2;
 125     D1 = ((filn / L2) / L2) % L1;
 126     if (F1 != D1)
 127         return 0;
 128     D2 = (filn / L2) % L2;
 129     if (F2 != D2)
 130         return 0;
 131     return 1;
 132 }
 133
 134 static int
 135 storeCreateDirectory(const char *path, int should_exist)
 136 {
 137     int created = 0;
 138     struct stat st;
 139     getCurrentTime();
 140     if (0 == stat(path, &st)) {
 141         if (S_ISDIR(st.st_mode)) {
 142             debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
 143         } else {
 144             fatalf("Swap directory %s is not a directory.", path);
 145         }
 146     } else if (0 == mkdir(path, 0755)) {
 147         debug(20, should_exist ? 1 : 3) ("%s created\n", path);
 148         created = 1;
 149     } else {
 150         fatalf("Failed to make swap directory %s: %s",
 151             path, xstrerror());
 152     }
 153     return created;
 154 }
 155
 156 static int
 157 storeVerifyDirectory(const char *path)
 158 {
 159     struct stat sb;
 160     if (stat(path, &sb) < 0) {
 161         debug(20, 0) ("%s: %s\n", path, xstrerror());
 162         return -1;
 163     }
 164     if (S_ISDIR(sb.st_mode) == 0) {
 165         debug(20, 0) ("%s is not a directory\n", path);
 166         return -1;
 167     }
 168     return 0;
 169 }
 170
 171 /*
 172  * This function is called by storeInit().  If this returns < 0,
 173  * then Squid exits, complains about swap directories not
 174  * existing, and instructs the admin to run 'squid -z'
 175  */
 176 int
 177 storeVerifyCacheDirs(void)
 178 {
 179     int i;
 180     int j;
 181     const char *path;
 182     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 183         path = Config.cacheSwap.swapDirs[i].path;
 184         if (storeVerifyDirectory(path) < 0)
 185             return -1;
 186         for (j = 0; j < Config.cacheSwap.swapDirs[i].l1; j++) {
 187             path = storeSwapSubDir(i, j);
 188             if (storeVerifyDirectory(path) < 0)
 189                 return -1;
 190         }
 191     }
 192     return 0;
 193 }
 194
 195 void
 196 storeCreateSwapDirectories(void)
 197 {
 198     int i;
 199     const char *path = NULL;
 200     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 201         path = Config.cacheSwap.swapDirs[i].path;
 202         debug(47, 3) ("Creating swap space in %s\n", path);
 203         storeCreateDirectory(path, 0);
 204         storeCreateSwapSubDirs(i);
 205     }
 206 }
 207
 208 static void
 209 storeCreateSwapSubDirs(int j)
 210 {
 211     int i, k;
 212     int should_exist;
 213     SwapDir *SD = &Config.cacheSwap.swapDirs[j];
 214     LOCAL_ARRAY(char, name, MAXPATHLEN);
 215     for (i = 0; i < SD->l1; i++) {
 216         snprintf(name, MAXPATHLEN, "%s/%02X", SD->path, i);
 217         if (storeCreateDirectory(name, 0))
 218             should_exist = 0;
 219         else
 220             should_exist = 1;
 221         debug(47, 1) ("Making directories in %s\n", name);
 222         for (k = 0; k < SD->l2; k++) {
 223             snprintf(name, MAXPATHLEN, "%s/%02X/%02X", SD->path, i, k);
 224             storeCreateDirectory(name, should_exist);
 225         }
 226     }
 227 }
 228
 229 /*
 230  *Spread load across least 3/4 of the store directories
 231  */
 232 static int
 233 storeDirSelectSwapDir(void)
 234 {
 235     double least_used = 1.0;
 236     int dirn;
 237     int i, j;
 238     SwapDir *SD;
 239     static int nleast = 0;
 240     static int nconf = 0;
 241     static int *dirq = NULL;
 242     static double *diru = NULL;
 243     /*
 244      * Handle simplest case of a single swap directory immediately
 245      */
 246     if (Config.cacheSwap.n_configured == 1)
 247         return 0;
 248     /*
 249      * Initialise dirq on the first call or on change of number of dirs
 250      */
 251     if (nconf != Config.cacheSwap.n_configured) {
 252         nconf = Config.cacheSwap.n_configured;
 253         nleast = (nconf * 3) / 4;
 254         safe_free(dirq);
 255         dirq = (int *) xmalloc(sizeof(int) * nleast);
 256         safe_free(diru);
 257         diru = (double *) xmalloc(sizeof(double) * nconf);
 258         for (j = 0; j < nleast; j++)
 259             dirq[j] = -1;
 260     }
 261     /*
 262      * Scan for a non-negative dirn in the dirq array and return that one
 263      */
 264     dirn = -1;
 265     for (j = 0; j < nleast; j++) {
 266         dirn = dirq[j];
 267         if (dirn < 0)
 268             continue;
 269         dirq[j] = -1;
 270         break;
 271     }
 272     /*
 273      * If we found a valid dirn return it
 274      */
 275     if (dirn >= 0)
 276         return dirn;
 277     /*
 278      * Now for the real guts of the algorithm - building the dirq array
 279      */
 280     for (i = 0; i < nconf; i++) {
 281         diru[i] = 1.1;
 282         SD = &Config.cacheSwap.swapDirs[i];
 283         if (SD->read_only)
 284             continue;
 285         diru[i] = (double) SD->cur_size;
 286         diru[i] /= SD->max_size;
 287     }
 288     for (j = 0; j < nleast; j++) {
 289         dirq[j] = -1;
 290         least_used = 1.0;
 291         dirn = -1;
 292         for (i = 0; i < nconf; i++) {
 293             if (diru[i] < least_used) {
 294                 least_used = diru[i];
 295                 dirn = i;
 296             }
 297         }
 298         if (dirn < 0)
 299             break;
 300         dirq[j] = dirn;
 301         diru[dirn] = 1.1;
 302     }
 303     /*
 304      * Setup default return of 0 if no least found
 305      */
 306     if (dirq[0] < 0)
 307         dirq[0] = 0;
 308     dirn = dirq[0];
 309     dirq[0] = -1;
 310     return dirn;
 311 }
 312
 313 int
 314 storeDirValidFileno(int fn)
 315 {
 316     int dirn = fn >> SWAP_DIR_SHIFT;
 317     int filn = fn & SWAP_FILE_MASK;
 318     if (dirn > Config.cacheSwap.n_configured)
 319         return 0;
 320     if (dirn < 0)
 321         return 0;
 322     if (filn < 0)
 323         return 0;
 324     if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
 325         return 0;
 326     return 1;
 327 }
 328
 329 int
 330 storeDirMapBitTest(int fn)
 331 {
 332     int dirn = fn >> SWAP_DIR_SHIFT;
 333     int filn = fn & SWAP_FILE_MASK;
 334     return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn);
 335 }
 336
 337 void
 338 storeDirMapBitSet(int fn)
 339 {
 340     int dirn = fn >> SWAP_DIR_SHIFT;
 341     int filn = fn & SWAP_FILE_MASK;
 342     file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn);
 343 }
 344
 345 void
 346 storeDirMapBitReset(int fn)
 347 {
 348     int dirn = fn >> SWAP_DIR_SHIFT;
 349     int filn = fn & SWAP_FILE_MASK;
 350     file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn);
 351 }
 352
 353 int
 354 storeDirMapAllocate(void)
 355 {
 356     int dirn = storeDirSelectSwapDir();
 357     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 358     int filn = file_map_allocate(SD->map, SD->suggest);
 359     SD->suggest = filn + 1;
 360     return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK);
 361 }
 362
 363 char *
 364 storeSwapDir(int dirn)
 365 {
 366     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
 367     return Config.cacheSwap.swapDirs[dirn].path;
 368 }
 369
 370 int
 371 storeDirNumber(int swap_file_number)
 372 {
 373     return swap_file_number >> SWAP_DIR_SHIFT;
 374 }
 375
 376 int
 377 storeDirProperFileno(int dirn, int fn)
 378 {
 379     return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK);
 380 }
 381
 382 /*
 383  * An entry written to the swap log MUST have the following
 384  * properties.
 385  *   1.  It MUST be a public key.  It does no good to log
 386  *       a public ADD, change the key, then log a private
 387  *       DEL.  So we need to log a DEL before we change a
 388  *       key from public to private.
 389  *   2.  It MUST have a valid (> -1) swap_file_number.
 390  */
 391 void
 392 storeDirSwapLog(const StoreEntry * e, int op)
 393 {
 394     storeSwapLogData *s;
 395     int dirn;
 396     dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
 397     assert(dirn < Config.cacheSwap.n_configured);
 398     assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
 399     assert(e->swap_file_number >= 0);
 400     /*
 401      * icons and such; don't write them to the swap log
 402      */
 403     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 404         return;
 405     assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
 406     debug(20, 3) ("storeDirSwapLog: %s %s %08X\n",
 407         swap_log_op_str[op],
 408         storeKeyText(e->key),
 409         e->swap_file_number);
 410     s = xcalloc(1, sizeof(storeSwapLogData));
 411     s->op = (char) op;
 412     s->swap_file_number = e->swap_file_number;
 413     s->timestamp = e->timestamp;
 414     s->lastref = e->lastref;
 415     s->expires = e->expires;
 416     s->lastmod = e->lastmod;
 417     s->swap_file_sz = e->swap_file_sz;
 418     s->refcount = e->refcount;
 419     s->flags = e->flags;
 420     xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
 421     file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd,
 422         -1,
 423         s,
 424         sizeof(storeSwapLogData),
 425         NULL,
 426         NULL,
 427         xfree);
 428 }
 429
 430 char *
 431 storeDirSwapLogFile(int dirn, const char *ext)
 432 {
 433     LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
 434     LOCAL_ARRAY(char, digit, 32);
 435     if (Config.Log.swap) {
 436         xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64);
 437         strcat(path, ".");
 438         snprintf(digit, 32, "%02d", dirn);
 439         strncat(path, digit, 3);
 440     } else {
 441         xstrncpy(path, storeSwapDir(dirn), SQUID_MAXPATHLEN - 64);
 442         strcat(path, "/swap.state");
 443     }
 444     if (ext)
 445         strncat(path, ext, 16);
 446     return path;
 447 }
 448
 449 void
 450 storeDirOpenSwapLogs(void)
 451 {
 452     int i;
 453     char *path;
 454     int fd;
 455     SwapDir *SD;
 456     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 457         SD = &Config.cacheSwap.swapDirs[i];
 458         path = storeDirSwapLogFile(i, NULL);
 459         fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
 460         if (fd < 0) {
 461             debug(50, 1) ("%s: %s\n", path, xstrerror());
 462             fatal("storeDirOpenSwapLogs: Failed to open swap log.");
 463         }
 464         debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", i, fd);
 465         SD->swaplog_fd = fd;
 466     }
 467 }
 468
 469 void
 470 storeDirCloseSwapLogs(void)
 471 {
 472     int i;
 473     SwapDir *SD;
 474     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 475         SD = &Config.cacheSwap.swapDirs[i];
 476         if (SD->swaplog_fd < 0) /* not open */
 477             continue;
 478         file_close(SD->swaplog_fd);
 479         debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", i, SD->swaplog_fd);
 480         SD->swaplog_fd = -1;
 481     }
 482 }
 483
 484 FILE *
 485 storeDirOpenTmpSwapLog(int dirn, int *clean_flag, int *zero_flag)
 486 {
 487     char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
 488     char *clean_path = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
 489     char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
 490     struct stat log_sb;
 491     struct stat clean_sb;
 492     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 493     FILE *fp;
 494     int fd;
 495     if (stat(swaplog_path, &log_sb) < 0) {
 496         debug(47, 1) ("Cache Dir #%d: No log file\n", dirn);
 497         safe_free(swaplog_path);
 498         safe_free(clean_path);
 499         safe_free(new_path);
 500         return NULL;
 501     }
 502     *zero_flag = log_sb.st_size == 0 ? 1 : 0;
 503     /* close the existing write-only FD */
 504     if (SD->swaplog_fd >= 0)
 505         file_close(SD->swaplog_fd);
 506     /* open a write-only FD for the new log */
 507     fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL);
 508     if (fd < 0) {
 509         debug(50, 1) ("%s: %s\n", new_path, xstrerror());
 510         fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
 511     }
 512     SD->swaplog_fd = fd;
 513     /* open a read-only stream of the old log */
 514     fp = fopen(swaplog_path, "r");
 515     if (fp == NULL) {
 516         debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
 517         fatal("Failed to open swap log for reading");
 518     }
 519     memset(&clean_sb, '\0', sizeof(struct stat));
 520     if (stat(clean_path, &clean_sb) < 0)
 521         *clean_flag = 0;
 522     else if (clean_sb.st_mtime < log_sb.st_mtime)
 523         *clean_flag = 0;
 524     else
 525         *clean_flag = 1;
 526     safeunlink(clean_path, 1);
 527     safe_free(swaplog_path);
 528     safe_free(clean_path);
 529     safe_free(new_path);
 530     return fp;
 531 }
 532
 533 void
 534 storeDirCloseTmpSwapLog(int dirn)
 535 {
 536     char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
 537     char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new"));
 538     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 539     int fd;
 540     file_close(SD->swaplog_fd);
 541     if (rename(new_path, swaplog_path) < 0) {
 542         debug(50, 0) ("%s,%s: %s\n", new_path, swaplog_path, xstrerror());
 543         fatal("storeDirCloseTmpSwapLog: rename failed");
 544     }
 545     fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL, NULL);
 546     if (fd < 0) {
 547         debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
 548         fatal("storeDirCloseTmpSwapLog: Failed to open swap log.");
 549     }
 550     safe_free(swaplog_path);
 551     safe_free(new_path);
 552     SD->swaplog_fd = fd;
 553     debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", dirn, fd);
 554 }
 555
 556 void
 557 storeDirUpdateSwapSize(int fn, size_t size, int sign)
 558 {
 559     int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
 560     int k = ((size + 1023) >> 10) * sign;
 561     Config.cacheSwap.swapDirs[dirn].cur_size += k;
 562     store_swap_size += k;
 563     if (sign > 0)
 564         n_disk_objects++;
 565     else if (sign < 0)
 566         n_disk_objects--;
 567 }
 568
 569 void
 570 storeDirStats(StoreEntry * sentry)
 571 {
 572     int i;
 573     SwapDir *SD;
 574 #if HAVE_STATVFS
 575     struct statvfs sfs;
 576 #endif
 577     storeAppendPrintf(sentry, "Store Directory Statistics:\n");
 578     storeAppendPrintf(sentry, "Store Entries          : %d\n",
 579         memInUse(MEM_STOREENTRY));
 580     storeAppendPrintf(sentry, "Maximum Swap Size      : %8d KB\n",
 581         Config.Swap.maxSize);
 582     storeAppendPrintf(sentry, "Current Store Swap Size: %8d KB\n",
 583         store_swap_size);
 584     storeAppendPrintf(sentry, "Current Capacity       : %d%% used, %d%% free\n",
 585         percent((int) store_swap_size, (int) Config.Swap.maxSize),
 586         percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize));
 587     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 588         SD = &Config.cacheSwap.swapDirs[i];
 589         storeAppendPrintf(sentry, "\n");
 590         storeAppendPrintf(sentry, "Store Directory #%d: %s\n", i, SD->path);
 591         storeAppendPrintf(sentry, "First level subdirectories: %d\n", SD->l1);
 592         storeAppendPrintf(sentry, "Second level subdirectories: %d\n", SD->l2);
 593         storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
 594         storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
 595         storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
 596             100.0 * SD->cur_size / SD->max_size);
 597         storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
 598             SD->map->n_files_in_map, SD->map->max_n_files,
 599             percent(SD->map->n_files_in_map, SD->map->max_n_files));
 600 #if HAVE_STATVFS
 601 #define fsbtoblk(num, fsbs, bs) \
 602         (((fsbs) != 0 && (fsbs) < (bs)) ? \
 603                 (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
 604         if (!statvfs(SD->path, &sfs)) {
 605             storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d (%d%%)\n",
 606                 fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_bsize, 1024),
 607                 fsbtoblk(sfs.f_blocks, sfs.f_bsize, 1024),
 608                 percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
 609             storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
 610                 sfs.f_files - sfs.f_ffree, sfs.f_files,
 611                 percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
 612         }
 613 #endif
 614     }
 615 }
 616
 617 int
 618 storeDirMapBitsInUse(void)
 619 {
 620     int i;
 621     int n = 0;
 622     for (i = 0; i < Config.cacheSwap.n_configured; i++)
 623         n += Config.cacheSwap.swapDirs[i].map->n_files_in_map;
 624     return n;
 625 }
 626
 627 /*
 628  *  storeDirWriteCleanLogs
 629  *
 630  *  Writes a "clean" swap log file from in-memory metadata.
 631  */
 632 #define CLEAN_BUF_SZ 16384
 633 int
 634 storeDirWriteCleanLogs(int reopen)
 635 {
 636     StoreEntry *e = NULL;
 637     int *fd;
 638     int n = 0;
 639     time_t start, stop, r;
 640     struct stat sb;
 641     char **cur;
 642     char **new;
 643     char **cln;
 644     int dirn;
 645     int N = Config.cacheSwap.n_configured;
 646     dlink_node *m;
 647     char **outbuf;
 648     off_t *outbufoffset;
 649     storeSwapLogData s;
 650     size_t ss = sizeof(storeSwapLogData);
 651     if (store_rebuilding) {
 652         debug(20, 1) ("Not currently OK to rewrite swap log.\n");
 653         debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n");
 654         return 0;
 655     }
 656     debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n");
 657     start = squid_curtime;
 658     fd = xcalloc(N, sizeof(int));
 659     cur = xcalloc(N, sizeof(char *));
 660     new = xcalloc(N, sizeof(char *));
 661     cln = xcalloc(N, sizeof(char *));
 662     for (dirn = 0; dirn < N; dirn++) {
 663         fd[dirn] = -1;
 664         cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL));
 665         new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean"));
 666         cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
 667         unlink(new[dirn]);
 668         unlink(cln[dirn]);
 669         fd[dirn] = file_open(new[dirn],
 670             O_WRONLY | O_CREAT | O_TRUNC,
 671             NULL,
 672             NULL,
 673             NULL);
 674         if (fd[dirn] < 0) {
 675             debug(50, 0) ("storeDirWriteCleanLogs: %s: %s\n", new[dirn], xstrerror());
 676             continue;
 677         }
 678         debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
 679             new[dirn], fd[dirn]);
 680 #if HAVE_FCHMOD
 681         if (stat(cur[dirn], &sb) == 0)
 682             fchmod(fd[dirn], sb.st_mode);
 683 #endif
 684     }
 685     outbuf = xcalloc(N, sizeof(char *));
 686     outbufoffset = xcalloc(N, sizeof(*outbufoffset));
 687     for (dirn = 0; dirn < N; dirn++) {
 688         outbuf[dirn] = xcalloc(CLEAN_BUF_SZ, 1);
 689         outbufoffset[dirn] = 0;
 690     }
 691     for (m = store_list.tail; m; m = m->prev) {
 692         e = m->data;
 693         if (e->swap_file_number < 0)
 694             continue;
 695         if (e->swap_status != SWAPOUT_DONE)
 696             continue;
 697         if (e->swap_file_sz <= 0)
 698             continue;
 699         if (EBIT_TEST(e->flags, RELEASE_REQUEST))
 700             continue;
 701         if (EBIT_TEST(e->flags, KEY_PRIVATE))
 702             continue;
 703         if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 704             continue;
 705         dirn = storeDirNumber(e->swap_file_number);
 706         assert(dirn < N);
 707         if (fd[dirn] < 0)
 708             continue;
 709         memset(&s, '\0', ss);
 710         s.op = (char) SWAP_LOG_ADD;
 711         s.swap_file_number = e->swap_file_number;
 712         s.timestamp = e->timestamp;
 713         s.lastref = e->lastref;
 714         s.expires = e->expires;
 715         s.lastmod = e->lastmod;
 716         s.swap_file_sz = e->swap_file_sz;
 717         s.refcount = e->refcount;
 718         s.flags = e->flags;
 719         xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
 720         xmemcpy(outbuf[dirn] + outbufoffset[dirn], &s, ss);
 721         outbufoffset[dirn] += ss;
 722         /* buffered write */
 723         if (outbufoffset[dirn] + ss > CLEAN_BUF_SZ) {
 724             if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
 725                 debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
 726                     new[dirn], xstrerror());
 727                 debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
 728                 file_close(fd[dirn]);
 729                 fd[dirn] = -1;
 730                 unlink(new[dirn]);
 731                 continue;
 732             }
 733             outbufoffset[dirn] = 0;
 734         }
 735         if ((++n & 0xFFFF) == 0) {
 736             getCurrentTime();
 737             debug(20, 1) ("  %7d entries written so far.\n", n);
 738         }
 739     }
 740     /* flush */
 741     for (dirn = 0; dirn < N; dirn++) {
 742         if (outbufoffset[dirn] == 0)
 743             continue;
 744         if (fd[dirn] < 0)
 745             continue;
 746         if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
 747             debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
 748                 new[dirn], xstrerror());
 749             debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
 750             file_close(fd[dirn]);
 751             fd[dirn] = -1;
 752             unlink(new[dirn]);
 753             continue;
 754         }
 755         safe_free(outbuf[dirn]);
 756     }
 757     safe_free(outbuf);
 758     safe_free(outbufoffset);
 759 #ifdef _SQUID_MSWIN_
 760     /*
 761      * You can't rename open files on Microsoft "operating systems"
 762      * so we close before renaming.
 763      */
 764     storeDirCloseSwapLogs();
 765 #endif
 766     /* rename */
 767     for (dirn = 0; dirn < N; dirn++) {
 768         if (fd[dirn] < 0)
 769             continue;
 770         if (rename(new[dirn], cur[dirn]) < 0) {
 771             debug(50, 0) ("storeDirWriteCleanLogs: rename failed: %s, %s -> %s\n",
 772                 xstrerror(), new[dirn], cur[dirn]);
 773         }
 774     }
 775 #ifndef _SQUID_MSWIN_
 776     storeDirCloseSwapLogs();
 777 #endif
 778     if (reopen)
 779         storeDirOpenSwapLogs();
 780     stop = squid_curtime;
 781     r = stop - start;
 782     debug(20, 1) ("  Finished.  Wrote %d entries.\n", n);
 783     debug(20, 1) ("  Took %d seconds (%6.1f entries/sec).\n",
 784         r > 0 ? (int) r : 0,
 785         (double) n / (r > 0 ? r : 1));
 786     /* touch a timestamp file if we're not still validating */
 787     if (!store_rebuilding) {
 788         for (dirn = 0; dirn < N; dirn++) {
 789             if (fd[dirn] < 0)
 790                 continue;
 791             file_close(file_open(cln[dirn],
 792                     O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL));
 793         }
 794     }
 795     /* close */
 796     for (dirn = 0; dirn < N; dirn++) {
 797         safe_free(cur[dirn]);
 798         safe_free(new[dirn]);
 799         safe_free(cln[dirn]);
 800         if (fd[dirn] < 0)
 801             continue;
 802         file_close(fd[dirn]);
 803         fd[dirn] = -1;
 804     }
 805     safe_free(cur);
 806     safe_free(new);
 807     safe_free(cln);
 808     safe_free(fd);
 809     return n;
 810 }
 811 #undef CLEAN_BUF_SZ
 812
 813 void
 814 storeDirConfigure(void)
 815 {
 816     SwapDir *SD;
 817     int n;
 818     int i;
 819     fileMap *fm;
 820     Config.Swap.maxSize = 0;
 821     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 822         SD = &Config.cacheSwap.swapDirs[i];;
 823         Config.Swap.maxSize += SD->max_size;
 824         n = 2 * SD->max_size / Config.Store.avgObjectSize;
 825         if (NULL == SD->map) {
 826             /* first time */
 827             SD->map = file_map_create(n);
 828         } else if (n > SD->map->max_n_files) {
 829             /* it grew, need to expand */
 830             fm = file_map_create(n);
 831             filemapCopy(SD->map, fm);
 832             filemapFreeMemory(SD->map);
 833             SD->map = fm;
 834         }
 835         /* else it shrunk, and we leave the old one in place */
 836     }
 837 }
 838
 839 void
 840 storeDirDiskFull(int fn)
 841 {
 842     int dirn = fn >> SWAP_DIR_SHIFT;
 843     SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
 844     assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
 845     SD->max_size = SD->cur_size;
 846     debug(20, 1) ("WARNING: Shrinking cache_dir #%d to %d KB\n",
 847         dirn, SD->cur_size);
 848 }