From: wessels <> Date: Tue, 3 Feb 1998 08:16:54 +0000 (+0000) Subject: STORE.C SPLIT X-Git-Tag: SQUID_3_0_PRE1~4198 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f09f5b266e7283bf0dbcb02c7024e906fb9e9ab3;p=thirdparty%2Fsquid.git STORE.C SPLIT Also made MD5 cache keys standard --- diff --git a/ChangeLog b/ChangeLog index f7941ebaf8..75792a3ca2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -68,6 +68,7 @@ called after the fdset had been built which was wrong because the callbacks were changing the state of the read/write handlers prior to the poll/select() calls (SLF). + - Fixed ARP ACL memory leaks (Dale). Changes to squid-1.2.beta12 (Jan 30, 1998): diff --git a/configure b/configure index 904cc07613..ab14283105 100755 --- a/configure +++ b/configure @@ -16,9 +16,6 @@ ac_help="$ac_help --enable-dlmalloc Compile & use the malloc package by Doug Lea" ac_help="$ac_help --enable-gnuregex Compile GNUregex" -ac_help="$ac_help - --enable-new-storekey[=sha|md5] - Use SHA/MD5 as store index, instead of URL" ac_help="$ac_help --enable-acltree[=bin|splay] Use tree function to store ACL lists" @@ -558,7 +555,7 @@ fi -# From configure.in Revision: 1.96 +# From configure.in Revision: 1.97 ac_aux_dir= for ac_dir in aux $srcdir/aux; do if test -f $ac_dir/install-sh; then @@ -586,7 +583,7 @@ else { echo "configure: error: can not run $ac_config_sub" 1>&2; exit 1; } fi echo $ac_n "checking host system type""... $ac_c" 1>&6 -echo "configure:590: checking host system type" >&5 +echo "configure:587: checking host system type" >&5 host_alias=$host case "$host_alias" in @@ -641,7 +638,7 @@ PRESET_CFLAGS="$CFLAGS" # Extract the first word of "gcc", so it can be a program name with args. set dummy gcc; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:645: checking for $ac_word" >&5 +echo "configure:642: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -670,7 +667,7 @@ if test -z "$CC"; then # Extract the first word of "cc", so it can be a program name with args. set dummy cc; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:674: checking for $ac_word" >&5 +echo "configure:671: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -718,7 +715,7 @@ fi fi echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works""... $ac_c" 1>&6 -echo "configure:722: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5 +echo "configure:719: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5 ac_ext=c # CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options. @@ -728,11 +725,11 @@ ac_link='${CC-cc} -o conftest $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS cross_compiling=$ac_cv_prog_cc_cross cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:733: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then ac_cv_prog_cc_works=yes # If we can't run a trivial program, we are probably using a cross compiler. if (./conftest; exit) 2>/dev/null; then @@ -752,12 +749,12 @@ if test $ac_cv_prog_cc_works = no; then { echo "configure: error: installation or configuration problem: C compiler cannot create executables." 1>&2; exit 1; } fi echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6 -echo "configure:756: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5 +echo "configure:753: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5 echo "$ac_t""$ac_cv_prog_cc_cross" 1>&6 cross_compiling=$ac_cv_prog_cc_cross echo $ac_n "checking whether we are using GNU C""... $ac_c" 1>&6 -echo "configure:761: checking whether we are using GNU C" >&5 +echo "configure:758: checking whether we are using GNU C" >&5 if eval "test \"`echo '$''{'ac_cv_prog_gcc'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -766,7 +763,7 @@ else yes; #endif EOF -if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:770: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then +if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:767: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then ac_cv_prog_gcc=yes else ac_cv_prog_gcc=no @@ -781,7 +778,7 @@ if test $ac_cv_prog_gcc = yes; then ac_save_CFLAGS="$CFLAGS" CFLAGS= echo $ac_n "checking whether ${CC-cc} accepts -g""... $ac_c" 1>&6 -echo "configure:785: checking whether ${CC-cc} accepts -g" >&5 +echo "configure:782: checking whether ${CC-cc} accepts -g" >&5 if eval "test \"`echo '$''{'ac_cv_prog_cc_g'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -923,47 +920,6 @@ if test "${enable_gnuregex+set}" = set; then fi -# Check whether --enable-new_storekey or --disable-new_storekey was given. -if test "${enable_new_storekey+set}" = set; then - enableval="$enable_new_storekey" - case "$enableval" in - yes|sha) - echo "Store key set to SHA" - cat >> confdefs.h <<\EOF -#define STORE_KEY_SHA 1 -EOF - - STORE_KEY=sha - ;; - md5) - echo "Store key set to MD5" - cat >> confdefs.h <<\EOF -#define STORE_KEY_MD5 1 -EOF - - STORE_KEY=md5 - ;; - no|url) - echo "Store key set to URL" - cat >> confdefs.h <<\EOF -#define STORE_KEY_URL 1 -EOF - - STORE_KEY=url - esac - -else - - STORE_KEY=url - cat >> confdefs.h <<\EOF -#define STORE_KEY_URL 1 -EOF - - -fi - - - # Check whether --enable-acltree or --disable-acltree was given. if test "${enable_acltree+set}" = set; then enableval="$enable_acltree" @@ -1179,7 +1135,7 @@ fi echo $ac_n "checking how to run the C preprocessor""... $ac_c" 1>&6 -echo "configure:1183: checking how to run the C preprocessor" >&5 +echo "configure:1139: checking how to run the C preprocessor" >&5 # On Suns, sometimes $CPP names a directory. if test -n "$CPP" && test -d "$CPP"; then CPP= @@ -1194,13 +1150,13 @@ else # On the NeXT, cc -E runs the code through the compiler's parser, # not just through cpp. cat > conftest.$ac_ext < Syntax Error EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1204: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1160: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then : @@ -1211,13 +1167,13 @@ else rm -rf conftest* CPP="${CC-cc} -E -traditional-cpp" cat > conftest.$ac_ext < Syntax Error EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1221: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1177: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then : @@ -1250,7 +1206,7 @@ echo "$ac_t""$CPP" 1>&6 # SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" # ./install, which can be erroneously created by make from ./install.sh. echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6 -echo "configure:1254: checking for a BSD compatible install" >&5 +echo "configure:1210: checking for a BSD compatible install" >&5 if test -z "$INSTALL"; then if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -1302,7 +1258,7 @@ test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' # Extract the first word of "ranlib", so it can be a program name with args. set dummy ranlib; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1306: checking for $ac_word" >&5 +echo "configure:1262: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_prog_RANLIB'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1329,7 +1285,7 @@ else fi echo $ac_n "checking whether ln -s works""... $ac_c" 1>&6 -echo "configure:1333: checking whether ln -s works" >&5 +echo "configure:1289: checking whether ln -s works" >&5 if eval "test \"`echo '$''{'ac_cv_prog_LN_S'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1352,7 +1308,7 @@ fi # Extract the first word of "sh", so it can be a program name with args. set dummy sh; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1356: checking for $ac_word" >&5 +echo "configure:1312: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_SH'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1384,7 +1340,7 @@ fi # Extract the first word of "false", so it can be a program name with args. set dummy false; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1388: checking for $ac_word" >&5 +echo "configure:1344: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_FALSE'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1416,7 +1372,7 @@ fi # Extract the first word of "true", so it can be a program name with args. set dummy true; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1420: checking for $ac_word" >&5 +echo "configure:1376: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_TRUE'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1448,7 +1404,7 @@ fi # Extract the first word of "rm", so it can be a program name with args. set dummy rm; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1452: checking for $ac_word" >&5 +echo "configure:1408: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_RM'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1480,7 +1436,7 @@ fi # Extract the first word of "mv", so it can be a program name with args. set dummy mv; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1484: checking for $ac_word" >&5 +echo "configure:1440: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_MV'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1512,7 +1468,7 @@ fi # Extract the first word of "mkdir", so it can be a program name with args. set dummy mkdir; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1516: checking for $ac_word" >&5 +echo "configure:1472: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_MKDIR'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1544,7 +1500,7 @@ fi # Extract the first word of "ln", so it can be a program name with args. set dummy ln; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1548: checking for $ac_word" >&5 +echo "configure:1504: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_LN'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1576,7 +1532,7 @@ fi # Extract the first word of "perl", so it can be a program name with args. set dummy perl; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1580: checking for $ac_word" >&5 +echo "configure:1536: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_PERL'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1608,7 +1564,7 @@ fi # Extract the first word of "makedepend", so it can be a program name with args. set dummy makedepend; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1612: checking for $ac_word" >&5 +echo "configure:1568: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_MAKEDEPEND'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1640,7 +1596,7 @@ fi # Extract the first word of "ar", so it can be a program name with args. set dummy ar; ac_word=$2 echo $ac_n "checking for $ac_word""... $ac_c" 1>&6 -echo "configure:1644: checking for $ac_word" >&5 +echo "configure:1600: checking for $ac_word" >&5 if eval "test \"`echo '$''{'ac_cv_path_AR'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -1690,12 +1646,12 @@ for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h do ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'` echo $ac_n "checking for $ac_hdr that defines DIR""... $ac_c" 1>&6 -echo "configure:1694: checking for $ac_hdr that defines DIR" >&5 +echo "configure:1650: checking for $ac_hdr that defines DIR" >&5 if eval "test \"`echo '$''{'ac_cv_header_dirent_$ac_safe'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include <$ac_hdr> @@ -1703,7 +1659,7 @@ int main() { DIR *dirp = 0; ; return 0; } EOF -if { (eval echo configure:1707: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:1663: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* eval "ac_cv_header_dirent_$ac_safe=yes" else @@ -1728,7 +1684,7 @@ done # Two versions of opendir et al. are in -ldir and -lx on SCO Xenix. if test $ac_header_dirent = dirent.h; then echo $ac_n "checking for opendir in -ldir""... $ac_c" 1>&6 -echo "configure:1732: checking for opendir in -ldir" >&5 +echo "configure:1688: checking for opendir in -ldir" >&5 ac_lib_var=`echo dir'_'opendir | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -1736,7 +1692,7 @@ else ac_save_LIBS="$LIBS" LIBS="-ldir $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:1707: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -1769,7 +1725,7 @@ fi else echo $ac_n "checking for opendir in -lx""... $ac_c" 1>&6 -echo "configure:1773: checking for opendir in -lx" >&5 +echo "configure:1729: checking for opendir in -lx" >&5 ac_lib_var=`echo x'_'opendir | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -1777,7 +1733,7 @@ else ac_save_LIBS="$LIBS" LIBS="-lx $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:1748: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -1811,12 +1767,12 @@ fi fi echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6 -echo "configure:1815: checking for ANSI C header files" >&5 +echo "configure:1771: checking for ANSI C header files" >&5 if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include @@ -1824,7 +1780,7 @@ else #include EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1828: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1784: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then rm -rf conftest* @@ -1841,7 +1797,7 @@ rm -f conftest* if test $ac_cv_header_stdc = yes; then # SunOS 4.x string.h does not declare mem*, contrary to ANSI. cat > conftest.$ac_ext < EOF @@ -1859,7 +1815,7 @@ fi if test $ac_cv_header_stdc = yes; then # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. cat > conftest.$ac_ext < EOF @@ -1880,7 +1836,7 @@ if test "$cross_compiling" = yes; then : else cat > conftest.$ac_ext < #define ISLOWER(c) ('a' <= (c) && (c) <= 'z') @@ -1891,7 +1847,7 @@ if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2); exit (0); } EOF -if { (eval echo configure:1895: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:1851: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then : else @@ -1967,17 +1923,17 @@ for ac_hdr in \ do ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'` echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6 -echo "configure:1971: checking for $ac_hdr" >&5 +echo "configure:1927: checking for $ac_hdr" >&5 if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < EOF ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out" -{ (eval echo configure:1981: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } +{ (eval echo configure:1937: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; } ac_err=`grep -v '^ *+' conftest.out` if test -z "$ac_err"; then rm -rf conftest* @@ -2005,12 +1961,12 @@ done echo $ac_n "checking for working const""... $ac_c" 1>&6 -echo "configure:2009: checking for working const" >&5 +echo "configure:1965: checking for working const" >&5 if eval "test \"`echo '$''{'ac_cv_c_const'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2019: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_c_const=yes else @@ -2080,14 +2036,14 @@ EOF fi echo $ac_n "checking whether byte ordering is bigendian""... $ac_c" 1>&6 -echo "configure:2084: checking whether byte ordering is bigendian" >&5 +echo "configure:2040: checking whether byte ordering is bigendian" >&5 if eval "test \"`echo '$''{'ac_cv_c_bigendian'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else ac_cv_c_bigendian=unknown # See if sys/param.h defines the BYTE_ORDER macro. cat > conftest.$ac_ext < #include @@ -2098,11 +2054,11 @@ int main() { #endif ; return 0; } EOF -if { (eval echo configure:2102: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2058: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* # It does; now see whether it defined to BIG_ENDIAN or not. cat > conftest.$ac_ext < #include @@ -2113,7 +2069,7 @@ int main() { #endif ; return 0; } EOF -if { (eval echo configure:2117: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2073: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_c_bigendian=yes else @@ -2133,7 +2089,7 @@ if test "$cross_compiling" = yes; then { echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; } else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:2106: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_c_bigendian=no else @@ -2171,20 +2127,20 @@ fi echo $ac_n "checking if ANSI prototypes work""... $ac_c" 1>&6 -echo "configure:2175: checking if ANSI prototypes work" >&5 +echo "configure:2131: checking if ANSI prototypes work" >&5 if eval "test \"`echo '$''{'ac_cv_have_ansi_prototypes'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2144: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_have_ansi_prototypes="yes" else @@ -2206,13 +2162,13 @@ EOF fi echo $ac_n "checking for tm->tm_gmtoff""... $ac_c" 1>&6 -echo "configure:2210: checking for tm->tm_gmtoff" >&5 +echo "configure:2166: checking for tm->tm_gmtoff" >&5 if eval "test \"`echo '$''{'ac_cv_have_tm_gmoff'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include @@ -2221,7 +2177,7 @@ struct tm foo; foo.tm_gmtoff = 0; ; return 0; } EOF -if { (eval echo configure:2225: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2181: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_have_tm_gmoff="yes" else @@ -2243,13 +2199,13 @@ EOF fi echo $ac_n "checking for extended mallinfo""... $ac_c" 1>&6 -echo "configure:2247: checking for extended mallinfo" >&5 +echo "configure:2203: checking for extended mallinfo" >&5 if eval "test \"`echo '$''{'ac_cv_have_ext_mallinfo'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include @@ -2258,7 +2214,7 @@ struct mallinfo foo; foo.mxfast = 0; ; return 0; } EOF -if { (eval echo configure:2262: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2218: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_have_ext_mallinfo="yes" else @@ -2280,13 +2236,13 @@ EOF fi echo $ac_n "checking for struct rusage""... $ac_c" 1>&6 -echo "configure:2284: checking for struct rusage" >&5 +echo "configure:2240: checking for struct rusage" >&5 if eval "test \"`echo '$''{'ac_cv_have_struct_rusage'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2259: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_have_struct_rusage="yes" else @@ -2321,13 +2277,13 @@ EOF fi echo $ac_n "checking for ip->ip_hl""... $ac_c" 1>&6 -echo "configure:2325: checking for ip->ip_hl" >&5 +echo "configure:2281: checking for ip->ip_hl" >&5 if eval "test \"`echo '$''{'ac_cv_have_ip_hl'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #include @@ -2344,7 +2300,7 @@ struct iphdr ip; ip.ip_hl= 0; ; return 0; } EOF -if { (eval echo configure:2348: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:2304: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_have_ip_hl="yes" else @@ -2366,7 +2322,7 @@ EOF fi echo $ac_n "checking size of int""... $ac_c" 1>&6 -echo "configure:2370: checking size of int" >&5 +echo "configure:2326: checking size of int" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_int'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -2374,7 +2330,7 @@ else { echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; } else cat > conftest.$ac_ext < main() @@ -2385,7 +2341,7 @@ main() exit(0); } EOF -if { (eval echo configure:2389: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:2345: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_int=`cat conftestval` else @@ -2405,7 +2361,7 @@ EOF echo $ac_n "checking size of long""... $ac_c" 1>&6 -echo "configure:2409: checking size of long" >&5 +echo "configure:2365: checking size of long" >&5 if eval "test \"`echo '$''{'ac_cv_sizeof_long'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -2413,7 +2369,7 @@ else { echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; } else cat > conftest.$ac_ext < main() @@ -2424,7 +2380,7 @@ main() exit(0); } EOF -if { (eval echo configure:2428: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:2384: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_sizeof_long=`cat conftestval` else @@ -2447,19 +2403,19 @@ EOF # The Ultrix 4.2 mips builtin alloca declared by alloca.h only works # for constant arguments. Useless! echo $ac_n "checking for working alloca.h""... $ac_c" 1>&6 -echo "configure:2451: checking for working alloca.h" >&5 +echo "configure:2407: checking for working alloca.h" >&5 if eval "test \"`echo '$''{'ac_cv_header_alloca_h'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < int main() { char *p = alloca(2 * sizeof(int)); ; return 0; } EOF -if { (eval echo configure:2463: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2419: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* ac_cv_header_alloca_h=yes else @@ -2480,12 +2436,12 @@ EOF fi echo $ac_n "checking for alloca""... $ac_c" 1>&6 -echo "configure:2484: checking for alloca" >&5 +echo "configure:2440: checking for alloca" >&5 if eval "test \"`echo '$''{'ac_cv_func_alloca_works'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2468: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* ac_cv_func_alloca_works=yes else @@ -2540,12 +2496,12 @@ EOF echo $ac_n "checking whether alloca needs Cray hooks""... $ac_c" 1>&6 -echo "configure:2544: checking whether alloca needs Cray hooks" >&5 +echo "configure:2500: checking whether alloca needs Cray hooks" >&5 if eval "test \"`echo '$''{'ac_cv_os_cray'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&6 if test $ac_cv_os_cray = yes; then for ac_func in _getb67 GETB67 getb67; do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 -echo "configure:2574: checking for $ac_func" >&5 +echo "configure:2530: checking for $ac_func" >&5 if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2558: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_$ac_func=yes" else @@ -2625,7 +2581,7 @@ done fi echo $ac_n "checking stack direction for C alloca""... $ac_c" 1>&6 -echo "configure:2629: checking stack direction for C alloca" >&5 +echo "configure:2585: checking stack direction for C alloca" >&5 if eval "test \"`echo '$''{'ac_cv_c_stack_direction'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -2633,7 +2589,7 @@ else ac_cv_c_stack_direction=0 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:2612: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_c_stack_direction=1 else @@ -2675,12 +2631,12 @@ fi echo $ac_n "checking for pid_t""... $ac_c" 1>&6 -echo "configure:2679: checking for pid_t" >&5 +echo "configure:2635: checking for pid_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_pid_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #if STDC_HEADERS @@ -2708,12 +2664,12 @@ EOF fi echo $ac_n "checking for size_t""... $ac_c" 1>&6 -echo "configure:2712: checking for size_t" >&5 +echo "configure:2668: checking for size_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_size_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #if STDC_HEADERS @@ -2741,12 +2697,12 @@ EOF fi echo $ac_n "checking for ssize_t""... $ac_c" 1>&6 -echo "configure:2745: checking for ssize_t" >&5 +echo "configure:2701: checking for ssize_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_ssize_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #if STDC_HEADERS @@ -2774,12 +2730,12 @@ EOF fi echo $ac_n "checking for off_t""... $ac_c" 1>&6 -echo "configure:2778: checking for off_t" >&5 +echo "configure:2734: checking for off_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_off_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #if STDC_HEADERS @@ -2807,12 +2763,12 @@ EOF fi echo $ac_n "checking for mode_t""... $ac_c" 1>&6 -echo "configure:2811: checking for mode_t" >&5 +echo "configure:2767: checking for mode_t" >&5 if eval "test \"`echo '$''{'ac_cv_type_mode_t'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < #if STDC_HEADERS @@ -2841,7 +2797,7 @@ fi echo $ac_n "checking for main in -lnsl""... $ac_c" 1>&6 -echo "configure:2845: checking for main in -lnsl" >&5 +echo "configure:2801: checking for main in -lnsl" >&5 ac_lib_var=`echo nsl'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2849,14 +2805,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lnsl $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2816: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -2884,7 +2840,7 @@ else fi echo $ac_n "checking for main in -lsocket""... $ac_c" 1>&6 -echo "configure:2888: checking for main in -lsocket" >&5 +echo "configure:2844: checking for main in -lsocket" >&5 ac_lib_var=`echo socket'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2892,14 +2848,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lsocket $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2859: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -2927,7 +2883,7 @@ else fi echo $ac_n "checking for main in -lgnumalloc""... $ac_c" 1>&6 -echo "configure:2931: checking for main in -lgnumalloc" >&5 +echo "configure:2887: checking for main in -lgnumalloc" >&5 ac_lib_var=`echo gnumalloc'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2935,14 +2891,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lgnumalloc $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2902: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -2984,7 +2940,7 @@ else ;; *) echo $ac_n "checking for main in -lmalloc""... $ac_c" 1>&6 -echo "configure:2988: checking for main in -lmalloc" >&5 +echo "configure:2944: checking for main in -lmalloc" >&5 ac_lib_var=`echo malloc'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -2992,14 +2948,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lmalloc $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:2959: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3030,7 +2986,7 @@ fi esac fi echo $ac_n "checking for main in -lbsd""... $ac_c" 1>&6 -echo "configure:3034: checking for main in -lbsd" >&5 +echo "configure:2990: checking for main in -lbsd" >&5 ac_lib_var=`echo bsd'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3038,14 +2994,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lbsd $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3005: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3073,7 +3029,7 @@ else fi echo $ac_n "checking for main in -lregex""... $ac_c" 1>&6 -echo "configure:3077: checking for main in -lregex" >&5 +echo "configure:3033: checking for main in -lregex" >&5 ac_lib_var=`echo regex'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3081,14 +3037,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lregex $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3048: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3114,7 +3070,7 @@ case "$host" in ;; *) echo $ac_n "checking for inet_aton in -lresolv""... $ac_c" 1>&6 -echo "configure:3118: checking for inet_aton in -lresolv" >&5 +echo "configure:3074: checking for inet_aton in -lresolv" >&5 ac_lib_var=`echo resolv'_'inet_aton | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3122,7 +3078,7 @@ else ac_save_LIBS="$LIBS" LIBS="-lresolv $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3093: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3149,7 +3105,7 @@ fi if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then echo "$ac_t""yes" 1>&6 echo $ac_n "checking for inet_aton in -l44bsd""... $ac_c" 1>&6 -echo "configure:3153: checking for inet_aton in -l44bsd" >&5 +echo "configure:3109: checking for inet_aton in -l44bsd" >&5 ac_lib_var=`echo 44bsd'_'inet_aton | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3157,7 +3113,7 @@ else ac_save_LIBS="$LIBS" LIBS="-l44bsd $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3128: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3200,7 +3156,7 @@ else fi echo $ac_n "checking for main in -lresolv""... $ac_c" 1>&6 -echo "configure:3204: checking for main in -lresolv" >&5 +echo "configure:3160: checking for main in -lresolv" >&5 ac_lib_var=`echo resolv'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3208,14 +3164,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lresolv $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3175: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3245,7 +3201,7 @@ fi ;; esac echo $ac_n "checking for main in -lm""... $ac_c" 1>&6 -echo "configure:3249: checking for main in -lm" >&5 +echo "configure:3205: checking for main in -lm" >&5 ac_lib_var=`echo m'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3253,14 +3209,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lm $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3220: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3289,7 +3245,7 @@ fi echo $ac_n "checking for crypt in -lcrypt""... $ac_c" 1>&6 -echo "configure:3293: checking for crypt in -lcrypt" >&5 +echo "configure:3249: checking for crypt in -lcrypt" >&5 ac_lib_var=`echo crypt'_'crypt | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3297,7 +3253,7 @@ else ac_save_LIBS="$LIBS" LIBS="-lcrypt $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3268: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3331,7 +3287,7 @@ fi echo $ac_n "checking for main in -lpthread""... $ac_c" 1>&6 -echo "configure:3335: checking for main in -lpthread" >&5 +echo "configure:3291: checking for main in -lpthread" >&5 ac_lib_var=`echo pthread'_'main | sed 'y%./+-%__p_%'` if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 @@ -3339,14 +3295,14 @@ else ac_save_LIBS="$LIBS" LIBS="-lpthread $LIBS" cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3306: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_lib_$ac_lib_var=yes" else @@ -3449,12 +3405,12 @@ for ac_func in \ do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 -echo "configure:3453: checking for $ac_func" >&5 +echo "configure:3409: checking for $ac_func" >&5 if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3437: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_$ac_func=yes" else @@ -3515,12 +3471,12 @@ case "$host" in ;; *) echo $ac_n "checking for poll""... $ac_c" 1>&6 -echo "configure:3519: checking for poll" >&5 +echo "configure:3475: checking for poll" >&5 if eval "test \"`echo '$''{'ac_cv_func_poll'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3503: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_poll=yes" else @@ -3566,7 +3522,7 @@ fi esac echo $ac_n "checking if setresuid is implemented""... $ac_c" 1>&6 -echo "configure:3570: checking if setresuid is implemented" >&5 +echo "configure:3526: checking if setresuid is implemented" >&5 if eval "test \"`echo '$''{'ac_cv_func_setresuid'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else @@ -3574,7 +3530,7 @@ else { echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; } else cat > conftest.$ac_ext < @@ -3587,7 +3543,7 @@ else } EOF -if { (eval echo configure:3591: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3547: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then ac_cv_func_setresuid="yes" else @@ -3611,7 +3567,7 @@ EOF fi echo $ac_n "checking if GNUregex needs to be compiled""... $ac_c" 1>&6 -echo "configure:3615: checking if GNUregex needs to be compiled" >&5 +echo "configure:3571: checking if GNUregex needs to be compiled" >&5 if test "$ac_cv_func_regcomp" = "no" ; then USE_GNUREGEX="yes" else @@ -3643,12 +3599,12 @@ for ac_func in \ do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 -echo "configure:3647: checking for $ac_func" >&5 +echo "configure:3603: checking for $ac_func" >&5 if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3631: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* eval "ac_cv_func_$ac_func=yes" else @@ -3699,12 +3655,12 @@ done echo $ac_n "checking Default FD_SETSIZE value""... $ac_c" 1>&6 -echo "configure:3703: checking Default FD_SETSIZE value" >&5 +echo "configure:3659: checking Default FD_SETSIZE value" >&5 if test "$cross_compiling" = yes; then DEFAULT_FD_SETSIZE=256 else cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3688: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then DEFAULT_FD_SETSIZE=`cat conftestval` else @@ -3747,12 +3703,12 @@ EOF echo $ac_n "checking Maximum number of filedescriptors we can open""... $ac_c" 1>&6 -echo "configure:3751: checking Maximum number of filedescriptors we can open" >&5 +echo "configure:3707: checking Maximum number of filedescriptors we can open" >&5 if test "$cross_compiling" = yes; then SQUID_MAXFD=256 else cat > conftest.$ac_ext < @@ -3804,7 +3760,7 @@ main() { } EOF -if { (eval echo configure:3808: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3764: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then SQUID_MAXFD=`cat conftestval` else @@ -3823,12 +3779,12 @@ EOF echo $ac_n "checking Default UDP send buffer size""... $ac_c" 1>&6 -echo "configure:3827: checking Default UDP send buffer size" >&5 +echo "configure:3783: checking Default UDP send buffer size" >&5 if test "$cross_compiling" = yes; then SQUID_UDP_SO_SNDBUF=16384 else cat > conftest.$ac_ext < @@ -3847,7 +3803,7 @@ main () } EOF -if { (eval echo configure:3851: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3807: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then SQUID_UDP_SO_SNDBUF=`cat conftestval` else @@ -3866,12 +3822,12 @@ EOF echo $ac_n "checking Default UDP receive buffer size""... $ac_c" 1>&6 -echo "configure:3870: checking Default UDP receive buffer size" >&5 +echo "configure:3826: checking Default UDP receive buffer size" >&5 if test "$cross_compiling" = yes; then SQUID_UDP_SO_RCVBUF=16384 else cat > conftest.$ac_ext < @@ -3890,7 +3846,7 @@ main () } EOF -if { (eval echo configure:3894: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3850: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then SQUID_UDP_SO_RCVBUF=`cat conftestval` else @@ -3909,12 +3865,12 @@ EOF echo $ac_n "checking Default TCP send buffer size""... $ac_c" 1>&6 -echo "configure:3913: checking Default TCP send buffer size" >&5 +echo "configure:3869: checking Default TCP send buffer size" >&5 if test "$cross_compiling" = yes; then SQUID_TCP_SO_SNDBUF=16384 else cat > conftest.$ac_ext < @@ -3933,7 +3889,7 @@ main () } EOF -if { (eval echo configure:3937: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3893: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then SQUID_TCP_SO_SNDBUF=`cat conftestval` else @@ -3952,12 +3908,12 @@ EOF echo $ac_n "checking Default TCP receive buffer size""... $ac_c" 1>&6 -echo "configure:3956: checking Default TCP receive buffer size" >&5 +echo "configure:3912: checking Default TCP receive buffer size" >&5 if test "$cross_compiling" = yes; then SQUID_TCP_SO_RCVBUF=16384 else cat > conftest.$ac_ext < @@ -3976,7 +3932,7 @@ main () } EOF -if { (eval echo configure:3980: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null +if { (eval echo configure:3936: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null then SQUID_TCP_SO_RCVBUF=`cat conftestval` else @@ -3995,19 +3951,19 @@ EOF echo $ac_n "checking if sys_errlist is already defined""... $ac_c" 1>&6 -echo "configure:3999: checking if sys_errlist is already defined" >&5 +echo "configure:3955: checking if sys_errlist is already defined" >&5 if eval "test \"`echo '$''{'ac_cv_needs_sys_errlist'+set}'`\" = set"; then echo $ac_n "(cached) $ac_c" 1>&6 else cat > conftest.$ac_ext < int main() { char *s = sys_errlist0; ; return 0; } EOF -if { (eval echo configure:4011: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then +if { (eval echo configure:3967: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then rm -rf conftest* ac_cv_needs_sys_errlist="no" else @@ -4029,16 +3985,16 @@ EOF fi echo $ac_n "checking for libresolv _dns_ttl_ hack""... $ac_c" 1>&6 -echo "configure:4033: checking for libresolv _dns_ttl_ hack" >&5 +echo "configure:3989: checking for libresolv _dns_ttl_ hack" >&5 cat > conftest.$ac_ext <&5; (eval $ac_link) 2>&5; } && test -s conftest; then +if { (eval echo configure:3998: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then rm -rf conftest* echo "$ac_t""yes" 1>&6 cat >> confdefs.h <<\EOF @@ -4226,7 +4182,6 @@ s%@CACHE_ICP_PORT@%$CACHE_ICP_PORT%g s%@CC@%$CC%g s%@LIBDLMALLOC@%$LIBDLMALLOC%g s%@LIB_MALLOC@%$LIB_MALLOC%g -s%@STORE_KEY@%$STORE_KEY%g s%@ASYNC_OBJS@%$ASYNC_OBJS%g s%@SQUID_PTHREAD_LIB@%$SQUID_PTHREAD_LIB%g s%@SNMPLIB@%$SNMPLIB%g diff --git a/configure.in b/configure.in index a6cf06a29c..097fa65328 100644 --- a/configure.in +++ b/configure.in @@ -3,13 +3,13 @@ dnl Configuration input file for Squid dnl dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9) dnl -dnl $Id: configure.in,v 1.97 1998/02/02 19:39:22 wessels Exp $ +dnl $Id: configure.in,v 1.98 1998/02/03 01:16:56 wessels Exp $ dnl dnl dnl AC_INIT(src/main.c) AC_CONFIG_HEADER(include/autoconf.h) -AC_REVISION($Revision: 1.97 $)dnl +AC_REVISION($Revision: 1.98 $)dnl AC_PREFIX_DEFAULT(/usr/local/squid) AC_CONFIG_AUX_DIR(aux) @@ -171,31 +171,6 @@ AC_ARG_ENABLE(gnuregex, [ --enable-gnuregex Compile GNUregex], [USE_GNUREGEX=$enableval]) -AC_ARG_ENABLE(new_storekey, -[ --enable-new-storekey[=sha|md5] - Use SHA/MD5 as store index, instead of URL], -[ case "$enableval" in - yes|sha) - echo "Store key set to SHA" - AC_DEFINE(STORE_KEY_SHA) - STORE_KEY=sha - ;; - md5) - echo "Store key set to MD5" - AC_DEFINE(STORE_KEY_MD5) - STORE_KEY=md5 - ;; - no|url) - echo "Store key set to URL" - AC_DEFINE(STORE_KEY_URL) - STORE_KEY=url - esac -],[ - STORE_KEY=url - AC_DEFINE(STORE_KEY_URL) -]) -AC_SUBST(STORE_KEY) - AC_ARG_ENABLE(acltree, [ --enable-acltree[=bin|splay] Use tree function to store ACL lists], diff --git a/include/autoconf.h.in b/include/autoconf.h.in index 4d6e4a9513..2327231338 100644 --- a/include/autoconf.h.in +++ b/include/autoconf.h.in @@ -74,18 +74,6 @@ */ #undef CACHEMGR_HOSTNAME -/* - * Normally Squid uses URLs as cache keys, and these are kept in memory. - * For large caches, this can become a significant use of memory. Define - * one of the options below for alternatives. SHA (Secure Hash Algorithm) - * is a 20-byte cryptographic digest. MD5 is a 16-byte cryptographic - * digest. Calculating SHA digests requires more CPU, and MD5 digests - * are slighly more likely to have collisions. - */ -#undef STORE_KEY_SHA -#undef STORE_KEY_MD5 -#undef STORE_KEY_URL - /* Define to do simple malloc debugging */ #undef XMALLOC_DEBUG diff --git a/src/Makefile.in b/src/Makefile.in index 8d69d1fbdc..c6ed5497ad 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.117 1998/02/02 19:39:27 wessels Exp $ +# $Id: Makefile.in,v 1.118 1998/02/03 01:17:02 wessels Exp $ # # Uncomment and customize the following to suit your needs: # @@ -116,8 +116,13 @@ OBJS = \ stmem.o \ store.o \ store_clean.o \ + store_client.o \ store_dir.o \ - store_key_@STORE_KEY@.o \ + store_key_md5.o \ + store_rebuild.o \ + store_swapin.o \ + store_swapmeta.o \ + store_swapout.o \ string_arrays.o \ tools.o \ unlinkd.o \ diff --git a/src/defines.h b/src/defines.h index 6bccc266f9..68bbeed2e6 100644 --- a/src/defines.h +++ b/src/defines.h @@ -182,3 +182,17 @@ #define IPC_TCP_SOCKET 1 #define IPC_UDP_SOCKET 2 #define IPC_FIFO 3 + +#define STORE_META_KEY STORE_META_KEY_MD5 + +#define STORE_META_TLD_START sizeof(int)+sizeof(char) +#define STORE_META_TLD_SIZE STORE_META_TLD_START +#define SwapMetaType(x) (char)x[0] +#define SwapMetaSize(x) &x[sizeof(char)] +#define SwapMetaData(x) &x[STORE_META_TLD_START] +#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(int)) + +#define STORE_ENTRY_WITH_MEMOBJ 1 +#define STORE_ENTRY_WITHOUT_MEMOBJ 0 +#define STORE_SWAP_BUF DISK_PAGE_SIZE +#define VM_WINDOW_SZ DISK_PAGE_SIZE diff --git a/src/enums.h b/src/enums.h index 6129b20b1d..1dbff23169 100644 --- a/src/enums.h +++ b/src/enums.h @@ -450,12 +450,19 @@ typedef enum { } mem_type; enum { - SWAP_META_VOID, /* should not come up */ - SWAP_META_KEY_URL, /* key w/ keytype */ - SWAP_META_KEY_SHA, - SWAP_META_KEY_MD5, - SWAP_META_URL, /* the url , if not in the header */ - SWAP_META_STD, /* standard metadata */ - SWAP_META_HITMETERING, /* reserved for hit metering */ - SWAP_META_VALID + STORE_META_VOID, /* should not come up */ + STORE_META_KEY_URL, /* key w/ keytype */ + STORE_META_KEY_SHA, + STORE_META_KEY_MD5, + STORE_META_URL, /* the url , if not in the header */ + STORE_META_STD, /* standard metadata */ + STORE_META_HITMETERING, /* reserved for hit metering */ + STORE_META_VALID +}; + +enum { + STORE_LOG_CREATE, + STORE_LOG_SWAPIN, + STORE_LOG_SWAPOUT, + STORE_LOG_RELEASE }; diff --git a/src/globals.h b/src/globals.h index b57bac6f2a..7b2df947fb 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.28 1998/01/31 05:31:57 wessels Exp $ + * $Id: globals.h,v 1.29 1998/02/03 01:17:04 wessels Exp $ */ extern FILE *debug_log; /* NULL */ @@ -98,6 +98,8 @@ extern char *err_type_str[]; extern char *icp_opcode_str[]; extern struct radix_node_head *AS_tree_head; extern double request_failure_ratio; /* 0.0 */ +extern int store_hash_buckets; /* 0 */ +extern hash_table *store_table; /* NULL */ #ifdef HAVE_SYSLOG extern int _db_level; diff --git a/src/protos.h b/src/protos.h index a544be2c8e..53a6298d9d 100644 --- a/src/protos.h +++ b/src/protos.h @@ -362,7 +362,6 @@ extern void shut_down(int); extern void start_announce(void *unused); extern void sslStart(int fd, const char *, request_t *, size_t * sz); extern void waisStart(request_t *, StoreEntry *); -extern void storeDirClean(void *unused); extern void passStart(int, const char *, request_t *, size_t *); extern void identStart(int, ConnStateData *, IDCB * callback); @@ -388,11 +387,13 @@ extern void stmemFreeData(mem_hdr *); /* ----------------------------------------------------------------- */ +/* + * store.c + */ +extern StoreEntry *new_StoreEntry(int, const char *, const char *); extern StoreEntry *storeGet(const cache_key *); extern StoreEntry *storeCreateEntry(const char *, const char *, int, method_t); extern void storeSetPublicKey(StoreEntry *); -extern StoreEntry *storeGetFirst(void); -extern StoreEntry *storeGetNext(void); extern void storeComplete(StoreEntry *); extern void storeInit(void); extern int storeClientWaiting(const StoreEntry *); @@ -414,7 +415,6 @@ extern int storePendingNClients(const StoreEntry *); extern int storeWriteCleanLogs(int reopen); extern HASHCMP urlcmp; extern EVH storeMaintainSwapSpace; - extern void storeExpireNow(StoreEntry *); extern void storeReleaseRequest(StoreEntry *); extern void storeRotateLog(void); @@ -424,7 +424,6 @@ extern void storeConfigure(void); extern void storeNegativeCache(StoreEntry *); extern void storeFreeMemory(void); extern int expiresMoreThan(time_t, time_t); -extern void storeClientListAdd(StoreEntry *, void *); extern int storeClientCopyPending(StoreEntry *, void *); extern void InvokeHandlers(StoreEntry *); extern int storeEntryValidToSend(StoreEntry *); @@ -433,13 +432,30 @@ extern time_t storeExpiredReferenceAge(void); extern void storeRegisterAbort(StoreEntry * e, STABH * cb, void *); extern void storeUnregisterAbort(StoreEntry * e); extern void storeMemObjectDump(MemObject * mem); +extern void storeEntryDump(StoreEntry * e); extern const char *storeUrl(const StoreEntry *); extern void storeCreateMemObject(StoreEntry *, const char *, const char *); extern void storeCopyNotModifiedReplyHeaders(MemObject * O, MemObject * N); extern void storeBuffer(StoreEntry *); extern void storeBufferFlush(StoreEntry *); +extern void storeHashInsert(StoreEntry * e, const cache_key *); +extern void storeSetMemStatus(StoreEntry * e, int); +#if !MONOTONIC_STORE +extern int storeGetUnusedFileno(void); +extern void storePutUnusedFileno(StoreEntry * e); +#endif +#ifdef __STDC__ +extern void storeAppendPrintf(StoreEntry *, const char *,...); +#else +extern void storeAppendPrintf(); +#endif +extern void storeLog(int tag, const StoreEntry * e); +extern int storeCheckCachable(StoreEntry * e); + -/* storeKey stuff */ +/* + * store_key_*.c + */ extern const cache_key *storeKeyDup(const cache_key *); extern void storeKeyFree(const cache_key *); extern const cache_key *storeKeyScan(const char *); @@ -450,12 +466,14 @@ extern int storeKeyHashBuckets(int); extern HASHHASH storeKeyHashHash; extern HASHCMP storeKeyHashCmp; -#ifdef __STDC__ -extern void storeAppendPrintf(StoreEntry *, const char *,...); -#else -extern void storeAppendPrintf(); -#endif +/* + * store_clean.c + */ +extern EVH storeDirClean; +/* + * store_dir.c + */ extern char *storeSwapFullPath(int, char *); extern char *storeSwapSubSubDir(int, char *); extern int storeVerifySwapDirs(void); @@ -477,6 +495,77 @@ extern int storeDirProperFileno(int dirn, int fn); extern void storeCreateSwapDirectories(void); extern int storeVerifyCacheDirs(void); +/* + * store_swapmeta.c + */ +extern int storeBuildMetaData(StoreEntry * e, char *swap_buf_c); +extern int getSwapHdr(int *, int *len, void *dst, char *write_buf, int hdr_len); +extern int getSwapHdr(int *, int *len, void *dst, char *write_buf, int hdr_len); +extern void addSwapHdr(int, int len, void *src, char *write_buf, int *write_len); +extern int storeGetMetaBuf(const char *buf, MemObject * mem); + +/* + * store_rebuild.c + */ +extern void storeDoRebuildFromSwapFiles(void *data); +extern void storeConvert(void); +extern void storeConvertFile(const cache_key * key, + int file_number, + int size, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean); +extern int storeGetNextFile(int *sfileno, int *size); +extern StoreEntry * storeAddDiskRestore(const cache_key * key, + int file_number, + int size, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean); +extern void storeDoConvertFromLog(void *data); +extern void storeCleanup(void *datanotused); +extern void storeValidate(StoreEntry *, STVLDCB, void *callback_data, void *tag); +extern void storeValidateComplete(void *data, int retcode, int errcode); +extern void storeStartRebuildFromDisk(void); + + +/* + * store_swapin.c + */ +extern void storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data); +extern void storeSwapInValidateComplete(void *data, int retcode, int errcode); +extern void storeSwapInFileOpened(void *data, int fd, int errcode); + +/* + * store_swapout.c + */ +extern void storeSwapOutStart(StoreEntry * e); +extern void storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data); +extern void storeCheckSwapOut(StoreEntry * e); +extern void storeSwapOutFileClose(StoreEntry * e); +extern void storeSwapOutFileClose(StoreEntry * e); + +/* + * store_client.c + */ +extern store_client * storeClientListSearch(const MemObject * mem, void *data); +extern void storeClientListAdd(StoreEntry * e, void *data); +extern void storeClientCopy(StoreEntry *, off_t , off_t , size_t , char *, STCB *, void *); +extern int storeClientCopyPending(StoreEntry * e, void *data); +extern int storeUnregister(StoreEntry * e, void *data); +extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); +extern void InvokeHandlers(StoreEntry * e); +extern int storePendingNClients(const StoreEntry * e); + + extern const char *getMyHostname(void); extern void safeunlink(const char *path, int quiet); extern void death(int sig); diff --git a/src/squid.h b/src/squid.h index 2a978d7e81..20a6133350 100644 --- a/src/squid.h +++ b/src/squid.h @@ -1,6 +1,6 @@ /* - * $Id: squid.h,v 1.153 1998/02/02 21:16:31 wessels Exp $ + * $Id: squid.h,v 1.154 1998/02/03 01:17:05 wessels Exp $ * * AUTHOR: Duane Wessels * @@ -169,6 +169,23 @@ #define assert(X) ((void)0) #endif +#if HAVE_DIRENT_H +#include +#define NAMLEN(dirent) strlen((dirent)->d_name) +#else /* HAVE_DIRENT_H */ +#define dirent direct +#define NAMLEN(dirent) (dirent)->d_namlen +#if HAVE_SYS_NDIR_H +#include +#endif /* HAVE_SYS_NDIR_H */ +#if HAVE_SYS_DIR_H +#include +#endif /* HAVE_SYS_DIR_H */ +#if HAVE_NDIR_H +#include +#endif /* HAVE_NDIR_H */ +#endif /* HAVE_DIRENT_H */ + #if defined(__QNX__) #include #endif @@ -282,21 +299,7 @@ struct rusage { #include #endif -#if STORE_KEY_SHA -#undef STORE_KEY_URL -#undef STORE_KEY_MD5 -#include "sha.h" -#elif STORE_KEY_MD5 -#undef STORE_KEY_URL -#undef STORE_KEY_SHA #include "md5.h" -#else -#undef STORE_KEY_SHA -#undef STORE_KEY_MD5 -#define STORE_KEY_URL 1 -#define storeKeyHashCmp urlcmp -#define storeKeyHashHash hash4 -#endif #ifdef SQUID_SNMP #include "snmp.h" diff --git a/src/stat.cc b/src/stat.cc index b7a2141b06..81b56bbbf5 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -1,6 +1,6 @@ /* - * $Id: stat.cc,v 1.188 1998/02/02 21:16:32 wessels Exp $ + * $Id: stat.cc,v 1.189 1998/02/03 01:17:07 wessels Exp $ * * DEBUG: section 18 Cache Manager Statistics * AUTHOR: Harvest Derived @@ -261,11 +261,14 @@ static void statObjects(StoreEntry * sentry, int vm_or_not) { StoreEntry *entry = NULL; + StoreEntry *next = NULL; MemObject *mem; int N = 0; int i; struct _store_client *sc; - for (entry = storeGetFirst(); entry != NULL; entry = storeGetNext()) { + next = (StoreEntry *) hash_first(store_table); + while (entry = next) { + next = (StoreEntry *) hash_next(store_table); mem = entry->mem_obj; if (vm_or_not && mem == NULL) continue; diff --git a/src/store.cc b/src/store.cc index 4da81544fe..b28c5784bc 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.370 1998/02/02 21:16:34 wessels Exp $ + * $Id: store.cc,v 1.371 1998/02/03 01:17:08 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -104,42 +104,12 @@ * re-implementations of code complying to this set of standards. */ -#include "squid.h" /* goes first */ +#include "squid.h" #define REBUILD_TIMESTAMP_DELTA_MAX 2 -#define SWAP_BUF DISK_PAGE_SIZE -#define VM_WINDOW_SZ DISK_PAGE_SIZE - -#define WITH_MEMOBJ 1 -#define WITHOUT_MEMOBJ 0 #define STORE_IN_MEM_BUCKETS (229) -#define STORE_LOG_CREATE 0 -#define STORE_LOG_SWAPIN 1 -#define STORE_LOG_SWAPOUT 2 -#define STORE_LOG_RELEASE 3 - -#if STORE_KEY_SHA -#define SWAP_META_KEY SWAP_META_KEY_SHA -#define squid_key_size SHA_DIGEST_INTS*sizeof(int) -#elif STORE_KEY_MD5 -#define SWAP_META_KEY SWAP_META_KEY_MD5 -#define squid_key_size MD5_DIGEST_CHARS -#else -#define SWAP_META_KEY SWAP_META_KEY_URL -#define squid_key_size -1 -#endif - - -#include -#define SWAP_META_TLD_START sizeof(int)+sizeof(char) -#define SWAP_META_TLD_SIZE SWAP_META_TLD_START -#define SwapMetaType(x) (char)x[0] -#define SwapMetaSize(x) &x[sizeof(char)] -#define SwapMetaData(x) &x[SWAP_META_TLD_START] -#define HDR_METASIZE (4*sizeof(time_t)+2*sizeof(u_short)+sizeof(int)) - static char *storeLogTags[] = { "CREATE", @@ -177,134 +147,30 @@ const char *swapStatusStr[] = "SWAPOUT_DONE" }; -struct storeRebuildState { - struct _rebuild_dir { - int dirn; - FILE *log; - int speed; - int clean; - struct _rebuild_dir *next; - } *rebuild_dir; - int objcount; /* # objects successfully reloaded */ - int expcount; /* # objects expired */ - int linecount; /* # lines parsed from cache logfile */ - int clashcount; /* # swapfile clashes avoided */ - int cancelcount; /* # objects cancelled */ - int dupcount; /* # duplicates purged */ - int invalid; /* # bad lines */ - int badflags; /* # bad e->flags */ - int need_to_validate; - time_t start; - time_t stop; - char *line_in; - size_t line_in_sz; -}; - -typedef struct storeCleanList { - const cache_key *key; - struct storeCleanList *next; -} storeCleanList; - -typedef void (VCB) (void *, int, int); - -typedef struct valid_ctrl_t { - struct stat *sb; - StoreEntry *e; - VCB *callback; - void *callback_data; -} valid_ctrl_t; - -typedef struct swapin_ctrl_t { - StoreEntry *e; - char *path; - SIH *callback; - void *callback_data; - store_client *sc; -} swapin_ctrl_t; - typedef struct lock_ctrl_t { SIH *callback; void *callback_data; StoreEntry *e; } lock_ctrl_t; -typedef struct swapout_ctrl_t { - char *swapfilename; - int oldswapstatus; - StoreEntry *e; -} swapout_ctrl_t; - /* Static Functions */ static int storeCheckExpired(const StoreEntry *, int flag); -static store_client *storeClientListSearch(const MemObject *, void *); static int storeEntryLocked(const StoreEntry *); static int storeEntryValidLength(const StoreEntry *); static void storeGetMemSpace(int); static void storeHashDelete(StoreEntry *); -static VCB storeSwapInValidateComplete; static MemObject *new_MemObject(const char *, const char *); -static StoreEntry *new_StoreEntry(int, const char *, const char *); -static StoreEntry *storeAddDiskRestore(const cache_key *, - int, - int, - time_t, - time_t, - time_t, - time_t, - u_num32, - u_num32, - int); static void destroy_MemObject(StoreEntry *); static void destroy_MemObjectData(MemObject *); static void destroy_StoreEntry(StoreEntry *); static void storePurgeMem(StoreEntry *); -static void storeStartRebuildFromDisk(void); -static void storeSwapOutStart(StoreEntry * e); -static DWCB storeSwapOutHandle; static void storeSetPrivateKey(StoreEntry *); -static EVH storeDoConvertFromLog; -static EVH storeCleanup; #if OLD_CODE -static VCB storeCleanupComplete; +static STVLDCB storeCleanupComplete; #endif -static void storeValidate(StoreEntry *, VCB *, void *, void *); -static AIOCB storeValidateComplete; -static void storeRebuiltFromDisk(struct storeRebuildState *data); static unsigned int getKeyCounter(void); -static void storePutUnusedFileno(StoreEntry * e); -static int storeGetUnusedFileno(void); -static void storeCheckSwapOut(StoreEntry * e); -static void storeSwapoutFileOpened(void *data, int fd, int errcode); -static int storeCheckCachable(StoreEntry * e); static int storeKeepInMemory(const StoreEntry *); -static SIH storeClientCopyFileOpened; -static DRCB storeClientCopyHandleRead; -static FOCB storeSwapInFileOpened; -static void storeClientCopyFileRead(store_client * sc); -static void storeSetMemStatus(StoreEntry * e, int); -static void storeClientCopy2(StoreEntry *, store_client *); -static void storeHashInsert(StoreEntry * e, const cache_key *); -static void storeSwapOutFileClose(StoreEntry * e); -static void storeEntryDump(StoreEntry * e); - -/* functions implementing meta data on store */ -static void storeConvert(void); -static void storeConvertFile(const cache_key *, int, int, time_t, time_t, time_t, time_t, u_num32, u_num32, int); - -static int storeBuildMetaData(StoreEntry *, char *); -static int storeGetMetaBuf(const char *, MemObject *); -#if 0 -static int storeParseMetaBuf(StoreEntry *); -#endif -static int storeGetNextFile(int *sfileno, int *size); -static void addSwapHdr(int, int, void *, char *, int *); -static int getSwapHdr(int *, int *, void *, char *, int); -static EVH storeDoRebuildFromSwapFiles; - -/* Now, this table is inaccessible to outsider. They have to use a method - * to access a value in internal storage data structure. */ -static hash_table *store_table = NULL; static dlink_list inmem_list; static dlink_list all_list; @@ -317,7 +183,6 @@ static int store_swap_low = 0; static int storelog_fd = -1; /* expiration parameters and stats */ -static int store_hash_buckets; static int store_maintain_rate; static int store_maintain_buckets; @@ -343,7 +208,7 @@ new_MemObject(const char *url, const char *log_url) return mem; } -static StoreEntry * +StoreEntry * new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) { StoreEntry *e = NULL; @@ -402,7 +267,7 @@ destroy_MemObjectData(MemObject * mem) /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ -static void +void storeHashInsert(StoreEntry * e, const cache_key * key) { debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", @@ -423,7 +288,7 @@ storeHashDelete(StoreEntry * e) /* -------------------------------------------------------------------------- */ -static void +void storeLog(int tag, const StoreEntry * e) { LOCAL_ARRAY(char, logmsg, MAX_URL << 1); @@ -595,7 +460,7 @@ storeCreateEntry(const char *url, const char *log_url, int flags, method_t metho MemObject *mem = NULL; debug(20, 3) ("storeCreateEntry: '%s' icp flags=%x\n", url, flags); - e = new_StoreEntry(WITH_MEMOBJ, url, log_url); + e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url); e->lock_count = 1; /* Note lock here w/o calling storeLock() */ mem = e->mem_obj; mem->method = method; @@ -627,129 +492,6 @@ storeCreateEntry(const char *url, const char *log_url, int flags, method_t metho return e; } -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -static StoreEntry * -storeAddDiskRestore(const cache_key * key, - int file_number, - int size, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_num32 flags, - int clean) -{ - StoreEntry *e = NULL; - debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new_StoreEntry(WITHOUT_MEMOBJ, NULL, NULL); - storeHashInsert(e, key); - e->store_status = STORE_OK; - storeSetMemStatus(e, NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_file_number = file_number; - e->object_len = size; - e->lock_count = 0; - e->refcount = 0; - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flag = flags; - EBIT_SET(e->flag, ENTRY_CACHABLE); - EBIT_CLR(e->flag, RELEASE_REQUEST); - EBIT_CLR(e->flag, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flag, ENTRY_VALIDATED); - storeDirMapBitSet(e->swap_file_number); - return e; -} - -int -storeUnregister(StoreEntry * e, void *data) -{ - MemObject *mem = e->mem_obj; - store_client *sc; - store_client **S; - STCB *callback; - if (mem == NULL) - return 0; - debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyText(e->key)); - for (S = &mem->clients; (sc = *S) != NULL; S = &(*S)->next) { - if (sc->callback_data == data) - break; - } - if (sc == NULL) - return 0; - *S = sc->next; - mem->nclients--; - sc->disk_op_in_progress = 0; - if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) - storeCheckSwapOut(e); - if (sc->swapin_fd > -1) { - commSetSelect(sc->swapin_fd, COMM_SELECT_READ, NULL, NULL, 0); - file_close(sc->swapin_fd); - } -#if USE_ASYNC_IO - else - aioCancel(-1, sc); -#endif - if ((callback = sc->callback) != NULL) { - /* callback with ssize = -1 to indicate unexpected termination */ - debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", - mem->url); - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, -1); - } - cbdataFree(sc); - return 1; -} - -off_t -storeLowestMemReaderOffset(const StoreEntry * entry) -{ - const MemObject *mem = entry->mem_obj; - off_t lowest = mem->inmem_hi; - store_client *sc; - store_client *nx = NULL; - for (sc = mem->clients; sc; sc = nx) { - nx = sc->next; - if (sc->callback_data == NULL) /* open slot */ - continue; - if (sc->type != STORE_MEM_CLIENT) - continue; - if (sc->copy_offset < lowest) - lowest = sc->copy_offset; - } - return lowest; -} - -/* Call handlers waiting for data to be appended to E. */ -void -InvokeHandlers(StoreEntry * e) -{ - int i = 0; - MemObject *mem = e->mem_obj; - store_client *sc; - store_client *nx = NULL; - assert(mem->clients != NULL || mem->nclients == 0); - debug(20, 3) ("InvokeHandlers: %s\n", storeKeyText(e->key)); - /* walk the entire list looking for valid callbacks */ - for (sc = mem->clients; sc; sc = nx) { - nx = sc->next; - debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); - if (sc->callback_data == NULL) - continue; - if (sc->callback == NULL) - continue; - storeClientCopy2(e, sc); - } -} - /* Mark object as expired */ void storeExpireNow(StoreEntry * e) @@ -758,238 +500,6 @@ storeExpireNow(StoreEntry * e) e->expires = squid_curtime; } -static void -storeSwapoutFileOpened(void *data, int fd, int errcode) -{ - swapout_ctrl_t *ctrlp = data; - int oldswapstatus = ctrlp->oldswapstatus; - char *swapfilename = ctrlp->swapfilename; - StoreEntry *e = ctrlp->e; - MemObject *mem; - - xfree(ctrlp); - if (fd == -2 && errcode == -2) { /* Cancelled - Clean up */ - xfree(swapfilename); - return; - } - assert(e->swap_status == SWAPOUT_OPENING); - if (fd < 0) { - debug(20, 0) ("storeSwapoutFileOpened: Unable to open swapfile: %s\n", - swapfilename); - storeDirMapBitReset(e->swap_file_number); - e->swap_file_number = -1; - e->swap_status = oldswapstatus; - xfree(swapfilename); - return; - } - mem = e->mem_obj; - mem->swapout.fd = (short) fd; - e->swap_status = SWAPOUT_WRITING; - debug(20, 5) ("storeSwapoutFileOpened: Begin SwapOut '%s' to FD %d FILE %s.\n", - mem->url, fd, swapfilename); - xfree(swapfilename); - debug(20, 5) ("swap_file_number=%08X\n", e->swap_file_number); - storeCheckSwapOut(e); -} - -/* start swapping object to disk */ -static void -storeSwapOutStart(StoreEntry * e) -{ - swapout_ctrl_t *ctrlp; - LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); - storeLockObject(e); -#if !MONOTONIC_STORE - if ((e->swap_file_number = storeGetUnusedFileno()) < 0) -#endif - e->swap_file_number = storeDirMapAllocate(); - storeSwapFullPath(e->swap_file_number, swapfilename); - ctrlp = xmalloc(sizeof(swapout_ctrl_t)); - ctrlp->swapfilename = xstrdup(swapfilename); - ctrlp->e = e; - ctrlp->oldswapstatus = e->swap_status; - e->swap_status = SWAPOUT_OPENING; - file_open(swapfilename, - O_WRONLY | O_CREAT | O_TRUNC, - storeSwapoutFileOpened, - ctrlp, e); -} - -static void -storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) -{ - StoreEntry *e = data; - MemObject *mem = e->mem_obj; - debug(20, 3) ("storeSwapOutHandle: '%s', len=%d\n", storeKeyText(e->key), (int) len); - if (flag < 0) { - debug(20, 1) ("storeSwapOutHandle: SwapOut failure (err code = %d).\n", - flag); - e->swap_status = SWAPOUT_NONE; - if (e->swap_file_number > -1) { -#if MONOTONIC_STORE -#if USE_ASYNC_IO - safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); -#else - unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL)); -#endif -#else - storePutUnusedFileno(e); -#endif - e->swap_file_number = -1; - } - if (flag == DISK_NO_SPACE_LEFT) { - /* reduce the swap_size limit to the current size. */ - Config.Swap.maxSize = store_swap_size; - storeConfigure(); - } - storeReleaseRequest(e); - storeSwapOutFileClose(e); - return; - } -#if USE_ASYNC_IO - if (mem == NULL) { - debug(20, 1) ("storeSwapOutHandle: mem == NULL : Cancelling swapout\n"); - return; - } -#else - assert(mem != NULL); -#endif - mem->swapout.done_offset += len; - if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len + mem->swapout.meta_len) { - storeCheckSwapOut(e); - return; - } - /* swapping complete */ - debug(20, 5) ("storeSwapOutHandle: SwapOut complete: '%s' to %s.\n", - mem->url, storeSwapFullPath(e->swap_file_number, NULL)); - e->swap_status = SWAPOUT_DONE; - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); - if (storeCheckCachable(e)) { - storeLog(STORE_LOG_SWAPOUT, e); -#if 0 - storeDirSwapLog(e); -#endif - } - /* Note, we don't otherwise call storeReleaseRequest() here because - * storeCheckCachable() does it for is if necessary */ - storeSwapOutFileClose(e); -} - -static void -storeCheckSwapOut(StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - off_t lowest_offset; - off_t new_mem_lo; - size_t swapout_size; - char *swap_buf; - ssize_t swap_buf_len; - int x; - int hdr_len = 0; - assert(mem != NULL); - /* should we swap something out to disk? */ - debug(20, 3) ("storeCheckSwapOut: %s\n", mem->url); - debug(20, 3) ("storeCheckSwapOut: store_status = %s\n", - storeStatusStr[e->store_status]); - if (e->store_status == STORE_ABORTED) { - assert(EBIT_TEST(e->flag, RELEASE_REQUEST)); - storeSwapOutFileClose(e); - return; - } - debug(20, 3) ("storeCheckSwapOut: mem->inmem_lo = %d\n", - (int) mem->inmem_lo); - debug(20, 3) ("storeCheckSwapOut: mem->inmem_hi = %d\n", - (int) mem->inmem_hi); - debug(20, 3) ("storeCheckSwapOut: swapout.queue_offset = %d\n", - (int) mem->swapout.queue_offset); - debug(20, 3) ("storeCheckSwapOut: swapout.done_offset = %d\n", - (int) mem->swapout.done_offset); -#if USE_ASYNC_IO - if (mem->inmem_hi < mem->swapout.queue_offset) { - storeAbort(e, 0); - assert(EBIT_TEST(e->flag, RELEASE_REQUEST)); - storeSwapOutFileClose(e); - return; - } -#else - assert(mem->inmem_hi >= mem->swapout.queue_offset); -#endif - swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); - lowest_offset = storeLowestMemReaderOffset(e); - debug(20, 3) ("storeCheckSwapOut: lowest_offset = %d\n", - (int) lowest_offset); - assert(lowest_offset >= mem->inmem_lo); - - new_mem_lo = lowest_offset; - if (!EBIT_TEST(e->flag, ENTRY_CACHABLE)) { - if (!EBIT_TEST(e->flag, KEY_PRIVATE)) - debug(20, 0) ("storeCheckSwapOut: Attempt to swap out a non-cacheable non-private object!\n"); - stmemFreeDataUpto(mem->data, new_mem_lo); - mem->inmem_lo = new_mem_lo; - return; - } - if (mem->swapout.queue_offset < new_mem_lo) - new_mem_lo = mem->swapout.queue_offset; - stmemFreeDataUpto(mem->data, new_mem_lo); - mem->inmem_lo = new_mem_lo; - - swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); - debug(20, 3) ("storeCheckSwapOut: swapout_size = %d\n", - (int) swapout_size); - if (swapout_size == 0) - return; - if (e->store_status == STORE_PENDING && swapout_size < VM_WINDOW_SZ) - return; /* wait for a full block */ - /* Ok, we have stuff to swap out. Is there a swapout.fd open? */ - if (e->swap_status == SWAPOUT_NONE) { - assert(mem->swapout.fd == -1); - if (storeCheckCachable(e)) - storeSwapOutStart(e); - /* else ENTRY_CACHABLE will be cleared and we'll never get - * here again */ - return; - } - if (e->swap_status == SWAPOUT_OPENING) - return; - assert(mem->swapout.fd > -1); - swap_buf = memAllocate(MEM_DISK_BUF, 1); - if (mem->swapout.queue_offset == 0) - hdr_len = storeBuildMetaData(e, swap_buf); - - if (swapout_size > SWAP_BUF - hdr_len) - swapout_size = SWAP_BUF - hdr_len; - - swap_buf_len = stmemCopy(mem->data, - mem->swapout.queue_offset, - swap_buf + hdr_len, - swapout_size) + hdr_len; - - if (swap_buf_len < 0) { - debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); - /* XXX This is probably wrong--we should storeRelease()? */ - storeDirMapBitReset(e->swap_file_number); - safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); - e->swap_file_number = -1; - e->swap_status = SWAPOUT_NONE; - memFree(MEM_DISK_BUF, swap_buf); - storeSwapOutFileClose(e); - return; - } - debug(20, 3) ("storeCheckSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); - assert(swap_buf_len > 0); - debug(20, 3) ("storeCheckSwapOut: swapping out %d bytes from %d\n", - swap_buf_len, mem->swapout.queue_offset); - mem->swapout.queue_offset += swap_buf_len - hdr_len; - x = file_write(mem->swapout.fd, - -1, - swap_buf, - swap_buf_len, - storeSwapOutHandle, - e, - memFreeDISK); - assert(x == DISK_OK); -} - /* Append incoming data from a primary server to an entry. */ void storeAppend(StoreEntry * e, const char *buf, int len) @@ -1037,446 +547,7 @@ storeAppendPrintf(va_alist) va_end(args); } -/* start swapping in */ -/* callback_data will become the tag on which the stat/open can be aborted */ -void -storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) -{ - swapin_ctrl_t *ctrlp; - assert(e->mem_status == NOT_IN_MEMORY); -#if OLD_CODE - if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { - if (storeDirMapBitTest(e->swap_file_number)) { - /* someone took our file while we weren't looking */ - callback(-1, callback_data); - return; - } - } -#endif - debug(20, 3) ("storeSwapInStart: called for %08X %s \n", - e->swap_file_number, e->key ? e->key : "[no key]"); - assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); - assert(e->swap_file_number >= 0); - assert(e->mem_obj != NULL); - ctrlp = xmalloc(sizeof(swapin_ctrl_t)); - ctrlp->e = e; - ctrlp->callback = callback; - ctrlp->callback_data = callback_data; - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) - storeSwapInValidateComplete(ctrlp, 0, 0); - else - storeValidate(e, storeSwapInValidateComplete, ctrlp, callback_data); -} - - -static void -storeSwapInValidateComplete(void *data, int retcode, int errcode) -{ - swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; - StoreEntry *e; - - if (retcode == -2 && errcode == -2) { - xfree(ctrlp); - return; - } - e = ctrlp->e; - assert(e->mem_status == NOT_IN_MEMORY); - if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { - /* Invoke a store abort that should free the memory object */ - (ctrlp->callback) (-1, ctrlp->callback_data); - xfree(ctrlp); - return; - } - ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); - debug(20, 3) ("storeSwapInValidateComplete: Opening %s\n", ctrlp->path); - file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp, ctrlp->callback_data); -} - -static void -storeSwapInFileOpened(void *data, int fd, int errcode) -{ - swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; - StoreEntry *e = ctrlp->e; - MemObject *mem = e->mem_obj; - struct stat sb; - - if (fd == -2 && errcode == -2) { - xfree(ctrlp->path); - xfree(ctrlp); - return; - } - assert(mem != NULL); - assert(e->mem_status == NOT_IN_MEMORY); - assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); - if (e->swap_status == SWAPOUT_DONE && (fd >= 0) && fstat(fd, &sb) == 0) - if (sb.st_size == 0 || sb.st_size != e->object_len) { - debug(20, 0) ("storeSwapInFileOpened: %s: Size mismatch: %d(fstat) != %d(object)\n", ctrlp->path, sb.st_size, e->object_len); - file_close(fd); - fd = -1; - } - if (fd < 0) { - debug(20, 0) ("storeSwapInStartComplete: Failed for '%s' (%s)\n", mem->url, - ctrlp->path); - /* Invoke a store abort that should free the memory object */ - (ctrlp->callback) (-1, ctrlp->callback_data); - xfree(ctrlp->path); - xfree(ctrlp); - return; - } - debug(20, 5) ("storeSwapInStart: initialized swap file '%s' for '%s'\n", - ctrlp->path, mem->url); - (ctrlp->callback) (fd, ctrlp->callback_data); - xfree(ctrlp->path); - xfree(ctrlp); -} - -/* convert storage .. this is the old storeDoRebuildFromDisk() */ - -static void -storeDoConvertFromLog(void *data) -{ - struct storeRebuildState *RB = data; - LOCAL_ARRAY(char, swapfile, MAXPATHLEN); - LOCAL_ARRAY(char, keytext, MAX_URL); - StoreEntry *e = NULL; - time_t expires; - time_t timestamp; - time_t lastref; - time_t lastmod; - int scan1; - int scan2; - int scan3; - int scan4; - int scan5; - int scan6; - int scan7; - off_t size; - int sfileno = 0; - int count; - int x; - struct _rebuild_dir *d; - struct _rebuild_dir **D; - int used; /* is swapfile already in use? */ - int newer; /* is the log entry newer than current entry? */ - const cache_key *key; - - /* load a number of objects per invocation */ - - if ((d = RB->rebuild_dir) == NULL) { - debug(20, 3) ("Done Converting, here are the stats.\n"); - storeRebuiltFromDisk(RB); - return; - } - for (count = 0; count < d->speed; count++) { - if (fgets(RB->line_in, RB->line_in_sz, d->log) == NULL) { - debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); - fclose(d->log); - d->log = NULL; - storeDirCloseTmpSwapLog(d->dirn); - RB->rebuild_dir = d->next; - safe_free(d); - eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); - return; - } - if ((++RB->linecount & 0x3FFF) == 0) - debug(20, 1) (" %7d Lines read so far.\n", RB->linecount); - debug(20, 9) ("line_in: %s", RB->line_in); - if (RB->line_in[0] == '\0') - continue; - if (RB->line_in[0] == '\n') - continue; - if (RB->line_in[0] == '#') - continue; - keytext[0] = '\0'; - sfileno = 0; - scan1 = 0; - scan2 = 0; - scan3 = 0; - scan4 = 0; - scan5 = 0; - scan6 = 0; - scan7 = 0; - x = sscanf(RB->line_in, "%x %x %x %x %x %d %d %x %s", - &sfileno, /* swap_file_number */ - &scan1, /* timestamp */ - &scan2, /* lastref */ - &scan3, /* expires */ - &scan4, /* last modified */ - &scan5, /* size */ - &scan6, /* refcount */ - &scan7, /* flags */ - keytext); /* key */ - if (x < 1) { - RB->invalid++; - continue; - } - if (x != 9) { - RB->invalid++; - continue; - } - timestamp = (time_t) scan1; - lastref = (time_t) scan2; - expires = (time_t) scan3; - lastmod = (time_t) scan4; - size = (off_t) scan5; - - if (size < 0) { - if ((key = storeKeyScan(keytext)) == NULL) - continue; - if ((e = storeGet(key)) == NULL) - continue; - if (e->lastref > lastref) - continue; - debug(20, 3) ("storeRebuildFromDisk: Cancelling: '%s'\n", keytext); - storeRelease(e); - RB->objcount--; - RB->cancelcount++; - continue; - } - storeSwapFullPath(sfileno, swapfile); - if (EBIT_TEST(scan7, KEY_PRIVATE)) { - RB->badflags++; - continue; - } - sfileno = storeDirProperFileno(d->dirn, sfileno); - - key = storeKeyScan(keytext); - if (key == NULL) { - debug(20, 1) ("storeDoConvertFromLog: bad key: '%s'\n", keytext); - continue; - } - e = storeGet(key); - used = storeDirMapBitTest(sfileno); - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - newer = e ? (lastref > e->lastref ? 1 : 0) : 0; - if (used && !newer) { - /* log entry is old, ignore it */ - RB->clashcount++; - continue; - } else if (used && e && e->swap_file_number == sfileno) { - /* swapfile taken, same URL, newer, update meta */ - e->lastref = timestamp; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->flag |= (u_num32) scan6; - e->refcount += (u_num32) scan7; - continue; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", - sfileno); - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - /* assert(RB->need_to_validate); */ - RB->clashcount++; - continue; - } else if (e) { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - storeRelease(e); /* release old entry */ - RB->dupcount++; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - /* update store_swap_size */ - RB->objcount++; - storeConvertFile(key, - sfileno, - (int) size, - expires, - timestamp, - lastref, - lastmod, - (u_num32) scan6, /* refcount */ - (u_num32) scan7, /* flags */ - d->clean); -#if 0 - storeDirSwapLog(e); -#endif - } - RB->rebuild_dir = d->next; - for (D = &RB->rebuild_dir; *D; D = &(*D)->next); - *D = d; - d->next = NULL; - eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); -} - -static void -storeCleanup(void *datanotused) -{ - static int bucketnum = -1; - static int validnum = 0; - StoreEntry *e; - hash_link *link_ptr = NULL; - - if (++bucketnum >= store_hash_buckets) { - debug(20, 1) (" Completed Validation Procedure\n"); - debug(20, 1) (" Validated %d Entries\n", validnum); - debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); - store_rebuilding = 0; - return; - } - link_ptr = hash_get_bucket(store_table, bucketnum); - for (; link_ptr; link_ptr = link_ptr->next) { - e = (StoreEntry *) link_ptr; - if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) - continue; - if (EBIT_TEST(e->flag, RELEASE_REQUEST)) - continue; - EBIT_SET(e->flag, ENTRY_VALIDATED); - /* Only set the file bit if we know its a valid entry */ - /* otherwise, set it in the validation procedure */ - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); - if ((++validnum & 0xFFFF) == 0) - debug(20, 1) (" %7d Entries Validated so far.\n", validnum); - assert(validnum <= memInUse(MEM_STOREENTRY)); - } - eventAdd("storeCleanup", storeCleanup, NULL, 0); -} - -#if OLD_CODE -static void -storeCleanupComplete(void *data, int retcode, int errcode) -{ - StoreEntry *e = data; - storeUnlockObject(e); - outvalid--; - if (retcode == -2 && errcode == -2) - return; - if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) - storeRelease(e); -} -#endif - -static void -storeValidate(StoreEntry * e, VCB callback, void *callback_data, void *tag) -{ - valid_ctrl_t *ctrlp; - char *path; - struct stat *sb; -#if !USE_ASYNC_IO - int x; -#endif - assert(!EBIT_TEST(e->flag, ENTRY_VALIDATED)); - if (e->swap_file_number < 0) { - EBIT_CLR(e->flag, ENTRY_VALIDATED); - callback(callback_data, 0, 0); - return; - } - path = storeSwapFullPath(e->swap_file_number, NULL); - sb = xmalloc(sizeof(struct stat)); - ctrlp = xmalloc(sizeof(valid_ctrl_t)); - ctrlp->sb = sb; - ctrlp->e = e; - ctrlp->callback = callback; - ctrlp->callback_data = callback_data; -#if USE_ASYNC_IO - aioStat(path, sb, storeValidateComplete, ctrlp, tag); -#else - /* When evaluating the actual arguments in a function call, the order - * in which the arguments and the function expression are evaluated is - * not specified; */ - x = stat(path, sb); - storeValidateComplete(ctrlp, x, errno); -#endif - return; -} - -static void -storeValidateComplete(void *data, int retcode, int errcode) -{ - valid_ctrl_t *ctrlp = data; - struct stat *sb = ctrlp->sb; - StoreEntry *e = ctrlp->e; - char *path; - - if (retcode == -2 && errcode == -2) { - xfree(sb); - xfree(ctrlp); - ctrlp->callback(ctrlp->callback_data, retcode, errcode); - return; - } - if (retcode < 0 && errcode == EWOULDBLOCK) { - path = storeSwapFullPath(e->swap_file_number, NULL); - retcode = stat(path, sb); - } - if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { - EBIT_CLR(e->flag, ENTRY_VALIDATED); - } else { - EBIT_SET(e->flag, ENTRY_VALIDATED); - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); - } - errno = errcode; - ctrlp->callback(ctrlp->callback_data, retcode, errcode); - xfree(sb); - xfree(ctrlp); -} - -/* meta data recreated from disk image in swap directory */ -static void -storeRebuiltFromDisk(struct storeRebuildState *data) -{ - time_t r; - time_t stop; - stop = squid_curtime; - r = stop - data->start; - debug(20, 1) ("Finished rebuilding storage from disk image.\n"); - debug(20, 1) (" %7d Lines read from previous logfile.\n", data->linecount); - debug(20, 1) (" %7d Invalid lines.\n", data->invalid); - debug(20, 1) (" %7d With invalid flags.\n", data->badflags); - debug(20, 1) (" %7d Objects loaded.\n", data->objcount); - debug(20, 1) (" %7d Objects expired.\n", data->expcount); - debug(20, 1) (" %7d Objects cancelled.\n", data->cancelcount); - debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); - debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); - debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", - r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); - debug(20, 1) ("Beginning Validation Procedure\n"); - eventAdd("storeCleanup", storeCleanup, NULL, 0); - memFree(MEM_4K_BUF, data->line_in); - safe_free(data); -} - -static void -storeStartRebuildFromDisk(void) -{ - struct storeRebuildState *RB; - struct _rebuild_dir *d; - int clean = 1; - RB = xcalloc(1, sizeof(struct storeRebuildState)); - RB->start = squid_curtime; - d = xcalloc(1, sizeof(struct _rebuild_dir)); - d->clean = clean; - d->speed = opt_foreground_rebuild ? 1 << 30 : 50; - RB->rebuild_dir = d; - if (!clean) - RB->need_to_validate = 1; - debug(20, 1) ("Rebuilding storage (%s)\n", - clean ? "CLEAN" : "DIRTY"); - if (opt_foreground_rebuild) { - storeDoRebuildFromSwapFiles(RB); - } else { - eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); - } -} - -static int +int storeCheckCachable(StoreEntry * e) { #if CACHE_ALL_METHODS @@ -1567,22 +638,6 @@ storeAbort(StoreEntry * e, int cbflag) storeSwapOutFileClose(e); } -/* get the first entry in the storage */ -StoreEntry * -storeGetFirst(void) -{ - return ((StoreEntry *) hash_first(store_table)); -} - - -/* get the next entry in the storage for a given search pointer */ -StoreEntry * -storeGetNext(void) -{ - return ((StoreEntry *) hash_next(store_table)); -} - - /* Clear Memory storage to accommodate the given object len */ static void storeGetMemSpace(int size) @@ -1764,231 +819,6 @@ storeEntryLocked(const StoreEntry * e) return 0; } -/* check if there is any client waiting for this object at all */ -/* return 1 if there is at least one client */ -int -storeClientWaiting(const StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - store_client *sc; - for (sc = mem->clients; sc; sc = sc->next) { - if (sc->callback_data != NULL) - return 1; - } - return 0; -} - -static store_client * -storeClientListSearch(const MemObject * mem, void *data) -{ - store_client *sc; - for (sc = mem->clients; sc; sc = sc->next) { - if (sc->callback_data == data) - break; - } - return sc; -} - -/* add client with fd to client list */ -void -storeClientListAdd(StoreEntry * e, void *data) -{ - MemObject *mem = e->mem_obj; - store_client **T; - store_client *sc; - assert(mem); - if (storeClientListSearch(mem, data) != NULL) - return; - mem->nclients++; - sc = memAllocate(MEM_STORE_CLIENT, 1); - cbdataAdd(sc, MEM_STORE_CLIENT); /* sc is callback_data for file_read */ - sc->callback_data = data; - sc->seen_offset = 0; - sc->copy_offset = 0; - sc->swapin_fd = -1; - sc->disk_op_in_progress = 0; - sc->mem = mem; - if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) - sc->type = STORE_MEM_CLIENT; - else - sc->type = STORE_DISK_CLIENT; - for (T = &mem->clients; *T; T = &(*T)->next); - *T = sc; -} - -/* copy bytes requested by the client */ -void -storeClientCopy(StoreEntry * e, - off_t seen_offset, - off_t copy_offset, - size_t size, - char *buf, - STCB * callback, - void *data) -{ - store_client *sc; - static int recurse_detect = 0; - assert(e->store_status != STORE_ABORTED); - assert(recurse_detect < 3); /* could == 1 for IMS not modified's */ - debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", - storeKeyText(e->key), - (int) seen_offset, - (int) copy_offset, - (int) size, - callback, - data); - sc = storeClientListSearch(e->mem_obj, data); - assert(sc != NULL); - assert(sc->callback == NULL); - sc->copy_offset = copy_offset; - sc->seen_offset = seen_offset; - sc->callback = callback; - sc->copy_buf = buf; - sc->copy_size = size; - sc->copy_offset = copy_offset; - storeClientCopy2(e, sc); - recurse_detect--; -} - -static void -storeClientCopy2(StoreEntry * e, store_client * sc) -{ - STCB *callback = sc->callback; - MemObject *mem = e->mem_obj; - size_t sz; - static int loopdetect = 0; - assert(++loopdetect < 10); - debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->key)); - assert(callback != NULL); - if (e->store_status == STORE_ABORTED) { -#if USE_ASYNC_IO - if (sc->disk_op_in_progress == 1) { - if (sc->swapin_fd >= 0) - aioCancel(sc->swapin_fd, NULL); - else - aioCancel(-1, sc); - sc->disk_op_in_progress = 0; - } -#endif - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, 0); - } else if (e->store_status == STORE_OK && sc->copy_offset == e->object_len) { - /* There is no more to send! */ -#if USE_ASYNC_IO - if (sc->disk_op_in_progress == 1) { - if (sc->swapin_fd >= 0) - aioCancel(sc->swapin_fd, NULL); - else - aioCancel(-1, sc); - sc->disk_op_in_progress = 0; - } -#endif - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, 0); - } else if (e->store_status == STORE_PENDING && sc->seen_offset == mem->inmem_hi) { - /* client has already seen this, wait for more */ - debug(20, 3) ("storeClientCopy2: Waiting for more\n"); - } else if (sc->copy_offset >= mem->inmem_lo && mem->inmem_lo < mem->inmem_hi) { - /* What the client wants is in memory */ - debug(20, 3) ("storeClientCopy2: Copying from memory\n"); - sz = stmemCopy(mem->data, sc->copy_offset, sc->copy_buf, sc->copy_size); -#if USE_ASYNC_IO - if (sc->disk_op_in_progress == 1) { - if (sc->swapin_fd >= 0) - aioCancel(sc->swapin_fd, NULL); - else - aioCancel(-1, sc); - sc->disk_op_in_progress = 0; - } -#endif - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, sz); - } else if (sc->swapin_fd < 0) { - debug(20, 3) ("storeClientCopy2: Need to open swap in file\n"); - assert(sc->type == STORE_DISK_CLIENT); - /* gotta open the swapin file */ - /* assert(sc->copy_offset == 0); */ - if (sc->disk_op_in_progress == 0) { - sc->disk_op_in_progress = 1; - storeSwapInStart(e, storeClientCopyFileOpened, sc); - } else { - debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); - } - } else { - debug(20, 3) ("storeClientCopy: reading from disk FD %d\n", - sc->swapin_fd); - assert(sc->type == STORE_DISK_CLIENT); - if (sc->disk_op_in_progress == 0) { - sc->disk_op_in_progress = 1; - storeClientCopyFileRead(sc); - } else { - debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); - } - } - --loopdetect; -} - -static void -storeClientCopyFileOpened(int fd, void *data) -{ - store_client *sc = data; - STCB *callback = sc->callback; - if (fd < 0) { - debug(20, 3) ("storeClientCopyFileOpened: failed\n"); - sc->disk_op_in_progress = 0; - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, -1); - return; - } - sc->swapin_fd = fd; - storeClientCopyFileRead(sc); -} - -static void -storeClientCopyFileRead(store_client * sc) -{ - assert(sc->callback != NULL); - file_read(sc->swapin_fd, - sc->copy_buf, - sc->copy_size, - sc->copy_offset, - storeClientCopyHandleRead, - sc); -} - -static void -storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, void *data) -{ - store_client *sc = data; - MemObject *mem = sc->mem; - STCB *callback = sc->callback; - int hdr_len = 0; - assert(sc->disk_op_in_progress != 0); - sc->disk_op_in_progress = 0; - assert(sc->callback != NULL); - debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); - if (sc->copy_offset == 0 && len > 0 && mem != NULL) { - hdr_len = storeGetMetaBuf(buf, mem); - memmove((char *) buf, (char *) (buf + hdr_len), len - hdr_len); - len -= hdr_len; - httpParseReplyHeaders(buf, mem->reply); - } - sc->callback = NULL; - callback(sc->callback_data, sc->copy_buf, len); -} - -int -storeClientCopyPending(StoreEntry * e, void *data) -{ - /* return 1 if there is a callback registered for this client */ - store_client *sc = storeClientListSearch(e->mem_obj, data); - if (sc == NULL) - return 0; - if (sc->callback == NULL) - return 0; - return 1; -} - static int storeEntryValidLength(const StoreEntry * e) { @@ -2277,27 +1107,6 @@ storeWriteCleanLogs(int reopen) } #undef CLEAN_BUF_SZ -int -storePendingNClients(const StoreEntry * e) -{ - int npend = 0; - MemObject *mem = e->mem_obj; - store_client *sc; - store_client *nx = NULL; - if (mem == NULL) - return 0; - for (sc = mem->clients; sc; sc = nx) { - nx = sc->next; - /* Changed from callback_data to just callback. There can be no use */ - /* for callback_data without a callback, and sc->callback we know */ - /* gets reset, but not necessarily sc->callback_data */ - if (sc->callback == NULL) - continue; - npend++; - } - return npend; -} - void storeRotateLog(void) { @@ -2471,7 +1280,7 @@ storeTimestampsSet(StoreEntry * entry) static int fileno_stack[FILENO_STACK_SIZE]; #if !MONOTONIC_STORE -static int +int storeGetUnusedFileno(void) { int fn; @@ -2483,7 +1292,7 @@ storeGetUnusedFileno(void) return fn; } -static void +void storePutUnusedFileno(StoreEntry * e) { assert(storeDirMapBitTest(e->swap_file_number)); @@ -2556,7 +1365,7 @@ storeMemObjectDump(MemObject * mem) checkNullString(mem->log_url)); } -static void +void storeEntryDump(StoreEntry * e) { debug(20, 1) ("StoreEntry->key: %s\n", storeKeyText(e->key)); @@ -2578,7 +1387,7 @@ storeEntryDump(StoreEntry * e) } /* NOTE, this function assumes only two mem states */ -static void +void storeSetMemStatus(StoreEntry * e, int new_status) { MemObject *mem = e->mem_obj; @@ -2596,20 +1405,6 @@ storeSetMemStatus(StoreEntry * e, int new_status) e->mem_status = new_status; } -static void -storeSwapOutFileClose(StoreEntry * e) -{ - MemObject *mem = e->mem_obj; - if (mem->swapout.fd > -1) - file_close(mem->swapout.fd); -#if USE_ASYNC_IO - else - aioCancel(-1, e); /* Make doubly certain pending ops are gone */ -#endif - mem->swapout.fd = -1; - storeUnlockObject(e); -} - const char * storeUrl(const StoreEntry * e) { @@ -2659,437 +1454,3 @@ storeBufferFlush(StoreEntry * e) InvokeHandlers(e); } - - -static int -storeGetNextFile(int *sfileno, int *size) -{ - static int dirn, curlvl1, curlvl2, flag, done, in_dir, fn; - static struct dirent *entry; - static DIR *td; - int fd = 0, used = 0; - LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, fullpath, SQUID_MAXPATHLEN); - - - debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", flag, - dirn, curlvl1, curlvl2); - - if (done) - return -2; - - while (!fd && !done) { - fd = 0; - if (!flag) { /* initialize, open first file */ - done = dirn = curlvl1 = curlvl2 = in_dir = 0; - flag = 1; - assert(Config.cacheSwap.n_configured > 0); - } - if (!in_dir) { /* we need to read in a new directory */ - snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", - Config.cacheSwap.swapDirs[dirn].path, - curlvl1, curlvl2); - if (flag && td) - closedir(td); - td = opendir(fullpath); - entry = readdir(td); /* skip . and .. */ - entry = readdir(td); - if (errno == ENOENT) { - debug(20, 3) ("storeGetNextFile: directory does not exist!.\n"); - } - debug(20, 3) ("storeGetNextFile: Directory %s/%02X/%02X\n", - Config.cacheSwap.swapDirs[dirn].path, - curlvl1, curlvl2); - } - if ((entry = readdir(td))) { - in_dir++; - if (sscanf(entry->d_name, "%x", sfileno) != 1) { - debug(20, 3) ("storeGetNextFile: invalid %s\n", - entry->d_name); - continue; - } - fn = *sfileno; - fn = storeDirProperFileno(dirn, fn); - *sfileno = fn; - used = storeDirMapBitTest(fn); - if (used) { - debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n"); - continue; - } - snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", - fullpath, entry->d_name); - debug(20, 3) ("storeGetNextFile: Opening %s\n", fullfilename); - fd = file_open(fullfilename, O_RDONLY, NULL, NULL, NULL); - continue; - } -#if 0 - else if (!in_dir) - debug(20, 3) ("storeGetNextFile: empty dir.\n"); -#endif - - in_dir = 0; - - if ((curlvl2 = (curlvl2 + 1) % Config.cacheSwap.swapDirs[dirn].l2)) - continue; - if ((curlvl1 = (curlvl1 + 1) % Config.cacheSwap.swapDirs[dirn].l1)) - continue; - if ((dirn = (dirn + 1) % Config.cacheSwap.n_configured)) - continue; - else - done = 1; - - } - return fd; -} - -static void -storeDoRebuildFromSwapFiles(void *data) -{ - struct storeRebuildState *RB = data; - LOCAL_ARRAY(char, hdr_buf, 2 * MAX_URL); - LOCAL_ARRAY(cache_key, keybuf, MAX_URL); - StoreEntry *e = NULL; - StoreEntry tmpe; - int sfileno = 0; - int count; - int size; - int x; - struct _rebuild_dir *d = RB->rebuild_dir; - struct stat fst; - static int filecount; - int hdr_len = 0; - int myt, myl; - int fd = 0; - debug(20, 3) (" Starting StoreRebuildFromSwapFiles at speed %d\n", d->speed); - - for (count = 0; count < d->speed; count++) { - if (fd) - file_close(fd); - fd = storeGetNextFile(&sfileno, &size); - if (fd == -2) { - debug(20, 1) ("StoreRebuildFromSwapFiles: done!\n"); - store_rebuilding = 0; - return; - } else if (fd == 0) { - continue; - } - assert(fd > 0); - - /* lets get file stats here */ - - x = fstat(fd, &fst); - assert(x == 0); - - if ((++filecount & 0x3FFF) == 0) - debug(20, 1) (" %7d objects read so far.\n", RB->linecount); - - debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); - - x = read(fd, hdr_buf, 4096); - if (x < SWAP_META_TLD_SIZE) { - debug(20, 1) (" Error reading header %s, small file, removing (read %d) %s\n", - xstrerror(), x, hdr_buf); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - if (SwapMetaType(hdr_buf) != META_OK) { - debug(20, 1) (" Found an old-style object or an invalid one\n"); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - xmemcpy(&hdr_len, SwapMetaSize(hdr_buf), sizeof(int)); - if (x < hdr_len) { - debug(20, 1) (" Error header size > x (%d)%d\n", hdr_len, x); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - debug(20, 3) (" header size %d\n", hdr_len); - - /* get key */ - - if (0 == getSwapHdr(&myt, &myl, keybuf, hdr_buf, hdr_len)) { - debug(20, 1) ("Error getting SWAP_META_KEY %d\n", x); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - keybuf[myl] = '\0'; - - debug(20, 3) (" hm, we have %s, %d, %d\n", keybuf, myt, myl); - - if (keybuf[0] == '\0' || myt != SWAP_META_KEY) { - debug(20, 1) ("storeDoRebuildFromSwapFiles: bad key\n"); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - /* get the standard meta data for the StoreEntry */ - - memset(&tmpe, '\0', sizeof(StoreEntry)); - if (0 == getSwapHdr(&myt, &myl, &tmpe.timestamp, hdr_buf, hdr_len)) { - debug(20, 1) ("storeDoRebuildFromSwapFiles:Error getting SWAP_META_STD %d\n", myl); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - - continue; - } - assert(myt == SWAP_META_STD); - assert(myl == HDR_METASIZE); - storeEntryDump(&tmpe); - - /* check sizes */ - - if (hdr_len + tmpe.object_len != fst.st_size) { - debug(20, 1) ("storeDoRebuildFromSwapFiles:INVALID swapfile, sizes dont match %d+%d!=%d\n", - hdr_len, tmpe.object_len, fst.st_size); - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - continue; - } - if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) { - safeunlink(storeSwapFullPath(sfileno, NULL), 1); - RB->badflags++; - continue; - } - if ((e = storeGet(keybuf)) != NULL) { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - storeRelease(e); /* release old entry */ - RB->dupcount++; - } - /* update store_swap_size */ - RB->objcount++; - debug(20, 4) ("storeDoRebuildFromSwapFiles: KEY=%20s , sfileno=%08X exp=%08X timest=%08X\n", - keybuf, sfileno, tmpe.expires, tmpe.timestamp); - debug(20, 4) (" lastref=%08X lastmod=%08X refcount=%08X flag=%08X\n", - tmpe.lastref, tmpe.lastmod, tmpe.refcount, tmpe.flag); - debug(20, 4) (" len=%d hdr_len=%d file_len=%d\n", tmpe.object_len, - hdr_len, fst.st_size); - - e = storeAddDiskRestore(keybuf, - sfileno, - (int) tmpe.object_len, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastmod, - (u_num32) tmpe.refcount, /* refcount */ - (u_num32) tmpe.flag, /* flags */ - d->clean); - } - eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); -} - - -/* build swapfile header */ -static int -storeBuildMetaData(StoreEntry * e, char *swap_buf_c) -{ - MemObject *mem; - int keylength; - int a = SWAP_META_TLD_START; - char *meta_buf; - - mem = e->mem_obj; - meta_buf = mem->swapout.meta_buf; - - debug(20, 3) ("storeBuildSwapFileHeader: called.\n"); - assert(e->swap_status == SWAPOUT_WRITING); - - if (!meta_buf) - meta_buf = mem->swapout.meta_buf = xmalloc(1024); - -/* construct header */ - - /* add Length(int)-Type(char)-Data encoded info */ - - if (squid_key_size < 0) - keylength = strlen(e->key); - else - keylength = squid_key_size; - - meta_buf[0] = META_OK; - xmemcpy(&meta_buf[1], &a, sizeof(int)); - mem->swapout.meta_len = SWAP_META_TLD_START; - - addSwapHdr(SWAP_META_KEY, keylength, (void *) e->key, - mem->swapout.meta_buf, &mem->swapout.meta_len); - addSwapHdr(SWAP_META_STD, HDR_METASIZE, (void *) &e->timestamp, - mem->swapout.meta_buf, &mem->swapout.meta_len); - debug(20, 3) ("storeBuildSwapFileHeader: len=%d.\n", mem->swapout.meta_len); - - if (swap_buf_c) - xmemcpy(swap_buf_c, mem->swapout.meta_buf, mem->swapout.meta_len); - return mem->swapout.meta_len; -} - - -static int -getSwapHdr(int *type, int *len, void *dst, char *write_buf, int hdr_len) -{ - static int cur = 0; - static char *curptr; - char *tmp_buf; - - if (cur == 0 || curptr != write_buf) { /* first call or rewind ! */ - cur = SWAP_META_TLD_START; - curptr = write_buf; - } - if (cur + SWAP_META_TLD_START > hdr_len) { - debug(20, 3) ("getSwapHdr: overflow, %d %d.\n", cur, hdr_len); - cur = 0; - return -1; - } - tmp_buf = &write_buf[cur]; /* position ourselves */ - - xmemcpy(len, SwapMetaSize(tmp_buf), sizeof(int)); /* length */ - *type = SwapMetaType(tmp_buf); /* type */ - xmemcpy(dst, SwapMetaData(tmp_buf), *len); /* data */ - - cur += SWAP_META_TLD_START + *len; /* advance position */ - - debug(20, 4) ("getSwapHdr: t=%d l=%d (cur=%d hdr_len=%d) (%p)\n", - *type, *len, cur, hdr_len, dst); - if (cur == hdr_len) { - debug(20, 4) ("getSwapHdr: finished with this.\n"); - cur = 0; - return 1; - } - return 1; /* ok ! */ -} - - -static void -addSwapHdr(int type, int len, void *src, char *write_buf, int *write_len) -{ - int hdr_len = *write_len; - char *base = &write_buf[hdr_len]; - debug(20, 3) ("addSwapHdr: at %d\n", hdr_len); - - base[0] = (char) type; - xmemcpy(&base[1], &len, sizeof(int)); - xmemcpy(SwapMetaData(base), src, len); - - hdr_len += SWAP_META_TLD_START + len; - - /* now we know length */ - - debug(20, 3) ("addSwapHdr: added type=%d len=%d data=%p. hdr_len=%d\n", - type, len, src, hdr_len); - - /* update header */ - xmemcpy(&write_buf[1], &hdr_len, sizeof(int)); - *write_len = hdr_len; -} - -static int -storeGetMetaBuf(const char *buf, MemObject * mem) -{ - int hdr_len; - - assert(mem != NULL); - - /* the key */ - if (SwapMetaType(buf) != META_OK) { - debug(20, 1) ("storeGetMetaBuf:Found an old-style object, damn.\n"); - return -1; - } - xmemcpy(&hdr_len, SwapMetaSize(buf), sizeof(int)); - mem->swapout.meta_len = hdr_len; - mem->swapout.meta_buf = xmalloc(hdr_len); - xmemcpy(mem->swapout.meta_buf, buf, hdr_len); - - debug(20, 3) (" header size %d\n", hdr_len); - - return hdr_len; -} - -#if 0 -static int -storeParseMetaBuf(StoreEntry * e) -{ - static char mbuf[1024]; - int myt, myl; - MemObject *mem = e->mem_obj; - - assert(e && e->mem_obj && e->key); - getSwapHdr(&myt, &myl, mbuf, mem->swapout.meta_buf, mem->swapout.meta_len); - mbuf[myl] = 0; - debug(20, 3) ("storeParseMetaBuf: key=%s\n", mbuf); - e->key = xstrdup(storeKeyScan(mbuf)); - getSwapHdr(&myt, &myl, &e->timestamp, mem->swapout.meta_buf, mem->swapout.meta_len); - - return 1; -} -#endif - -static void -storeConvert(void) -{ - int i; - struct storeRebuildState *RB; - struct _rebuild_dir *d; - FILE *fp; - int clean; - RB = xcalloc(1, sizeof(struct storeRebuildState)); - RB->start = squid_curtime; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - fp = storeDirOpenTmpSwapLog(i, &clean); - if (fp == NULL) - continue; - d = xcalloc(1, sizeof(struct _rebuild_dir)); - d->dirn = i; - d->log = fp; - d->clean = clean; - d->speed = 1 << 30; - d->next = RB->rebuild_dir; - RB->rebuild_dir = d; - if (!clean) - RB->need_to_validate = 1; - debug(20, 1) ("Converting storage in Cache Dir #%d (%s)\n", - i, clean ? "CLEAN" : "DIRTY"); - } - RB->line_in_sz = 4096; - RB->line_in = xcalloc(1, RB->line_in_sz); - storeDoConvertFromLog(RB); -} - -static void -storeConvertFile(const cache_key * key, - int file_number, - int size, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_num32 flags, - int clean) -{ - int fd_r, fd_w; - int hdr_len, x, y; - LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); - LOCAL_ARRAY(char, copybuf, SWAP_BUF); - StoreEntry e; - e.key = key; - e.object_len = size; - e.expires = expires; - e.lastref = lastref; - e.refcount = refcount; - e.flag = flags; - - - storeSwapFullPath(file_number, swapfilename); - fd_r = open(swapfilename, O_RDONLY); - if (fd_r < 0) { /* ERROR */ - - return; - } - safeunlink(swapfilename, 1); - fd_w = open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC); - - hdr_len = storeBuildMetaData(&e, copybuf); - x = write(fd_w, copybuf, hdr_len); - while (x > 0) { - y = read(fd_r, copybuf, SWAP_BUF); - x = write(fd_w, copybuf, y); - } - close(fd_r); - close(fd_w); -} diff --git a/src/store_client.cc b/src/store_client.cc new file mode 100644 index 0000000000..46c4536bfa --- /dev/null +++ b/src/store_client.cc @@ -0,0 +1,333 @@ +#include "squid.h" + +static void storeClientCopy2(StoreEntry * e, store_client * sc); +static SIH storeClientCopyFileOpened; +static void storeClientCopyFileRead(store_client * sc); +static DRCB storeClientCopyHandleRead; + +/* check if there is any client waiting for this object at all */ +/* return 1 if there is at least one client */ +int +storeClientWaiting(const StoreEntry * e) +{ + MemObject *mem = e->mem_obj; + store_client *sc; + for (sc = mem->clients; sc; sc = sc->next) { + if (sc->callback_data != NULL) + return 1; + } + return 0; +} + +store_client * +storeClientListSearch(const MemObject * mem, void *data) +{ + store_client *sc; + for (sc = mem->clients; sc; sc = sc->next) { + if (sc->callback_data == data) + break; + } + return sc; +} + +/* add client with fd to client list */ +void +storeClientListAdd(StoreEntry * e, void *data) +{ + MemObject *mem = e->mem_obj; + store_client **T; + store_client *sc; + assert(mem); + if (storeClientListSearch(mem, data) != NULL) + return; + mem->nclients++; + sc = memAllocate(MEM_STORE_CLIENT, 1); + cbdataAdd(sc, MEM_STORE_CLIENT); /* sc is callback_data for file_read */ + sc->callback_data = data; + sc->seen_offset = 0; + sc->copy_offset = 0; + sc->swapin_fd = -1; + sc->disk_op_in_progress = 0; + sc->mem = mem; + if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) + sc->type = STORE_MEM_CLIENT; + else + sc->type = STORE_DISK_CLIENT; + for (T = &mem->clients; *T; T = &(*T)->next); + *T = sc; +} + +/* copy bytes requested by the client */ +void +storeClientCopy(StoreEntry * e, + off_t seen_offset, + off_t copy_offset, + size_t size, + char *buf, + STCB * callback, + void *data) +{ + store_client *sc; + static int recurse_detect = 0; + assert(e->store_status != STORE_ABORTED); + assert(recurse_detect < 3); /* could == 1 for IMS not modified's */ + debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", + storeKeyText(e->key), + (int) seen_offset, + (int) copy_offset, + (int) size, + callback, + data); + sc = storeClientListSearch(e->mem_obj, data); + assert(sc != NULL); + assert(sc->callback == NULL); + sc->copy_offset = copy_offset; + sc->seen_offset = seen_offset; + sc->callback = callback; + sc->copy_buf = buf; + sc->copy_size = size; + sc->copy_offset = copy_offset; + storeClientCopy2(e, sc); + recurse_detect--; +} + +static void +storeClientCopy2(StoreEntry * e, store_client * sc) +{ + STCB *callback = sc->callback; + MemObject *mem = e->mem_obj; + size_t sz; + static int loopdetect = 0; + assert(++loopdetect < 10); + debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->key)); + assert(callback != NULL); + if (e->store_status == STORE_ABORTED) { +#if USE_ASYNC_IO + if (sc->disk_op_in_progress == 1) { + if (sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + } +#endif + sc->disk_op_in_progress = 0; + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, 0); + } else if (e->store_status == STORE_OK && sc->copy_offset == e->object_len) { + /* There is no more to send! */ +#if USE_ASYNC_IO + if (sc->disk_op_in_progress == 1) { + if (sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + } +#endif + sc->disk_op_in_progress = 0; + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, 0); + } else if (e->store_status == STORE_PENDING && sc->seen_offset == mem->inmem_hi) { + /* client has already seen this, wait for more */ + debug(20, 3) ("storeClientCopy2: Waiting for more\n"); + } else if (sc->copy_offset >= mem->inmem_lo && mem->inmem_lo < mem->inmem_hi) { + /* What the client wants is in memory */ + debug(20, 3) ("storeClientCopy2: Copying from memory\n"); + sz = stmemCopy(mem->data, sc->copy_offset, sc->copy_buf, sc->copy_size); +#if USE_ASYNC_IO + if (sc->disk_op_in_progress == 1) { + if (sc->swapin_fd >= 0) + aioCancel(sc->swapin_fd, NULL); + else + aioCancel(-1, sc); + } +#endif + sc->disk_op_in_progress = 0; + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, sz); + } else if (sc->swapin_fd < 0) { + debug(20, 3) ("storeClientCopy2: Need to open swap in file\n"); + assert(sc->type == STORE_DISK_CLIENT); + /* gotta open the swapin file */ + /* assert(sc->copy_offset == 0); */ + if (sc->disk_op_in_progress == 0) { + sc->disk_op_in_progress = 1; + storeSwapInStart(e, storeClientCopyFileOpened, sc); + } else { + debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); + } + } else { + debug(20, 3) ("storeClientCopy: reading from disk FD %d\n", + sc->swapin_fd); + assert(sc->type == STORE_DISK_CLIENT); + if (sc->disk_op_in_progress == 0) { + sc->disk_op_in_progress = 1; + storeClientCopyFileRead(sc); + } else { + debug(20, 2) ("storeClientCopy2: Averted multiple fd operation\n"); + } + } + --loopdetect; +} + +static void +storeClientCopyFileOpened(int fd, void *data) +{ + store_client *sc = data; + STCB *callback = sc->callback; + if (fd < 0) { + debug(20, 3) ("storeClientCopyFileOpened: failed\n"); + sc->disk_op_in_progress = 0; + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, -1); + return; + } + sc->swapin_fd = fd; + storeClientCopyFileRead(sc); +} + +static void +storeClientCopyFileRead(store_client * sc) +{ + assert(sc->callback != NULL); + file_read(sc->swapin_fd, + sc->copy_buf, + sc->copy_size, + sc->copy_offset, + storeClientCopyHandleRead, + sc); +} + +static void +storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, void *data) +{ + store_client *sc = data; + MemObject *mem = sc->mem; + STCB *callback = sc->callback; + int hdr_len = 0; + assert(sc->disk_op_in_progress != 0); + sc->disk_op_in_progress = 0; + assert(sc->callback != NULL); + debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); + if (sc->copy_offset == 0 && len > 0 && mem != NULL) { + hdr_len = storeGetMetaBuf(buf, mem); + memmove((char *) buf, (char *) (buf + hdr_len), len - hdr_len); + len -= hdr_len; + httpParseReplyHeaders(buf, mem->reply); + } + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, len); +} + +int +storeClientCopyPending(StoreEntry * e, void *data) +{ + /* return 1 if there is a callback registered for this client */ + store_client *sc = storeClientListSearch(e->mem_obj, data); + if (sc == NULL) + return 0; + if (sc->callback == NULL) + return 0; + return 1; +} + +int +storeUnregister(StoreEntry * e, void *data) +{ + MemObject *mem = e->mem_obj; + store_client *sc; + store_client **S; + STCB *callback; + if (mem == NULL) + return 0; + debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyText(e->key)); + for (S = &mem->clients; (sc = *S) != NULL; S = &(*S)->next) { + if (sc->callback_data == data) + break; + } + if (sc == NULL) + return 0; + *S = sc->next; + mem->nclients--; + sc->disk_op_in_progress = 0; + if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) + storeCheckSwapOut(e); + if (sc->swapin_fd > -1) { + commSetSelect(sc->swapin_fd, COMM_SELECT_READ, NULL, NULL, 0); + file_close(sc->swapin_fd); + } +#if USE_ASYNC_IO + else + aioCancel(-1, sc); +#endif + if ((callback = sc->callback) != NULL) { + /* callback with ssize = -1 to indicate unexpected termination */ + debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", + mem->url); + sc->callback = NULL; + callback(sc->callback_data, sc->copy_buf, -1); + } + cbdataFree(sc); + return 1; +} + +off_t +storeLowestMemReaderOffset(const StoreEntry * entry) +{ + const MemObject *mem = entry->mem_obj; + off_t lowest = mem->inmem_hi; + store_client *sc; + store_client *nx = NULL; + for (sc = mem->clients; sc; sc = nx) { + nx = sc->next; + if (sc->callback_data == NULL) /* open slot */ + continue; + if (sc->type != STORE_MEM_CLIENT) + continue; + if (sc->copy_offset < lowest) + lowest = sc->copy_offset; + } + return lowest; +} + +/* Call handlers waiting for data to be appended to E. */ +void +InvokeHandlers(StoreEntry * e) +{ + int i = 0; + MemObject *mem = e->mem_obj; + store_client *sc; + store_client *nx = NULL; + assert(mem->clients != NULL || mem->nclients == 0); + debug(20, 3) ("InvokeHandlers: %s\n", storeKeyText(e->key)); + /* walk the entire list looking for valid callbacks */ + for (sc = mem->clients; sc; sc = nx) { + nx = sc->next; + debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); + if (sc->callback_data == NULL) + continue; + if (sc->callback == NULL) + continue; + storeClientCopy2(e, sc); + } +} + +int +storePendingNClients(const StoreEntry * e) +{ + int npend = 0; + MemObject *mem = e->mem_obj; + store_client *sc; + store_client *nx = NULL; + if (mem == NULL) + return 0; + for (sc = mem->clients; sc; sc = nx) { + nx = sc->next; + /* Changed from callback_data to just callback. There can be no use */ + /* for callback_data without a callback, and sc->callback we know */ + /* gets reset, but not necessarily sc->callback_data */ + if (sc->callback == NULL) + continue; + npend++; + } + return npend; +} diff --git a/src/store_key_md5.cc b/src/store_key_md5.cc index 7bdc6417ca..28a7739dde 100644 --- a/src/store_key_md5.cc +++ b/src/store_key_md5.cc @@ -1,7 +1,5 @@ #include "squid.h" -#if STORE_KEY_MD5 - const char * storeKeyText(const unsigned char *key) { @@ -120,5 +118,3 @@ storeKeyHashBuckets(int nobj) return 0x8000; return 0x10000; } - -#endif /* STORE_KEY_MD5 */ diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc new file mode 100644 index 0000000000..b235051607 --- /dev/null +++ b/src/store_rebuild.cc @@ -0,0 +1,683 @@ +#include "squid.h" + +#define STORE_META_BUFSZ 4096 + +struct storeRebuildState { + struct _rebuild_dir { + int dirn; + FILE *log; + int speed; + int clean; + struct _rebuild_dir *next; + } *rebuild_dir; + int objcount; /* # objects successfully reloaded */ + int expcount; /* # objects expired */ + int linecount; /* # lines parsed from cache logfile */ + int clashcount; /* # swapfile clashes avoided */ + int cancelcount; /* # objects cancelled */ + int dupcount; /* # duplicates purged */ + int invalid; /* # bad lines */ + int badflags; /* # bad e->flags */ + int need_to_validate; + time_t start; + time_t stop; + char *line_in; + size_t line_in_sz; +}; + +typedef struct valid_ctrl_t { + struct stat *sb; + StoreEntry *e; + STVLDCB *callback; + void *callback_data; +} valid_ctrl_t; + +static void storeRebuiltFromDisk(struct storeRebuildState *data); + +void +storeDoRebuildFromSwapFiles(void *data) +{ + struct storeRebuildState *RB = data; + LOCAL_ARRAY(char, hdr_buf, 2 * MAX_URL); + LOCAL_ARRAY(cache_key, keybuf, MAX_URL); + StoreEntry *e = NULL; + StoreEntry tmpe; + int sfileno = 0; + int count; + int size; + int x; + struct _rebuild_dir *d = RB->rebuild_dir; + struct stat fst; + static int filecount; + int hdr_len = 0; + int myt, myl; + int fd = 0; + debug(20, 3) (" Starting StoreRebuildFromSwapFiles at speed %d\n", d->speed); + for (count = 0; count < d->speed; count++) { + if (fd) + file_close(fd); + fd = storeGetNextFile(&sfileno, &size); + if (fd == -2) { + debug(20, 1) ("StoreRebuildFromSwapFiles: done!\n"); + store_rebuilding = 0; + return; + } else if (fd == 0) { + continue; + } + assert(fd > 0); + /* lets get file stats here */ + x = fstat(fd, &fst); + assert(x == 0); + if ((++filecount & 0x3FFF) == 0) + debug(20, 1) (" %7d objects read so far.\n", RB->linecount); + debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); + x = read(fd, hdr_buf, 4096); + if (x < STORE_META_TLD_SIZE) { + debug(20, 1) (" Error reading header %s, small file, removing (read %d) %s\n", + xstrerror(), x, hdr_buf); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + if (SwapMetaType(hdr_buf) != META_OK) { + debug(20, 1) (" Found an old-style object or an invalid one\n"); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + xmemcpy(&hdr_len, SwapMetaSize(hdr_buf), sizeof(int)); + if (x < hdr_len) { + debug(20, 1) (" Error header size > x (%d)%d\n", hdr_len, x); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + debug(20, 3) (" header size %d\n", hdr_len); + /* get key */ + if (0 == getSwapHdr(&myt, &myl, keybuf, hdr_buf, hdr_len)) { + debug(20, 1) ("Error getting STORE_META_KEY %d\n", x); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + keybuf[myl] = '\0'; + debug(20, 3) (" hm, we have %s, %d, %d\n", keybuf, myt, myl); + if (keybuf[0] == '\0' || myt != STORE_META_KEY) { + debug(20, 1) ("storeDoRebuildFromSwapFiles: bad key\n"); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + /* get the standard meta data for the StoreEntry */ + memset(&tmpe, '\0', sizeof(StoreEntry)); + if (0 == getSwapHdr(&myt, &myl, &tmpe.timestamp, hdr_buf, hdr_len)) { + debug(20, 1) ("storeDoRebuildFromSwapFiles:Error getting STORE_META_STD %d\n", myl); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + assert(myt == STORE_META_STD); + assert(myl == STORE_HDR_METASIZE); + /* check sizes */ + if (hdr_len + tmpe.object_len != fst.st_size) { + debug(20, 1) ("storeDoRebuildFromSwapFiles:INVALID swapfile, sizes dont match %d+%d!=%d\n", + hdr_len, tmpe.object_len, fst.st_size); + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + continue; + } + if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) { + safeunlink(storeSwapFullPath(sfileno, NULL), 1); + RB->badflags++; + continue; + } + if ((e = storeGet(keybuf)) != NULL) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + RB->dupcount++; + } + /* update store_swap_size */ + RB->objcount++; + debug(20, 4) ("storeDoRebuildFromSwapFiles: KEY=%20s , sfileno=%08X exp=%08X timest=%08X\n", + keybuf, sfileno, tmpe.expires, tmpe.timestamp); + debug(20, 4) (" lastref=%08X lastmod=%08X refcount=%08X flag=%08X\n", + tmpe.lastref, tmpe.lastmod, tmpe.refcount, tmpe.flag); + debug(20, 4) (" len=%d hdr_len=%d file_len=%d\n", tmpe.object_len, + hdr_len, fst.st_size); + e = storeAddDiskRestore(keybuf, + sfileno, + (int) tmpe.object_len, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastmod, + (u_num32) tmpe.refcount, /* refcount */ + (u_num32) tmpe.flag, /* flags */ + d->clean); + } + eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); +} + + +void +storeConvert(void) +{ + int i; + struct storeRebuildState *RB; + struct _rebuild_dir *d; + FILE *fp; + int clean; + RB = xcalloc(1, sizeof(struct storeRebuildState)); + RB->start = squid_curtime; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + fp = storeDirOpenTmpSwapLog(i, &clean); + if (fp == NULL) + continue; + d = xcalloc(1, sizeof(struct _rebuild_dir)); + d->dirn = i; + d->log = fp; + d->clean = clean; + d->speed = 1 << 30; + d->next = RB->rebuild_dir; + RB->rebuild_dir = d; + if (!clean) + RB->need_to_validate = 1; + debug(20, 1) ("Converting storage in Cache Dir #%d (%s)\n", + i, clean ? "CLEAN" : "DIRTY"); + } + RB->line_in_sz = 4096; + RB->line_in = xcalloc(1, RB->line_in_sz); + storeDoConvertFromLog(RB); +} + +void +storeConvertFile(const cache_key * key, + int file_number, + int size, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean) +{ + int fd_r, fd_w; + int hdr_len, x, y; + LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, copybuf, STORE_META_BUFSZ); + StoreEntry e; + e.key = key; + e.object_len = size; + e.expires = expires; + e.lastref = lastref; + e.refcount = refcount; + e.flag = flags; + storeSwapFullPath(file_number, swapfilename); + fd_r = open(swapfilename, O_RDONLY); + if (fd_r < 0) { /* ERROR */ + return; + } + safeunlink(swapfilename, 1); + fd_w = open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC); + hdr_len = storeBuildMetaData(&e, copybuf); + assert(hdr_len < STORE_META_BUFSZ); + x = write(fd_w, copybuf, hdr_len); + while (x > 0) { + y = read(fd_r, copybuf, STORE_META_BUFSZ); + x = write(fd_w, copybuf, y); + } + close(fd_r); + close(fd_w); +} + +int +storeGetNextFile(int *sfileno, int *size) +{ + static int dirn, curlvl1, curlvl2, flag, done, in_dir, fn; + static struct dirent *entry; + static DIR *td; + int fd = 0, used = 0; + LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); + LOCAL_ARRAY(char, fullpath, SQUID_MAXPATHLEN); + debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", flag, + dirn, curlvl1, curlvl2); + if (done) + return -2; + while (!fd && !done) { + fd = 0; + if (!flag) { /* initialize, open first file */ + done = dirn = curlvl1 = curlvl2 = in_dir = 0; + flag = 1; + assert(Config.cacheSwap.n_configured > 0); + } + if (!in_dir) { /* we need to read in a new directory */ + snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", + Config.cacheSwap.swapDirs[dirn].path, + curlvl1, curlvl2); + if (flag && td) + closedir(td); + td = opendir(fullpath); + entry = readdir(td); /* skip . and .. */ + entry = readdir(td); + if (errno == ENOENT) { + debug(20, 3) ("storeGetNextFile: directory does not exist!.\n"); + } + debug(20, 3) ("storeGetNextFile: Directory %s/%02X/%02X\n", + Config.cacheSwap.swapDirs[dirn].path, + curlvl1, curlvl2); + } + if ((entry = readdir(td))) { + in_dir++; + if (sscanf(entry->d_name, "%x", sfileno) != 1) { + debug(20, 3) ("storeGetNextFile: invalid %s\n", + entry->d_name); + continue; + } + fn = *sfileno; + fn = storeDirProperFileno(dirn, fn); + *sfileno = fn; + used = storeDirMapBitTest(fn); + if (used) { + debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n"); + continue; + } + snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", + fullpath, entry->d_name); + debug(20, 3) ("storeGetNextFile: Opening %s\n", fullfilename); + fd = file_open(fullfilename, O_RDONLY, NULL, NULL, NULL); + continue; + } +#if 0 + else if (!in_dir) + debug(20, 3) ("storeGetNextFile: empty dir.\n"); +#endif + in_dir = 0; + if ((curlvl2 = (curlvl2 + 1) % Config.cacheSwap.swapDirs[dirn].l2)) + continue; + if ((curlvl1 = (curlvl1 + 1) % Config.cacheSwap.swapDirs[dirn].l1)) + continue; + if ((dirn = (dirn + 1) % Config.cacheSwap.n_configured)) + continue; + else + done = 1; + } + return fd; +} + +/* Add a new object to the cache with empty memory copy and pointer to disk + * use to rebuild store from disk. */ +StoreEntry * +storeAddDiskRestore(const cache_key * key, + int file_number, + int size, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean) +{ + StoreEntry *e = NULL; + debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); + /* if you call this you'd better be sure file_number is not + * already in use! */ + e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); + storeHashInsert(e, key); + e->store_status = STORE_OK; + storeSetMemStatus(e, NOT_IN_MEMORY); + e->swap_status = SWAPOUT_DONE; + e->swap_file_number = file_number; + e->object_len = size; + e->lock_count = 0; + e->refcount = 0; + e->lastref = lastref; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->refcount = refcount; + e->flag = flags; + EBIT_SET(e->flag, ENTRY_CACHABLE); + EBIT_CLR(e->flag, RELEASE_REQUEST); + EBIT_CLR(e->flag, KEY_PRIVATE); + e->ping_status = PING_NONE; + EBIT_CLR(e->flag, ENTRY_VALIDATED); + storeDirMapBitSet(e->swap_file_number); + return e; +} + +/* convert storage .. this is the old storeDoRebuildFromDisk() */ + +void +storeDoConvertFromLog(void *data) +{ + struct storeRebuildState *RB = data; + LOCAL_ARRAY(char, swapfile, MAXPATHLEN); + LOCAL_ARRAY(char, keytext, MAX_URL); + StoreEntry *e = NULL; + time_t expires; + time_t timestamp; + time_t lastref; + time_t lastmod; + int scan1; + int scan2; + int scan3; + int scan4; + int scan5; + int scan6; + int scan7; + off_t size; + int sfileno = 0; + int count; + int x; + struct _rebuild_dir *d; + struct _rebuild_dir **D; + int used; /* is swapfile already in use? */ + int newer; /* is the log entry newer than current entry? */ + const cache_key *key; + /* load a number of objects per invocation */ + if ((d = RB->rebuild_dir) == NULL) { + debug(20, 3) ("Done Converting, here are the stats.\n"); + storeRebuiltFromDisk(RB); + return; + } + for (count = 0; count < d->speed; count++) { + if (fgets(RB->line_in, RB->line_in_sz, d->log) == NULL) { + debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); + fclose(d->log); + d->log = NULL; + storeDirCloseTmpSwapLog(d->dirn); + RB->rebuild_dir = d->next; + safe_free(d); + eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); + return; + } + if ((++RB->linecount & 0x3FFF) == 0) + debug(20, 1) (" %7d Lines read so far.\n", RB->linecount); + debug(20, 9) ("line_in: %s", RB->line_in); + if (RB->line_in[0] == '\0') + continue; + if (RB->line_in[0] == '\n') + continue; + if (RB->line_in[0] == '#') + continue; + keytext[0] = '\0'; + sfileno = 0; + scan1 = 0; + scan2 = 0; + scan3 = 0; + scan4 = 0; + scan5 = 0; + scan6 = 0; + scan7 = 0; + x = sscanf(RB->line_in, "%x %x %x %x %x %d %d %x %s", + &sfileno, /* swap_file_number */ + &scan1, /* timestamp */ + &scan2, /* lastref */ + &scan3, /* expires */ + &scan4, /* last modified */ + &scan5, /* size */ + &scan6, /* refcount */ + &scan7, /* flags */ + keytext); /* key */ + if (x < 1) { + RB->invalid++; + continue; + } + if (x != 9) { + RB->invalid++; + continue; + } + timestamp = (time_t) scan1; + lastref = (time_t) scan2; + expires = (time_t) scan3; + lastmod = (time_t) scan4; + size = (off_t) scan5; + if (size < 0) { + if ((key = storeKeyScan(keytext)) == NULL) + continue; + if ((e = storeGet(key)) == NULL) + continue; + if (e->lastref > lastref) + continue; + debug(20, 3) ("storeRebuildFromDisk: Cancelling: '%s'\n", keytext); + storeRelease(e); + RB->objcount--; + RB->cancelcount++; + continue; + } + storeSwapFullPath(sfileno, swapfile); + if (EBIT_TEST(scan7, KEY_PRIVATE)) { + RB->badflags++; + continue; + } + sfileno = storeDirProperFileno(d->dirn, sfileno); + key = storeKeyScan(keytext); + if (key == NULL) { + debug(20, 1) ("storeDoConvertFromLog: bad key: '%s'\n", keytext); + continue; + } + e = storeGet(key); + used = storeDirMapBitTest(sfileno); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + newer = e ? (lastref > e->lastref ? 1 : 0) : 0; + if (used && !newer) { + /* log entry is old, ignore it */ + RB->clashcount++; + continue; + } else if (used && e && e->swap_file_number == sfileno) { + /* swapfile taken, same URL, newer, update meta */ + e->lastref = timestamp; + e->timestamp = timestamp; + e->expires = expires; + e->lastmod = lastmod; + e->flag |= (u_num32) scan6; + e->refcount += (u_num32) scan7; + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", + sfileno); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + /* assert(RB->need_to_validate); */ + RB->clashcount++; + continue; + } else if (e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + RB->dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + RB->objcount++; + storeConvertFile(key, + sfileno, + (int) size, + expires, + timestamp, + lastref, + lastmod, + (u_num32) scan6, /* refcount */ + (u_num32) scan7, /* flags */ + d->clean); +#if 0 + storeDirSwapLog(e); +#endif + } + RB->rebuild_dir = d->next; + for (D = &RB->rebuild_dir; *D; D = &(*D)->next); + *D = d; + d->next = NULL; + eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); +} + +void +storeCleanup(void *datanotused) +{ + static int bucketnum = -1; + static int validnum = 0; + StoreEntry *e; + hash_link *link_ptr = NULL; + if (++bucketnum >= store_hash_buckets) { + debug(20, 1) (" Completed Validation Procedure\n"); + debug(20, 1) (" Validated %d Entries\n", validnum); + debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); + store_rebuilding = 0; + return; + } + link_ptr = hash_get_bucket(store_table, bucketnum); + for (; link_ptr; link_ptr = link_ptr->next) { + e = (StoreEntry *) link_ptr; + if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) + continue; + if (EBIT_TEST(e->flag, RELEASE_REQUEST)) + continue; + EBIT_SET(e->flag, ENTRY_VALIDATED); + /* Only set the file bit if we know its a valid entry */ + /* otherwise, set it in the validation procedure */ + storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + if ((++validnum & 0xFFFF) == 0) + debug(20, 1) (" %7d Entries Validated so far.\n", validnum); + assert(validnum <= memInUse(MEM_STOREENTRY)); + } + eventAdd("storeCleanup", storeCleanup, NULL, 0); +} + +#if OLD_CODE +void +storeCleanupComplete(void *data, int retcode, int errcode) +{ + StoreEntry *e = data; + storeUnlockObject(e); + outvalid--; + if (retcode == -2 && errcode == -2) + return; + if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) + storeRelease(e); +} +#endif + +void +storeValidate(StoreEntry * e, STVLDCB callback, void *callback_data, void *tag) +{ + valid_ctrl_t *ctrlp; + char *path; + struct stat *sb; +#if !USE_ASYNC_IO + int x; +#endif + assert(!EBIT_TEST(e->flag, ENTRY_VALIDATED)); + if (e->swap_file_number < 0) { + EBIT_CLR(e->flag, ENTRY_VALIDATED); + callback(callback_data, 0, 0); + return; + } + path = storeSwapFullPath(e->swap_file_number, NULL); + sb = xmalloc(sizeof(struct stat)); + ctrlp = xmalloc(sizeof(valid_ctrl_t)); + ctrlp->sb = sb; + ctrlp->e = e; + ctrlp->callback = callback; + ctrlp->callback_data = callback_data; +#if USE_ASYNC_IO + aioStat(path, sb, storeValidateComplete, ctrlp, tag); +#else + /* When evaluating the actual arguments in a function call, the order + * in which the arguments and the function expression are evaluated is + * not specified; */ + x = stat(path, sb); + storeValidateComplete(ctrlp, x, errno); +#endif + return; +} + +void +storeValidateComplete(void *data, int retcode, int errcode) +{ + valid_ctrl_t *ctrlp = data; + struct stat *sb = ctrlp->sb; + StoreEntry *e = ctrlp->e; + char *path; + + if (retcode == -2 && errcode == -2) { + xfree(sb); + xfree(ctrlp); + ctrlp->callback(ctrlp->callback_data, retcode, errcode); + return; + } + if (retcode < 0 && errcode == EWOULDBLOCK) { + path = storeSwapFullPath(e->swap_file_number, NULL); + retcode = stat(path, sb); + } + if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { + EBIT_CLR(e->flag, ENTRY_VALIDATED); + } else { + EBIT_SET(e->flag, ENTRY_VALIDATED); + storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + } + errno = errcode; + ctrlp->callback(ctrlp->callback_data, retcode, errcode); + xfree(sb); + xfree(ctrlp); +} + +/* meta data recreated from disk image in swap directory */ +static void +storeRebuiltFromDisk(struct storeRebuildState *data) +{ + time_t r; + time_t stop; + stop = squid_curtime; + r = stop - data->start; + debug(20, 1) ("Finished rebuilding storage from disk image.\n"); + debug(20, 1) (" %7d Lines read from previous logfile.\n", data->linecount); + debug(20, 1) (" %7d Invalid lines.\n", data->invalid); + debug(20, 1) (" %7d With invalid flags.\n", data->badflags); + debug(20, 1) (" %7d Objects loaded.\n", data->objcount); + debug(20, 1) (" %7d Objects expired.\n", data->expcount); + debug(20, 1) (" %7d Objects cancelled.\n", data->cancelcount); + debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); + debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); + debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", + r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); + debug(20, 1) ("Beginning Validation Procedure\n"); + eventAdd("storeCleanup", storeCleanup, NULL, 0); + memFree(MEM_4K_BUF, data->line_in); + safe_free(data); +} + +void +storeStartRebuildFromDisk(void) +{ + struct storeRebuildState *RB; + struct _rebuild_dir *d; + int clean = 1; + RB = xcalloc(1, sizeof(struct storeRebuildState)); + RB->start = squid_curtime; + d = xcalloc(1, sizeof(struct _rebuild_dir)); + d->clean = clean; + d->speed = opt_foreground_rebuild ? 1 << 30 : 50; + RB->rebuild_dir = d; + if (!clean) + RB->need_to_validate = 1; + debug(20, 1) ("Rebuilding storage (%s)\n", + clean ? "CLEAN" : "DIRTY"); + if (opt_foreground_rebuild) { + storeDoRebuildFromSwapFiles(RB); + } else { + eventAdd("storeRebuild", storeDoRebuildFromSwapFiles, RB, 0); + } +} diff --git a/src/store_swapin.cc b/src/store_swapin.cc new file mode 100644 index 0000000000..c2716e079b --- /dev/null +++ b/src/store_swapin.cc @@ -0,0 +1,102 @@ +#include "squid.h" + +typedef struct swapin_ctrl_t { + StoreEntry *e; + char *path; + SIH *callback; + void *callback_data; + store_client *sc; +} swapin_ctrl_t; + +/* start swapping in */ +/* callback_data will become the tag on which the stat/open can be aborted */ +void +storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) +{ + swapin_ctrl_t *ctrlp; + assert(e->mem_status == NOT_IN_MEMORY); +#if OLD_CODE + if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { + if (storeDirMapBitTest(e->swap_file_number)) { + /* someone took our file while we weren't looking */ + callback(-1, callback_data); + return; + } + } +#endif + debug(20, 3) ("storeSwapInStart: called for %08X %s \n", + e->swap_file_number, storeKeyText(e->key)); + assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); + assert(e->swap_file_number >= 0); + assert(e->mem_obj != NULL); + ctrlp = xmalloc(sizeof(swapin_ctrl_t)); + ctrlp->e = e; + ctrlp->callback = callback; + ctrlp->callback_data = callback_data; + if (EBIT_TEST(e->flag, ENTRY_VALIDATED)) + storeSwapInValidateComplete(ctrlp, 0, 0); + else + storeValidate(e, storeSwapInValidateComplete, ctrlp, callback_data); +} + + +void +storeSwapInValidateComplete(void *data, int retcode, int errcode) +{ + swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; + StoreEntry *e; + + if (retcode == -2 && errcode == -2) { + xfree(ctrlp); + return; + } + e = ctrlp->e; + assert(e->mem_status == NOT_IN_MEMORY); + if (!EBIT_TEST(e->flag, ENTRY_VALIDATED)) { + /* Invoke a store abort that should free the memory object */ + (ctrlp->callback) (-1, ctrlp->callback_data); + xfree(ctrlp); + return; + } + ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); + debug(20, 3) ("storeSwapInValidateComplete: Opening %s\n", ctrlp->path); + file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp, ctrlp->callback_data); +} + +void +storeSwapInFileOpened(void *data, int fd, int errcode) +{ + swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; + StoreEntry *e = ctrlp->e; + MemObject *mem = e->mem_obj; + struct stat sb; + + if (fd == -2 && errcode == -2) { + xfree(ctrlp->path); + xfree(ctrlp); + return; + } + assert(mem != NULL); + assert(e->mem_status == NOT_IN_MEMORY); + assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); + if (e->swap_status == SWAPOUT_DONE && (fd >= 0) && fstat(fd, &sb) == 0) + if (sb.st_size == 0 || sb.st_size != e->object_len) { + debug(20, 0) ("storeSwapInFileOpened: %s: Size mismatch: %d(fstat) != %d(object)\n", ctrlp->path, sb.st_size, e->object_len); + file_close(fd); + fd = -1; + } + if (fd < 0) { + debug(20, 0) ("storeSwapInStartComplete: Failed for '%s' (%s)\n", mem->url, + ctrlp->path); + /* Invoke a store abort that should free the memory object */ + (ctrlp->callback) (-1, ctrlp->callback_data); + xfree(ctrlp->path); + xfree(ctrlp); + return; + } + debug(20, 5) ("storeSwapInStart: initialized swap file '%s' for '%s'\n", + ctrlp->path, mem->url); + (ctrlp->callback) (fd, ctrlp->callback_data); + xfree(ctrlp->path); + xfree(ctrlp); +} diff --git a/src/store_swapmeta.cc b/src/store_swapmeta.cc new file mode 100644 index 0000000000..dd687fbf57 --- /dev/null +++ b/src/store_swapmeta.cc @@ -0,0 +1,119 @@ +#include "squid.h" + +#define squid_key_size MD5_DIGEST_CHARS + +/* build swapfile header */ +int +storeBuildMetaData(StoreEntry * e, char *swap_buf_c) +{ + MemObject *mem; + int keylength; + int a = STORE_META_TLD_START; + char *meta_buf; + mem = e->mem_obj; + meta_buf = mem->swapout.meta_buf; + debug(20, 3) ("storeBuildSwapFileHeader: called.\n"); + assert(e->swap_status == SWAPOUT_WRITING); + if (!meta_buf) + meta_buf = mem->swapout.meta_buf = xmalloc(1024); + /* construct header */ + /* add Length(int)-Type(char)-Data encoded info */ + if (squid_key_size < 0) + keylength = strlen(e->key); + else + keylength = squid_key_size; + meta_buf[0] = META_OK; + xmemcpy(&meta_buf[1], &a, sizeof(int)); + mem->swapout.meta_len = STORE_META_TLD_START; + addSwapHdr(STORE_META_KEY, keylength, (void *) e->key, + mem->swapout.meta_buf, &mem->swapout.meta_len); + addSwapHdr(STORE_META_STD, STORE_HDR_METASIZE, (void *) &e->timestamp, + mem->swapout.meta_buf, &mem->swapout.meta_len); + debug(20, 3) ("storeBuildSwapFileHeader: len=%d.\n", mem->swapout.meta_len); + if (swap_buf_c) + xmemcpy(swap_buf_c, mem->swapout.meta_buf, mem->swapout.meta_len); + return mem->swapout.meta_len; +} + +int +getSwapHdr(int *type, int *len, void *dst, char *write_buf, int hdr_len) +{ + static int cur = 0; + static char *curptr; + char *tmp_buf; + if (cur == 0 || curptr != write_buf) { /* first call or rewind ! */ + cur = STORE_META_TLD_START; + curptr = write_buf; + } + if (cur + STORE_META_TLD_START > hdr_len) { + debug(20, 3) ("getSwapHdr: overflow, %d %d.\n", cur, hdr_len); + cur = 0; + return -1; + } + tmp_buf = &write_buf[cur]; /* position ourselves */ + xmemcpy(len, SwapMetaSize(tmp_buf), sizeof(int)); /* length */ + *type = SwapMetaType(tmp_buf); /* type */ + xmemcpy(dst, SwapMetaData(tmp_buf), *len); /* data */ + cur += STORE_META_TLD_START + *len; /* advance position */ + debug(20, 4) ("getSwapHdr: t=%d l=%d (cur=%d hdr_len=%d) (%p)\n", + *type, *len, cur, hdr_len, dst); + if (cur == hdr_len) { + debug(20, 4) ("getSwapHdr: finished with this.\n"); + cur = 0; + return 1; + } + return 1; /* ok ! */ +} + +void +addSwapHdr(int type, int len, void *src, char *write_buf, int *write_len) +{ + int hdr_len = *write_len; + char *base = &write_buf[hdr_len]; + debug(20, 3) ("addSwapHdr: at %d\n", hdr_len); + base[0] = (char) type; + xmemcpy(&base[1], &len, sizeof(int)); + xmemcpy(SwapMetaData(base), src, len); + hdr_len += STORE_META_TLD_START + len; + /* now we know length */ + debug(20, 3) ("addSwapHdr: added type=%d len=%d data=%p. hdr_len=%d\n", + type, len, src, hdr_len); + /* update header */ + xmemcpy(&write_buf[1], &hdr_len, sizeof(int)); + *write_len = hdr_len; +} + +int +storeGetMetaBuf(const char *buf, MemObject * mem) +{ + int hdr_len; + assert(mem != NULL); + /* the key */ + if (SwapMetaType(buf) != META_OK) { + debug(20, 1) ("storeGetMetaBuf:Found an old-style object, damn.\n"); + return -1; + } + xmemcpy(&hdr_len, SwapMetaSize(buf), sizeof(int)); + mem->swapout.meta_len = hdr_len; + mem->swapout.meta_buf = xmalloc(hdr_len); + xmemcpy(mem->swapout.meta_buf, buf, hdr_len); + debug(20, 3) (" header size %d\n", hdr_len); + return hdr_len; +} + +#if OLD_CODE +static int +storeParseMetaBuf(StoreEntry * e) +{ + static char mbuf[1024]; + int myt, myl; + MemObject *mem = e->mem_obj; + assert(e && e->mem_obj && e->key); + getSwapHdr(&myt, &myl, mbuf, mem->swapout.meta_buf, mem->swapout.meta_len); + mbuf[myl] = 0; + debug(20, 3) ("storeParseMetaBuf: key=%s\n", mbuf); + e->key = xstrdup(storeKeyScan(mbuf)); + getSwapHdr(&myt, &myl, &e->timestamp, mem->swapout.meta_buf, mem->swapout.meta_len); + return 1; +} +#endif diff --git a/src/store_swapout.cc b/src/store_swapout.cc new file mode 100644 index 0000000000..79736d40ba --- /dev/null +++ b/src/store_swapout.cc @@ -0,0 +1,254 @@ +#include "squid.h" + +typedef struct swapout_ctrl_t { + char *swapfilename; + int oldswapstatus; + StoreEntry *e; +} swapout_ctrl_t; + +static void storeSwapoutFileOpened(void *data, int fd, int errcode); + +/* start swapping object to disk */ +void +storeSwapOutStart(StoreEntry * e) +{ + swapout_ctrl_t *ctrlp; + LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); + storeLockObject(e); +#if !MONOTONIC_STORE + if ((e->swap_file_number = storeGetUnusedFileno()) < 0) +#endif + e->swap_file_number = storeDirMapAllocate(); + storeSwapFullPath(e->swap_file_number, swapfilename); + ctrlp = xmalloc(sizeof(swapout_ctrl_t)); + ctrlp->swapfilename = xstrdup(swapfilename); + ctrlp->e = e; + ctrlp->oldswapstatus = e->swap_status; + e->swap_status = SWAPOUT_OPENING; + file_open(swapfilename, + O_WRONLY | O_CREAT | O_TRUNC, + storeSwapoutFileOpened, + ctrlp, e); +} + +void +storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) +{ + StoreEntry *e = data; + MemObject *mem = e->mem_obj; + debug(20, 3) ("storeSwapOutHandle: '%s', len=%d\n", storeKeyText(e->key), (int) len); + if (flag < 0) { + debug(20, 1) ("storeSwapOutHandle: SwapOut failure (err code = %d).\n", + flag); + e->swap_status = SWAPOUT_NONE; + if (e->swap_file_number > -1) { +#if MONOTONIC_STORE +#if USE_ASYNC_IO + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); +#else + unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL)); +#endif +#else + storePutUnusedFileno(e); +#endif + e->swap_file_number = -1; + } + if (flag == DISK_NO_SPACE_LEFT) { + /* reduce the swap_size limit to the current size. */ + Config.Swap.maxSize = store_swap_size; + storeConfigure(); + } + storeReleaseRequest(e); + storeSwapOutFileClose(e); + return; + } +#if USE_ASYNC_IO + if (mem == NULL) { + debug(20, 1) ("storeSwapOutHandle: mem == NULL : Cancelling swapout\n"); + return; + } +#else + assert(mem != NULL); +#endif + mem->swapout.done_offset += len; + if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len + mem->swapout.meta_len) { + storeCheckSwapOut(e); + return; + } + /* swapping complete */ + debug(20, 5) ("storeSwapOutHandle: SwapOut complete: '%s' to %s.\n", + mem->url, storeSwapFullPath(e->swap_file_number, NULL)); + e->swap_status = SWAPOUT_DONE; + storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + if (storeCheckCachable(e)) { + storeLog(STORE_LOG_SWAPOUT, e); +#if 0 + storeDirSwapLog(e); +#endif + } + /* Note, we don't otherwise call storeReleaseRequest() here because + * storeCheckCachable() does it for is if necessary */ + storeSwapOutFileClose(e); +} + +void +storeCheckSwapOut(StoreEntry * e) +{ + MemObject *mem = e->mem_obj; + off_t lowest_offset; + off_t new_mem_lo; + size_t swapout_size; + char *swap_buf; + ssize_t swap_buf_len; + int x; + int hdr_len = 0; + assert(mem != NULL); + /* should we swap something out to disk? */ + debug(20, 3) ("storeCheckSwapOut: %s\n", mem->url); + debug(20, 3) ("storeCheckSwapOut: store_status = %s\n", + storeStatusStr[e->store_status]); + if (e->store_status == STORE_ABORTED) { + assert(EBIT_TEST(e->flag, RELEASE_REQUEST)); + storeSwapOutFileClose(e); + return; + } + debug(20, 3) ("storeCheckSwapOut: mem->inmem_lo = %d\n", + (int) mem->inmem_lo); + debug(20, 3) ("storeCheckSwapOut: mem->inmem_hi = %d\n", + (int) mem->inmem_hi); + debug(20, 3) ("storeCheckSwapOut: swapout.queue_offset = %d\n", + (int) mem->swapout.queue_offset); + debug(20, 3) ("storeCheckSwapOut: swapout.done_offset = %d\n", + (int) mem->swapout.done_offset); +#if USE_ASYNC_IO + if (mem->inmem_hi < mem->swapout.queue_offset) { + storeAbort(e, 0); + assert(EBIT_TEST(e->flag, RELEASE_REQUEST)); + storeSwapOutFileClose(e); + return; + } +#else + assert(mem->inmem_hi >= mem->swapout.queue_offset); +#endif + swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); + lowest_offset = storeLowestMemReaderOffset(e); + debug(20, 3) ("storeCheckSwapOut: lowest_offset = %d\n", + (int) lowest_offset); + assert(lowest_offset >= mem->inmem_lo); + + new_mem_lo = lowest_offset; + if (!EBIT_TEST(e->flag, ENTRY_CACHABLE)) { + if (!EBIT_TEST(e->flag, KEY_PRIVATE)) + debug(20, 0) ("storeCheckSwapOut: Attempt to swap out a non-cacheable non-private object!\n"); + stmemFreeDataUpto(mem->data, new_mem_lo); + mem->inmem_lo = new_mem_lo; + return; + } + if (mem->swapout.queue_offset < new_mem_lo) + new_mem_lo = mem->swapout.queue_offset; + stmemFreeDataUpto(mem->data, new_mem_lo); + mem->inmem_lo = new_mem_lo; + + swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); + debug(20, 3) ("storeCheckSwapOut: swapout_size = %d\n", + (int) swapout_size); + if (swapout_size == 0) + return; + if (e->store_status == STORE_PENDING && swapout_size < VM_WINDOW_SZ) + return; /* wait for a full block */ + /* Ok, we have stuff to swap out. Is there a swapout.fd open? */ + if (e->swap_status == SWAPOUT_NONE) { + assert(mem->swapout.fd == -1); + if (storeCheckCachable(e)) + storeSwapOutStart(e); + /* else ENTRY_CACHABLE will be cleared and we'll never get + * here again */ + return; + } + if (e->swap_status == SWAPOUT_OPENING) + return; + assert(mem->swapout.fd > -1); + swap_buf = memAllocate(MEM_DISK_BUF, 1); + if (mem->swapout.queue_offset == 0) + hdr_len = storeBuildMetaData(e, swap_buf); + + if (swapout_size > STORE_SWAP_BUF - hdr_len) + swapout_size = STORE_SWAP_BUF - hdr_len; + + swap_buf_len = stmemCopy(mem->data, + mem->swapout.queue_offset, + swap_buf + hdr_len, + swapout_size) + hdr_len; + + if (swap_buf_len < 0) { + debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); + /* XXX This is probably wrong--we should storeRelease()? */ + storeDirMapBitReset(e->swap_file_number); + safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); + e->swap_file_number = -1; + e->swap_status = SWAPOUT_NONE; + memFree(MEM_DISK_BUF, swap_buf); + storeSwapOutFileClose(e); + return; + } + debug(20, 3) ("storeCheckSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); + assert(swap_buf_len > 0); + debug(20, 3) ("storeCheckSwapOut: swapping out %d bytes from %d\n", + swap_buf_len, mem->swapout.queue_offset); + mem->swapout.queue_offset += swap_buf_len - hdr_len; + x = file_write(mem->swapout.fd, + -1, + swap_buf, + swap_buf_len, + storeSwapOutHandle, + e, + memFreeDISK); + assert(x == DISK_OK); +} + +void +storeSwapOutFileClose(StoreEntry * e) +{ + MemObject *mem = e->mem_obj; + if (mem->swapout.fd > -1) + file_close(mem->swapout.fd); +#if USE_ASYNC_IO + else + aioCancel(-1, e); /* Make doubly certain pending ops are gone */ +#endif + mem->swapout.fd = -1; + storeUnlockObject(e); +} + +static void +storeSwapoutFileOpened(void *data, int fd, int errcode) +{ + swapout_ctrl_t *ctrlp = data; + int oldswapstatus = ctrlp->oldswapstatus; + char *swapfilename = ctrlp->swapfilename; + StoreEntry *e = ctrlp->e; + MemObject *mem; + xfree(ctrlp); + if (fd == -2 && errcode == -2) { /* Cancelled - Clean up */ + xfree(swapfilename); + return; + } + assert(e->swap_status == SWAPOUT_OPENING); + if (fd < 0) { + debug(20, 0) ("storeSwapoutFileOpened: Unable to open swapfile: %s\n", + swapfilename); + storeDirMapBitReset(e->swap_file_number); + e->swap_file_number = -1; + e->swap_status = oldswapstatus; + xfree(swapfilename); + return; + } + mem = e->mem_obj; + mem->swapout.fd = (short) fd; + e->swap_status = SWAPOUT_WRITING; + debug(20, 5) ("storeSwapoutFileOpened: Begin SwapOut '%s' to FD %d FILE %s.\n", + mem->url, fd, swapfilename); + xfree(swapfilename); + debug(20, 5) ("swap_file_number=%08X\n", e->swap_file_number); + storeCheckSwapOut(e); +} diff --git a/src/typedefs.h b/src/typedefs.h index 398c449132..806a16738b 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -105,13 +105,7 @@ typedef void STABH(void *); typedef void ERCB(int fd, void *, size_t); typedef void OBJH(StoreEntry *); typedef void SIGHDLR(int sig); +typedef void STVLDCB(void *, int, int); -#if STORE_KEY_SHA -typedef int cache_key; -#endif -#if STORE_KEY_MD5 +/* MD5 cache keys */ typedef unsigned char cache_key; -#endif -#if STORE_KEY_URL -typedef char cache_key; -#endif