--enable-carp Enable CARP support"
ac_help="$ac_help
--enable-async-io[=N_THREADS]
- Do ASYNC disk I/O using threads.
- N_THREADS is the number of worker threads
- defaults to 16. See also src/squid.h for
- some additional platform tuning"
+ Shorthand for
+ --with-aio-threads=N_THREADS
+ --with-pthreads
+ --enable-storeio=ufs,aufs"
+ac_help="$ac_help
+ --with-aio-threads=N_THREADS
+ Tune the number of worker threads for the aufs object
+ store."
+ac_help="$ac_help
+ --with-pthreads Use POSIX Threads"
+ac_help="$ac_help
+ --enable-storeio=\"list of modules\"
+ Build support for the list of store I/O modules.
+ The default is only to build the "ufs" module.
+ See src/fs for a list of available modules, or
+ Programmers Guide section <not yet written>
+ for details on how to build your custom store module"
ac_help="$ac_help
--enable-icmp Enable ICMP pinging"
ac_help="$ac_help
ac_help="$ac_help
--enable-err-language=lang
Select language for Error pages (see errors dir) "
+ac_help="$ac_help
+ --with-coss-membuf-size COSS membuf size (default 1048576 bytes) "
ac_help="$ac_help
--enable-poll Enable poll() instead of select(). Normally poll
is preferred over select, but configure knows poll
--enable-heap-replacement
This option allows you to use various cache
replacement algorithms, instead of the standard
- LRU algorithm.
- "
+ LRU algorithm."
ac_help="$ac_help
--enable-auth-modules=\"list of modules\"
This option selects wich proxy_auth helper modules
to build and install as part of the normal build
process. For a list of available modules see
the auth_modules directory."
-ac_help="$ac_help
- --enable-diskd Uses shared memory, message queues, and external
- processes for disk I/O."
# Initialize some variables set by options.
# The variables have the same names as the options, with
-# From configure.in Revision
+# From configure.in Revision: 1.185
ac_aux_dir=
for ac_dir in cfgaux $srcdir/cfgaux; do
if test -f $ac_dir/install-sh; then
fi
echo $ac_n "checking host system type""... $ac_c" 1>&6
-echo "configure:664: checking host system type" >&5
+echo "configure:675: checking host system type" >&5
host_alias=$host
case "$host_alias" in
# Extract the first word of "gcc", so it can be a program name with args.
set dummy gcc; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:732: checking for $ac_word" >&5
+echo "configure:743: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "cc", so it can be a program name with args.
set dummy cc; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:762: checking for $ac_word" >&5
+echo "configure:773: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "cl", so it can be a program name with args.
set dummy cl; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:813: checking for $ac_word" >&5
+echo "configure:824: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
fi
echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works""... $ac_c" 1>&6
-echo "configure:845: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5
+echo "configure:856: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5
ac_ext=c
# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
cat > conftest.$ac_ext << EOF
-#line 856 "configure"
+#line 867 "configure"
#include "confdefs.h"
main(){return(0);}
EOF
-if { (eval echo configure:861: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:872: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
ac_cv_prog_cc_works=yes
# If we can't run a trivial program, we are probably using a cross compiler.
if (./conftest; exit) 2>/dev/null; then
{ echo "configure: error: installation or configuration problem: C compiler cannot create executables." 1>&2; exit 1; }
fi
echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6
-echo "configure:887: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5
+echo "configure:898: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5
echo "$ac_t""$ac_cv_prog_cc_cross" 1>&6
cross_compiling=$ac_cv_prog_cc_cross
echo $ac_n "checking whether we are using GNU C""... $ac_c" 1>&6
-echo "configure:892: checking whether we are using GNU C" >&5
+echo "configure:903: checking whether we are using GNU C" >&5
if eval "test \"`echo '$''{'ac_cv_prog_gcc'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
yes;
#endif
EOF
-if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:901: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
+if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:912: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
ac_cv_prog_gcc=yes
else
ac_cv_prog_gcc=no
ac_save_CFLAGS="$CFLAGS"
CFLAGS=
echo $ac_n "checking whether ${CC-cc} accepts -g""... $ac_c" 1>&6
-echo "configure:920: checking whether ${CC-cc} accepts -g" >&5
+echo "configure:931: checking whether ${CC-cc} accepts -g" >&5
if eval "test \"`echo '$''{'ac_cv_prog_cc_g'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
fi
-# Check whether --enable-async_io or --disable-async_io was given.
+# Check whether --enable-async-io or --disable-async-io was given.
if test "${enable_async_io+set}" = set; then
enableval="$enable_async_io"
- case "$enableval" in
+ case $enableval in
yes)
- async_io=yes
+ with_pthreads=yes
+ STORE_MODULES="ufs aufs"
;;
no)
- async_io=''
;;
*)
- async_io=yes
- cat >> confdefs.h <<EOF
-#define NUMTHREADS $enableval
-EOF
-
+ async_io_threads=$enableval
+ with_pthreads=yes
+ STORE_MODULES="ufs aufs"
;;
esac
fi
-if test -n "$async_io" ; then
- echo "Async I/O enabled"
- async_io=yes
- cat >> confdefs.h <<\EOF
-#define USE_ASYNC_IO 1
+# Check whether --with-aio-threads or --without-aio-threads was given.
+if test "${with_aio_threads+set}" = set; then
+ withval="$with_aio_threads"
+ async_io_threads=$withval
+fi
+
+if test "$async_io_threads"; then
+ echo "With $async_io_threads AIO threads"
+ with_pthreads=yes
+ cat >> confdefs.h <<EOF
+#define ASYNC_IO_THREADS $async_io_threads
EOF
- ASYNC_OBJS='$(ASYNC_OBJS)'
+fi
+
+# Check whether --with-pthreads or --without-pthreads was given.
+if test "${with_pthreads+set}" = set; then
+ withval="$with_pthreads"
+ if test "$enableval" = "yes"; then
+ with_pthreads=yes
+ fi
+
+fi
+
+if test "$with_pthreads"; then
+ echo "With pthreads"
SQUID_PTHREAD_LIB='$(PTHREADLIB)'
CFLAGS="$CFLAGS -D_REENTRANT"
case "$host" in
i386-unknown-freebsd*)
- if test "$GCC" = "yes" ; then
- if test -z "$PRESET_LDFLAGS"; then
- LDFLAGS="$LDFLAGS -pthread"
+ if test "$GCC" = "yes" ; then
+ if test -z "$PRESET_LDFLAGS"; then
+ LDFLAGS="$LDFLAGS -pthread"
+ fi
+ fi
+ ;;
+ esac
+fi
+
+
+# Check whether --enable-storeio or --disable-storeio was given.
+if test "${enable_storeio+set}" = set; then
+ enableval="$enable_storeio"
+ case $enableval in
+ yes)
+ for module in $srcdir/src/fs/*; do
+ if test -f $module/Makefile.in; then
+ STORE_MODULES="$STORE_MODULES `basename $module`"
fi
- fi
+ done
+ ;;
+ no)
+ ;;
+ *) STORE_MODULES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`"
;;
- esac
+ esac
+
+else
+ if test -z "$STORE_MODULES"; then
+ STORE_MODULES="ufs"
+ fi
+
fi
+echo "Store moules built: $STORE_MODULES"
+
+STORE_OBJS="fs/`echo $STORE_MODULES|sed -e's% %.a fs/%g'`.a"
# Check whether --enable-icmp or --disable-icmp was given.
+# Check whether --with-coss-membuf-size or --without-coss-membuf-size was given.
+if test "${with_coss_membuf_size+set}" = set; then
+ withval="$with_coss_membuf_size"
+ if test "$with_coss_membuf_size"; then
+ echo "Setting COSS membuf size to $with_coss_membuf_size bytes"
+ cat >> confdefs.h <<EOF
+#define COSS_MEMBUF_SZ $with_coss_membuf_size
+EOF
+
+ fi
+
+fi
+
+
# Check whether --enable-poll or --disable-poll was given.
if test "${enable_poll+set}" = set; then
enableval="$enable_poll"
fi
-# Check whether --enable-diskd or --disable-diskd was given.
-if test "${enable_diskd+set}" = set; then
- enableval="$enable_diskd"
- if test "$enableval" = "yes" ; then
- echo "DISKD enabled"
- cat >> confdefs.h <<\EOF
-#define USE_DISKD 1
-EOF
-
- OPT_DISKD_EXE='$(DISKD_EXE)'
-
- DISKD_OBJS='diskd.o'
-
- fi
-
-fi
-
-
# Force some compilers to use ANSI features
#
case "$host" in
esac
echo $ac_n "checking how to run the C preprocessor""... $ac_c" 1>&6
-echo "configure:1645: checking how to run the C preprocessor" >&5
+echo "configure:1696: checking how to run the C preprocessor" >&5
# On Suns, sometimes $CPP names a directory.
if test -n "$CPP" && test -d "$CPP"; then
CPP=
# On the NeXT, cc -E runs the code through the compiler's parser,
# not just through cpp.
cat > conftest.$ac_ext <<EOF
-#line 1660 "configure"
+#line 1711 "configure"
#include "confdefs.h"
#include <assert.h>
Syntax Error
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:1666: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:1717: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
:
rm -rf conftest*
CPP="${CC-cc} -E -traditional-cpp"
cat > conftest.$ac_ext <<EOF
-#line 1677 "configure"
+#line 1728 "configure"
#include "confdefs.h"
#include <assert.h>
Syntax Error
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:1683: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:1734: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
:
rm -rf conftest*
CPP="${CC-cc} -nologo -E"
cat > conftest.$ac_ext <<EOF
-#line 1694 "configure"
+#line 1745 "configure"
#include "confdefs.h"
#include <assert.h>
Syntax Error
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:1700: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:1751: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
:
# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
# ./install, which can be erroneously created by make from ./install.sh.
echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6
-echo "configure:1736: checking for a BSD compatible install" >&5
+echo "configure:1787: checking for a BSD compatible install" >&5
if test -z "$INSTALL"; then
if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
# Extract the first word of "ranlib", so it can be a program name with args.
set dummy ranlib; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1791: checking for $ac_word" >&5
+echo "configure:1842: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_prog_RANLIB'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
fi
echo $ac_n "checking whether ln -s works""... $ac_c" 1>&6
-echo "configure:1819: checking whether ln -s works" >&5
+echo "configure:1870: checking whether ln -s works" >&5
if eval "test \"`echo '$''{'ac_cv_prog_LN_S'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "sh", so it can be a program name with args.
set dummy sh; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1842: checking for $ac_word" >&5
+echo "configure:1893: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_SH'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "false", so it can be a program name with args.
set dummy false; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1878: checking for $ac_word" >&5
+echo "configure:1929: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_FALSE'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "true", so it can be a program name with args.
set dummy true; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1914: checking for $ac_word" >&5
+echo "configure:1965: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_TRUE'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "rm", so it can be a program name with args.
set dummy rm; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1950: checking for $ac_word" >&5
+echo "configure:2001: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_RM'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "mv", so it can be a program name with args.
set dummy mv; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:1986: checking for $ac_word" >&5
+echo "configure:2037: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_MV'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "mkdir", so it can be a program name with args.
set dummy mkdir; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2022: checking for $ac_word" >&5
+echo "configure:2073: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_MKDIR'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "ln", so it can be a program name with args.
set dummy ln; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2058: checking for $ac_word" >&5
+echo "configure:2109: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_LN'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "perl", so it can be a program name with args.
set dummy perl; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2094: checking for $ac_word" >&5
+echo "configure:2145: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_PERL'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "makedepend", so it can be a program name with args.
set dummy makedepend; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2130: checking for $ac_word" >&5
+echo "configure:2181: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_MAKEDEPEND'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
# Extract the first word of "ar", so it can be a program name with args.
set dummy ar; ac_word=$2
echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
-echo "configure:2166: checking for $ac_word" >&5
+echo "configure:2217: checking for $ac_word" >&5
if eval "test \"`echo '$''{'ac_cv_path_AR'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr that defines DIR""... $ac_c" 1>&6
-echo "configure:2220: checking for $ac_hdr that defines DIR" >&5
+echo "configure:2271: checking for $ac_hdr that defines DIR" >&5
if eval "test \"`echo '$''{'ac_cv_header_dirent_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2225 "configure"
+#line 2276 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <$ac_hdr>
DIR *dirp = 0;
; return 0; }
EOF
-if { (eval echo configure:2233: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2284: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
eval "ac_cv_header_dirent_$ac_safe=yes"
else
# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
if test $ac_header_dirent = dirent.h; then
echo $ac_n "checking for opendir in -ldir""... $ac_c" 1>&6
-echo "configure:2258: checking for opendir in -ldir" >&5
+echo "configure:2309: checking for opendir in -ldir" >&5
ac_lib_var=`echo dir'_'opendir | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-ldir $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 2266 "configure"
+#line 2317 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
opendir()
; return 0; }
EOF
-if { (eval echo configure:2277: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:2328: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
else
echo $ac_n "checking for opendir in -lx""... $ac_c" 1>&6
-echo "configure:2299: checking for opendir in -lx" >&5
+echo "configure:2350: checking for opendir in -lx" >&5
ac_lib_var=`echo x'_'opendir | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lx $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 2307 "configure"
+#line 2358 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
opendir()
; return 0; }
EOF
-if { (eval echo configure:2318: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:2369: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6
-echo "configure:2341: checking for ANSI C header files" >&5
+echo "configure:2392: checking for ANSI C header files" >&5
if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2346 "configure"
+#line 2397 "configure"
#include "confdefs.h"
#include <stdlib.h>
#include <stdarg.h>
#include <float.h>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:2354: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:2405: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
if test $ac_cv_header_stdc = yes; then
# SunOS 4.x string.h does not declare mem*, contrary to ANSI.
cat > conftest.$ac_ext <<EOF
-#line 2371 "configure"
+#line 2422 "configure"
#include "confdefs.h"
#include <string.h>
EOF
if test $ac_cv_header_stdc = yes; then
# ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
cat > conftest.$ac_ext <<EOF
-#line 2389 "configure"
+#line 2440 "configure"
#include "confdefs.h"
#include <stdlib.h>
EOF
:
else
cat > conftest.$ac_ext <<EOF
-#line 2410 "configure"
+#line 2461 "configure"
#include "confdefs.h"
#include <ctype.h>
#define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
exit (0); }
EOF
-if { (eval echo configure:2421: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:2472: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
:
else
do
ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
-echo "configure:2512: checking for $ac_hdr" >&5
+echo "configure:2563: checking for $ac_hdr" >&5
if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2517 "configure"
+#line 2568 "configure"
#include "confdefs.h"
#include <$ac_hdr>
EOF
ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
-{ (eval echo configure:2522: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+{ (eval echo configure:2573: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
if test -z "$ac_err"; then
rm -rf conftest*
echo $ac_n "checking for working const""... $ac_c" 1>&6
-echo "configure:2550: checking for working const" >&5
+echo "configure:2601: checking for working const" >&5
if eval "test \"`echo '$''{'ac_cv_c_const'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2555 "configure"
+#line 2606 "configure"
#include "confdefs.h"
int main() {
; return 0; }
EOF
-if { (eval echo configure:2604: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2655: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_c_const=yes
else
fi
echo $ac_n "checking whether byte ordering is bigendian""... $ac_c" 1>&6
-echo "configure:2625: checking whether byte ordering is bigendian" >&5
+echo "configure:2676: checking whether byte ordering is bigendian" >&5
if eval "test \"`echo '$''{'ac_cv_c_bigendian'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
ac_cv_c_bigendian=unknown
# See if sys/param.h defines the BYTE_ORDER macro.
cat > conftest.$ac_ext <<EOF
-#line 2632 "configure"
+#line 2683 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <sys/param.h>
#endif
; return 0; }
EOF
-if { (eval echo configure:2643: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2694: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
# It does; now see whether it defined to BIG_ENDIAN or not.
cat > conftest.$ac_ext <<EOF
-#line 2647 "configure"
+#line 2698 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <sys/param.h>
#endif
; return 0; }
EOF
-if { (eval echo configure:2658: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2709: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_c_bigendian=yes
else
{ echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; }
else
cat > conftest.$ac_ext <<EOF
-#line 2678 "configure"
+#line 2729 "configure"
#include "confdefs.h"
main () {
/* Are we little or big endian? From Harbison&Steele. */
exit (u.c[sizeof (long) - 1] == 1);
}
EOF
-if { (eval echo configure:2691: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:2742: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_c_bigendian=no
else
echo $ac_n "checking if ANSI prototypes work""... $ac_c" 1>&6
-echo "configure:2716: checking if ANSI prototypes work" >&5
+echo "configure:2767: checking if ANSI prototypes work" >&5
if eval "test \"`echo '$''{'ac_cv_have_ansi_prototypes'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2722 "configure"
+#line 2773 "configure"
#include "confdefs.h"
int foo(char *); int foo (char *bar) {return 1;}
int main() {
foo("bar")
; return 0; }
EOF
-if { (eval echo configure:2729: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2780: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_ansi_prototypes="yes"
else
fi
echo $ac_n "checking for tm->tm_gmtoff""... $ac_c" 1>&6
-echo "configure:2751: checking for tm->tm_gmtoff" >&5
+echo "configure:2802: checking for tm->tm_gmtoff" >&5
if eval "test \"`echo '$''{'ac_cv_have_tm_gmoff'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2757 "configure"
+#line 2808 "configure"
#include "confdefs.h"
#include <time.h>
#include <sys/time.h>
foo.tm_gmtoff = 0;
; return 0; }
EOF
-if { (eval echo configure:2766: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2817: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_tm_gmoff="yes"
else
fi
echo $ac_n "checking for struct mallinfo""... $ac_c" 1>&6
-echo "configure:2788: checking for struct mallinfo" >&5
+echo "configure:2839: checking for struct mallinfo" >&5
if eval "test \"`echo '$''{'ac_cv_have_struct_mallinfo'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2794 "configure"
+#line 2845 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if HAVE_MALLOC_H
foo.keepcost = 0;
; return 0; }
EOF
-if { (eval echo configure:2812: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2863: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_struct_mallinfo="yes"
else
fi
echo $ac_n "checking for extended mallinfo""... $ac_c" 1>&6
-echo "configure:2834: checking for extended mallinfo" >&5
+echo "configure:2885: checking for extended mallinfo" >&5
if eval "test \"`echo '$''{'ac_cv_have_ext_mallinfo'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2840 "configure"
+#line 2891 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <malloc.h>
foo.mxfast = 0;
; return 0; }
EOF
-if { (eval echo configure:2849: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2900: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_ext_mallinfo="yes"
else
fi
echo $ac_n "checking for struct rusage""... $ac_c" 1>&6
-echo "configure:2871: checking for struct rusage" >&5
+echo "configure:2922: checking for struct rusage" >&5
if eval "test \"`echo '$''{'ac_cv_have_struct_rusage'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2877 "configure"
+#line 2928 "configure"
#include "confdefs.h"
#if HAVE_SYS_TIME_H
struct rusage R;
; return 0; }
EOF
-if { (eval echo configure:2890: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2941: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_struct_rusage="yes"
else
fi
echo $ac_n "checking for ip->ip_hl""... $ac_c" 1>&6
-echo "configure:2912: checking for ip->ip_hl" >&5
+echo "configure:2963: checking for ip->ip_hl" >&5
if eval "test \"`echo '$''{'ac_cv_have_ip_hl'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 2918 "configure"
+#line 2969 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <netinet/in.h>
ip.ip_hl= 0;
; return 0; }
EOF
-if { (eval echo configure:2935: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:2986: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_ip_hl="yes"
else
fi
echo $ac_n "checking size of int""... $ac_c" 1>&6
-echo "configure:2957: checking size of int" >&5
+echo "configure:3008: checking size of int" >&5
if eval "test \"`echo '$''{'ac_cv_sizeof_int'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
{ echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; }
else
cat > conftest.$ac_ext <<EOF
-#line 2965 "configure"
+#line 3016 "configure"
#include "confdefs.h"
#include <stdio.h>
main()
exit(0);
}
EOF
-if { (eval echo configure:2976: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:3027: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_sizeof_int=`cat conftestval`
else
echo $ac_n "checking size of long""... $ac_c" 1>&6
-echo "configure:2996: checking size of long" >&5
+echo "configure:3047: checking size of long" >&5
if eval "test \"`echo '$''{'ac_cv_sizeof_long'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
{ echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; }
else
cat > conftest.$ac_ext <<EOF
-#line 3004 "configure"
+#line 3055 "configure"
#include "confdefs.h"
#include <stdio.h>
main()
exit(0);
}
EOF
-if { (eval echo configure:3015: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:3066: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_sizeof_long=`cat conftestval`
else
echo $ac_n "checking size of void *""... $ac_c" 1>&6
-echo "configure:3035: checking size of void *" >&5
+echo "configure:3086: checking size of void *" >&5
if eval "test \"`echo '$''{'ac_cv_sizeof_void_p'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
{ echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; }
else
cat > conftest.$ac_ext <<EOF
-#line 3043 "configure"
+#line 3094 "configure"
#include "confdefs.h"
#include <stdio.h>
main()
exit(0);
}
EOF
-if { (eval echo configure:3054: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:3105: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_sizeof_void_p=`cat conftestval`
else
# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
# for constant arguments. Useless!
echo $ac_n "checking for working alloca.h""... $ac_c" 1>&6
-echo "configure:3077: checking for working alloca.h" >&5
+echo "configure:3128: checking for working alloca.h" >&5
if eval "test \"`echo '$''{'ac_cv_header_alloca_h'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3082 "configure"
+#line 3133 "configure"
#include "confdefs.h"
#include <alloca.h>
int main() {
char *p = alloca(2 * sizeof(int));
; return 0; }
EOF
-if { (eval echo configure:3089: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3140: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
ac_cv_header_alloca_h=yes
else
fi
echo $ac_n "checking for alloca""... $ac_c" 1>&6
-echo "configure:3110: checking for alloca" >&5
+echo "configure:3161: checking for alloca" >&5
if eval "test \"`echo '$''{'ac_cv_func_alloca_works'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3115 "configure"
+#line 3166 "configure"
#include "confdefs.h"
#ifdef __GNUC__
char *p = (char *) alloca(1);
; return 0; }
EOF
-if { (eval echo configure:3143: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3194: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
ac_cv_func_alloca_works=yes
else
echo $ac_n "checking whether alloca needs Cray hooks""... $ac_c" 1>&6
-echo "configure:3175: checking whether alloca needs Cray hooks" >&5
+echo "configure:3226: checking whether alloca needs Cray hooks" >&5
if eval "test \"`echo '$''{'ac_cv_os_cray'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3180 "configure"
+#line 3231 "configure"
#include "confdefs.h"
#if defined(CRAY) && ! defined(CRAY2)
webecray
if test $ac_cv_os_cray = yes; then
for ac_func in _getb67 GETB67 getb67; do
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
-echo "configure:3205: checking for $ac_func" >&5
+echo "configure:3256: checking for $ac_func" >&5
if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3210 "configure"
+#line 3261 "configure"
#include "confdefs.h"
/* System header to define __stub macros and hopefully few prototypes,
which can conflict with char $ac_func(); below. */
; return 0; }
EOF
-if { (eval echo configure:3233: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3284: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_func_$ac_func=yes"
else
fi
echo $ac_n "checking stack direction for C alloca""... $ac_c" 1>&6
-echo "configure:3260: checking stack direction for C alloca" >&5
+echo "configure:3311: checking stack direction for C alloca" >&5
if eval "test \"`echo '$''{'ac_cv_c_stack_direction'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
ac_cv_c_stack_direction=0
else
cat > conftest.$ac_ext <<EOF
-#line 3268 "configure"
+#line 3319 "configure"
#include "confdefs.h"
find_stack_direction ()
{
exit (find_stack_direction() < 0);
}
EOF
-if { (eval echo configure:3287: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:3338: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_c_stack_direction=1
else
echo $ac_n "checking for pid_t""... $ac_c" 1>&6
-echo "configure:3310: checking for pid_t" >&5
+echo "configure:3361: checking for pid_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_pid_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3315 "configure"
+#line 3366 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for size_t""... $ac_c" 1>&6
-echo "configure:3343: checking for size_t" >&5
+echo "configure:3394: checking for size_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_size_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3348 "configure"
+#line 3399 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for ssize_t""... $ac_c" 1>&6
-echo "configure:3376: checking for ssize_t" >&5
+echo "configure:3427: checking for ssize_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_ssize_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3381 "configure"
+#line 3432 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for off_t""... $ac_c" 1>&6
-echo "configure:3409: checking for off_t" >&5
+echo "configure:3460: checking for off_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_off_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3414 "configure"
+#line 3465 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for mode_t""... $ac_c" 1>&6
-echo "configure:3442: checking for mode_t" >&5
+echo "configure:3493: checking for mode_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_mode_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3447 "configure"
+#line 3498 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for fd_mask""... $ac_c" 1>&6
-echo "configure:3475: checking for fd_mask" >&5
+echo "configure:3526: checking for fd_mask" >&5
if eval "test \"`echo '$''{'ac_cv_type_fd_mask'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3480 "configure"
+#line 3531 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
fi
echo $ac_n "checking for mtyp_t""... $ac_c" 1>&6
-echo "configure:3508: checking for mtyp_t" >&5
+echo "configure:3559: checking for mtyp_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_mtyp_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3513 "configure"
+#line 3564 "configure"
#include "confdefs.h"
#include <sys/types.h>
#if STDC_HEADERS
echo $ac_n "checking for socklen_t""... $ac_c" 1>&6
-echo "configure:3542: checking for socklen_t" >&5
+echo "configure:3593: checking for socklen_t" >&5
if eval "test \"`echo '$''{'ac_cv_type_socklen_t'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 3548 "configure"
+#line 3599 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <sys/socket.h>
fi
echo $ac_n "checking for main in -lnsl""... $ac_c" 1>&6
-echo "configure:3579: checking for main in -lnsl" >&5
+echo "configure:3630: checking for main in -lnsl" >&5
ac_lib_var=`echo nsl'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lnsl $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3587 "configure"
+#line 3638 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3594: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3645: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for main in -lsocket""... $ac_c" 1>&6
-echo "configure:3622: checking for main in -lsocket" >&5
+echo "configure:3673: checking for main in -lsocket" >&5
ac_lib_var=`echo socket'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lsocket $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3630 "configure"
+#line 3681 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3637: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3688: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
echo "skipping libmalloc check (--enable-dlmalloc specified)"
else
echo $ac_n "checking for main in -lgnumalloc""... $ac_c" 1>&6
-echo "configure:3669: checking for main in -lgnumalloc" >&5
+echo "configure:3720: checking for main in -lgnumalloc" >&5
ac_lib_var=`echo gnumalloc'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lgnumalloc $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3677 "configure"
+#line 3728 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3684: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3735: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
*)
echo $ac_n "checking for main in -lmalloc""... $ac_c" 1>&6
-echo "configure:3727: checking for main in -lmalloc" >&5
+echo "configure:3778: checking for main in -lmalloc" >&5
ac_lib_var=`echo malloc'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lmalloc $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3735 "configure"
+#line 3786 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3742: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3793: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for main in -lbsd""... $ac_c" 1>&6
-echo "configure:3775: checking for main in -lbsd" >&5
+echo "configure:3826: checking for main in -lbsd" >&5
ac_lib_var=`echo bsd'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lbsd $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3783 "configure"
+#line 3834 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3790: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3841: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for main in -lregex""... $ac_c" 1>&6
-echo "configure:3818: checking for main in -lregex" >&5
+echo "configure:3869: checking for main in -lregex" >&5
ac_lib_var=`echo regex'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lregex $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3826 "configure"
+#line 3877 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:3833: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3884: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for gethostbyname in -lbind""... $ac_c" 1>&6
-echo "configure:3854: checking for gethostbyname in -lbind" >&5
+echo "configure:3905: checking for gethostbyname in -lbind" >&5
ac_lib_var=`echo bind'_'gethostbyname | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lbind $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3862 "configure"
+#line 3913 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
gethostbyname()
; return 0; }
EOF
-if { (eval echo configure:3873: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3924: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
;;
*)
echo $ac_n "checking for inet_aton in -lresolv""... $ac_c" 1>&6
-echo "configure:3907: checking for inet_aton in -lresolv" >&5
+echo "configure:3958: checking for inet_aton in -lresolv" >&5
ac_lib_var=`echo resolv'_'inet_aton | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lresolv $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3915 "configure"
+#line 3966 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
inet_aton()
; return 0; }
EOF
-if { (eval echo configure:3926: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:3977: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
echo "$ac_t""yes" 1>&6
echo $ac_n "checking for inet_aton in -l44bsd""... $ac_c" 1>&6
-echo "configure:3942: checking for inet_aton in -l44bsd" >&5
+echo "configure:3993: checking for inet_aton in -l44bsd" >&5
ac_lib_var=`echo 44bsd'_'inet_aton | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-l44bsd $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 3950 "configure"
+#line 4001 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
inet_aton()
; return 0; }
EOF
-if { (eval echo configure:3961: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4012: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
fi
echo $ac_n "checking for main in -lresolv""... $ac_c" 1>&6
-echo "configure:3993: checking for main in -lresolv" >&5
+echo "configure:4044: checking for main in -lresolv" >&5
ac_lib_var=`echo resolv'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lresolv $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 4001 "configure"
+#line 4052 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:4008: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4059: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
esac
fi
echo $ac_n "checking for main in -lm""... $ac_c" 1>&6
-echo "configure:4039: checking for main in -lm" >&5
+echo "configure:4090: checking for main in -lm" >&5
ac_lib_var=`echo m'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lm $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 4047 "configure"
+#line 4098 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:4054: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4105: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
echo $ac_n "checking for crypt in -lcrypt""... $ac_c" 1>&6
-echo "configure:4083: checking for crypt in -lcrypt" >&5
+echo "configure:4134: checking for crypt in -lcrypt" >&5
ac_lib_var=`echo crypt'_'crypt | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lcrypt $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 4091 "configure"
+#line 4142 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
crypt()
; return 0; }
EOF
-if { (eval echo configure:4102: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4153: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
echo $ac_n "checking for main in -lpthread""... $ac_c" 1>&6
-echo "configure:4135: checking for main in -lpthread" >&5
+echo "configure:4186: checking for main in -lpthread" >&5
ac_lib_var=`echo pthread'_'main | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lpthread $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 4143 "configure"
+#line 4194 "configure"
#include "confdefs.h"
int main() {
main()
; return 0; }
EOF
-if { (eval echo configure:4150: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4201: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
case "$host" in
*-pc-sco3.2*)
echo $ac_n "checking for strftime in -lintl""... $ac_c" 1>&6
-echo "configure:4185: checking for strftime in -lintl" >&5
+echo "configure:4236: checking for strftime in -lintl" >&5
ac_lib_var=`echo intl'_'strftime | sed 'y%./+-%__p_%'`
if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
ac_save_LIBS="$LIBS"
LIBS="-lintl $LIBS"
cat > conftest.$ac_ext <<EOF
-#line 4193 "configure"
+#line 4244 "configure"
#include "confdefs.h"
/* Override any gcc2 internal prototype to avoid an error. */
/* We use char because int might match the return type of a gcc2
strftime()
; return 0; }
EOF
-if { (eval echo configure:4204: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4255: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_lib_$ac_lib_var=yes"
else
mktime \
mstats \
poll \
+ pthread_attr_setscope \
+ pthread_setschedparam \
+ pthread_attr_setschedparam \
putenv \
random \
regcomp \
do
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
-echo "configure:4369: checking for $ac_func" >&5
+echo "configure:4423: checking for $ac_func" >&5
if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 4374 "configure"
+#line 4428 "configure"
#include "confdefs.h"
/* System header to define __stub macros and hopefully few prototypes,
which can conflict with char $ac_func(); below. */
; return 0; }
EOF
-if { (eval echo configure:4397: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4451: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_func_$ac_func=yes"
else
done
-if test "$async_io" = "yes" ; then
- for ac_func in \
- pthread_attr_setscope \
- pthread_setschedparam \
- pthread_attr_setschedparam \
-
-do
-echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
-echo "configure:4430: checking for $ac_func" >&5
-if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
- echo $ac_n "(cached) $ac_c" 1>&6
-else
- cat > conftest.$ac_ext <<EOF
-#line 4435 "configure"
-#include "confdefs.h"
-/* System header to define __stub macros and hopefully few prototypes,
- which can conflict with char $ac_func(); below. */
-#include <assert.h>
-/* Override any gcc2 internal prototype to avoid an error. */
-/* We use char because int might match the return type of a gcc2
- builtin and then its argument prototype would still apply. */
-char $ac_func();
-
-int main() {
-
-/* The GNU C library defines this for functions which it implements
- to always fail with ENOSYS. Some functions are actually named
- something starting with __ and the normal name is an alias. */
-#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
-choke me
-#else
-$ac_func();
-#endif
-
-; return 0; }
-EOF
-if { (eval echo configure:4458: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
- rm -rf conftest*
- eval "ac_cv_func_$ac_func=yes"
-else
- echo "configure: failed program was:" >&5
- cat conftest.$ac_ext >&5
- rm -rf conftest*
- eval "ac_cv_func_$ac_func=no"
-fi
-rm -f conftest*
-fi
-
-if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
- echo "$ac_t""yes" 1>&6
- ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
- cat >> confdefs.h <<EOF
-#define $ac_tr_func 1
-EOF
-
-else
- echo "$ac_t""no" 1>&6
-fi
-done
-
-fi
-
echo $ac_n "checking if setresuid is implemented""... $ac_c" 1>&6
-echo "configure:4485: checking if setresuid is implemented" >&5
+echo "configure:4477: checking if setresuid is implemented" >&5
if eval "test \"`echo '$''{'ac_cv_func_setresuid'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
{ echo "configure: error: can not run test program while cross compiling" 1>&2; exit 1; }
else
cat > conftest.$ac_ext <<EOF
-#line 4493 "configure"
+#line 4485 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:4506: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4498: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
ac_cv_func_setresuid="yes"
else
if test "$IPF_TRANSPARENT" ; then
echo $ac_n "checking if IP-Filter header files are installed""... $ac_c" 1>&6
-echo "configure:4531: checking if IP-Filter header files are installed" >&5
+echo "configure:4523: checking if IP-Filter header files are installed" >&5
if test "$ac_cv_header_ip_compat_h" = "yes" &&
test "$ac_cv_header_ip_fil_h" = "yes" &&
test "$ac_cv_header_ip_nat_h" = "yes" ; then
esac
fi
echo $ac_n "checking if GNUregex needs to be compiled""... $ac_c" 1>&6
-echo "configure:4574: checking if GNUregex needs to be compiled" >&5
+echo "configure:4566: checking if GNUregex needs to be compiled" >&5
if test -z "$USE_GNUREGEX"; then
if test "$ac_cv_func_regcomp" = "no" || test "$USE_GNUREGEX" = "yes" ; then
USE_GNUREGEX="yes"
else
cat > conftest.$ac_ext <<EOF
-#line 4580 "configure"
+#line 4572 "configure"
#include "confdefs.h"
#include <sys/types.h>
#include <regex.h>
regex_t t; regcomp(&t,"",0);
; return 0; }
EOF
-if { (eval echo configure:4588: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:4580: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
USE_GNUREGEX="no"
else
do
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
-echo "configure:4619: checking for $ac_func" >&5
+echo "configure:4611: checking for $ac_func" >&5
if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 4624 "configure"
+#line 4616 "configure"
#include "confdefs.h"
/* System header to define __stub macros and hopefully few prototypes,
which can conflict with char $ac_func(); below. */
; return 0; }
EOF
-if { (eval echo configure:4647: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:4639: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
eval "ac_cv_func_$ac_func=yes"
else
echo $ac_n "checking Default FD_SETSIZE value""... $ac_c" 1>&6
-echo "configure:4675: checking Default FD_SETSIZE value" >&5
+echo "configure:4667: checking Default FD_SETSIZE value" >&5
if test "$cross_compiling" = yes; then
DEFAULT_FD_SETSIZE=256
else
cat > conftest.$ac_ext <<EOF
-#line 4680 "configure"
+#line 4672 "configure"
#include "confdefs.h"
#if HAVE_STDIO_H
}
EOF
-if { (eval echo configure:4705: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4697: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
DEFAULT_FD_SETSIZE=`cat conftestval`
else
echo $ac_n "checking Maximum number of filedescriptors we can open""... $ac_c" 1>&6
-echo "configure:4724: checking Maximum number of filedescriptors we can open" >&5
+echo "configure:4716: checking Maximum number of filedescriptors we can open" >&5
TLDFLAGS="$LDFLAGS"
case $host in
i386-unknown-freebsd*)
SQUID_MAXFD=256
else
cat > conftest.$ac_ext <<EOF
-#line 4736 "configure"
+#line 4728 "configure"
#include "confdefs.h"
#include <stdio.h>
}
EOF
-if { (eval echo configure:4793: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4785: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
SQUID_MAXFD=`cat conftestval`
else
LDFLAGS="$TLDFLAGS"
echo $ac_n "checking Default UDP send buffer size""... $ac_c" 1>&6
-echo "configure:4820: checking Default UDP send buffer size" >&5
+echo "configure:4812: checking Default UDP send buffer size" >&5
if test "$cross_compiling" = yes; then
SQUID_UDP_SO_SNDBUF=16384
else
cat > conftest.$ac_ext <<EOF
-#line 4825 "configure"
+#line 4817 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:4846: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4838: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
SQUID_UDP_SO_SNDBUF=`cat conftestval`
else
echo $ac_n "checking Default UDP receive buffer size""... $ac_c" 1>&6
-echo "configure:4865: checking Default UDP receive buffer size" >&5
+echo "configure:4857: checking Default UDP receive buffer size" >&5
if test "$cross_compiling" = yes; then
SQUID_UDP_SO_RCVBUF=16384
else
cat > conftest.$ac_ext <<EOF
-#line 4870 "configure"
+#line 4862 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:4891: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4883: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
SQUID_UDP_SO_RCVBUF=`cat conftestval`
else
echo $ac_n "checking Default TCP send buffer size""... $ac_c" 1>&6
-echo "configure:4910: checking Default TCP send buffer size" >&5
+echo "configure:4902: checking Default TCP send buffer size" >&5
if test "$cross_compiling" = yes; then
SQUID_TCP_SO_SNDBUF=16384
else
cat > conftest.$ac_ext <<EOF
-#line 4915 "configure"
+#line 4907 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:4936: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4928: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
SQUID_TCP_SO_SNDBUF=`cat conftestval`
else
echo $ac_n "checking Default TCP receive buffer size""... $ac_c" 1>&6
-echo "configure:4955: checking Default TCP receive buffer size" >&5
+echo "configure:4947: checking Default TCP receive buffer size" >&5
if test "$cross_compiling" = yes; then
SQUID_TCP_SO_RCVBUF=16384
else
cat > conftest.$ac_ext <<EOF
-#line 4960 "configure"
+#line 4952 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:4981: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:4973: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
SQUID_TCP_SO_RCVBUF=`cat conftestval`
else
echo $ac_n "checking if sys_errlist is already defined""... $ac_c" 1>&6
-echo "configure:5000: checking if sys_errlist is already defined" >&5
+echo "configure:4992: checking if sys_errlist is already defined" >&5
if eval "test \"`echo '$''{'ac_cv_needs_sys_errlist'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 5005 "configure"
+#line 4997 "configure"
#include "confdefs.h"
#include <stdio.h>
int main() {
char *s = sys_errlist;
; return 0; }
EOF
-if { (eval echo configure:5012: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:5004: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_needs_sys_errlist="no"
else
fi
echo $ac_n "checking for libresolv _dns_ttl_ hack""... $ac_c" 1>&6
-echo "configure:5034: checking for libresolv _dns_ttl_ hack" >&5
+echo "configure:5026: checking for libresolv _dns_ttl_ hack" >&5
cat > conftest.$ac_ext <<EOF
-#line 5036 "configure"
+#line 5028 "configure"
#include "confdefs.h"
extern int _dns_ttl_;
int main() {
return _dns_ttl_;
; return 0; }
EOF
-if { (eval echo configure:5043: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+if { (eval echo configure:5035: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
rm -rf conftest*
echo "$ac_t""yes" 1>&6
cat >> confdefs.h <<\EOF
rm -f conftest*
echo $ac_n "checking if inet_ntoa() actually works""... $ac_c" 1>&6
-echo "configure:5059: checking if inet_ntoa() actually works" >&5
+echo "configure:5051: checking if inet_ntoa() actually works" >&5
if test "$cross_compiling" = yes; then
INET_NTOA_RESULT="broken"
else
cat > conftest.$ac_ext <<EOF
-#line 5064 "configure"
+#line 5056 "configure"
#include "confdefs.h"
#include <stdlib.h>
}
EOF
-if { (eval echo configure:5083: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+if { (eval echo configure:5075: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
then
INET_NTOA_RESULT=`cat conftestval`
else
if test "$ac_cv_header_sys_statvfs_h" = "yes" ; then
echo $ac_n "checking for working statvfs() interface""... $ac_c" 1>&6
-echo "configure:5109: checking for working statvfs() interface" >&5
+echo "configure:5101: checking for working statvfs() interface" >&5
cat > conftest.$ac_ext <<EOF
-#line 5111 "configure"
+#line 5103 "configure"
#include "confdefs.h"
#include <stdlib.h>
; return 0; }
EOF
-if { (eval echo configure:5128: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:5120: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_func_statvfs=yes
else
fi
echo $ac_n "checking for _res.nsaddr_list""... $ac_c" 1>&6
-echo "configure:5148: checking for _res.nsaddr_list" >&5
+echo "configure:5140: checking for _res.nsaddr_list" >&5
if eval "test \"`echo '$''{'ac_cv_have_res_nsaddr_list'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 5153 "configure"
+#line 5145 "configure"
#include "confdefs.h"
#if HAVE_SYS_TYPES_H
_res.nsaddr_list[0];
; return 0; }
EOF
-if { (eval echo configure:5176: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:5168: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_res_nsaddr_list="yes"
else
if test $ac_cv_have_res_nsaddr_list = "no" ; then
echo $ac_n "checking for _res.ns_list""... $ac_c" 1>&6
-echo "configure:5198: checking for _res.ns_list" >&5
+echo "configure:5190: checking for _res.ns_list" >&5
if eval "test \"`echo '$''{'ac_cv_have_res_ns_list'+set}'`\" = set"; then
echo $ac_n "(cached) $ac_c" 1>&6
else
cat > conftest.$ac_ext <<EOF
-#line 5203 "configure"
+#line 5195 "configure"
#include "confdefs.h"
#if HAVE_SYS_TYPES_H
_res.ns_list[0].addr;
; return 0; }
EOF
-if { (eval echo configure:5226: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+if { (eval echo configure:5218: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
rm -rf conftest*
ac_cv_have_res_ns_list="yes"
else
./scripts/RunCache \
./scripts/RunAccel \
./src/Makefile \
+ ./src/fs/Makefile \
+ ./src/fs/ufs/Makefile \
+ ./src/fs/aufs/Makefile \
+ ./src/fs/coss/Makefile \
+ ./src/fs/diskd/Makefile \
./contrib/Makefile \
$SNMP_MAKEFILE \
./icons/Makefile \
s%@CC@%$CC%g
s%@LIBDLMALLOC@%$LIBDLMALLOC%g
s%@LIB_MALLOC@%$LIB_MALLOC%g
-s%@ASYNC_OBJS@%$ASYNC_OBJS%g
s%@SQUID_PTHREAD_LIB@%$SQUID_PTHREAD_LIB%g
+s%@STORE_MODULES@%$STORE_MODULES%g
+s%@STORE_OBJS@%$STORE_OBJS%g
s%@DELAY_OBJS@%$DELAY_OBJS%g
s%@SNMPLIB@%$SNMPLIB%g
s%@SNMP_OBJS@%$SNMP_OBJS%g
s%@ERR_LANGUAGE@%$ERR_LANGUAGE%g
s%@LEAKFINDER_OBJS@%$LEAKFINDER_OBJS%g
s%@AUTH_MODULES@%$AUTH_MODULES%g
-s%@OPT_DISKD_EXE@%$OPT_DISKD_EXE%g
-s%@DISKD_OBJS@%$DISKD_OBJS%g
s%@CPP@%$CPP%g
s%@INSTALL_PROGRAM@%$INSTALL_PROGRAM%g
s%@INSTALL_SCRIPT@%$INSTALL_SCRIPT%g
./scripts/RunCache \
./scripts/RunAccel \
./src/Makefile \
+ ./src/fs/Makefile \
+ ./src/fs/ufs/Makefile \
+ ./src/fs/aufs/Makefile \
+ ./src/fs/coss/Makefile \
+ ./src/fs/diskd/Makefile \
./contrib/Makefile \
$SNMP_MAKEFILE \
./icons/Makefile \
dnl
dnl Duane Wessels, wessels@nlanr.net, February 1996 (autoconf v2.9)
dnl
-dnl $Id: configure.in,v 1.185 2000/05/02 19:22:00 hno Exp $
+dnl $Id: configure.in,v 1.186 2000/05/03 17:15:39 adrian Exp $
dnl
dnl
dnl
AC_INIT(src/main.c)
AC_CONFIG_HEADER(include/autoconf.h)
-AC_REVISION($Revision: 1.185 $)dnl
+AC_REVISION($Revision: 1.186 $)dnl
AC_PREFIX_DEFAULT(/usr/local/squid)
AC_CONFIG_AUX_DIR(cfgaux)
fi
])
-AC_ARG_ENABLE(async_io,
+AC_ARG_ENABLE(async-io,
[ --enable-async-io[=N_THREADS]
- Do ASYNC disk I/O using threads.
- N_THREADS is the number of worker threads
- defaults to 16. See also src/squid.h for
- some additional platform tuning],
-[ case "$enableval" in
+ Shorthand for
+ --with-aio-threads=N_THREADS
+ --with-pthreads
+ --enable-storeio=ufs,aufs],
+[ case $enableval in
yes)
- async_io=yes
+ with_pthreads=yes
+ STORE_MODULES="ufs aufs"
;;
no)
- async_io=''
;;
*)
- async_io=yes
- AC_DEFINE_UNQUOTED(NUMTHREADS, $enableval)
+ async_io_threads=$enableval
+ with_pthreads=yes
+ STORE_MODULES="ufs aufs"
;;
esac
])
-if test -n "$async_io" ; then
- echo "Async I/O enabled"
- async_io=yes
- AC_DEFINE(USE_ASYNC_IO)
- ASYNC_OBJS='$(ASYNC_OBJS)'
+AC_ARG_WITH(aio-threads,
+[ --with-aio-threads=N_THREADS
+ Tune the number of worker threads for the aufs object
+ store.],
+[ async_io_threads=$withval ])
+if test "$async_io_threads"; then
+ echo "With $async_io_threads AIO threads"
+ with_pthreads=yes
+ AC_DEFINE_UNQUOTED(ASYNC_IO_THREADS,$async_io_threads)
+fi
+
+AC_ARG_WITH(pthreads,
+[ --with-pthreads Use POSIX Threads],
+[ if test "$enableval" = "yes"; then
+ with_pthreads=yes
+ fi
+])
+if test "$with_pthreads"; then
+ echo "With pthreads"
SQUID_PTHREAD_LIB='$(PTHREADLIB)'
CFLAGS="$CFLAGS -D_REENTRANT"
case "$host" in
i386-unknown-freebsd*)
- if test "$GCC" = "yes" ; then
- if test -z "$PRESET_LDFLAGS"; then
- LDFLAGS="$LDFLAGS -pthread"
- fi
- fi
- ;;
+ if test "$GCC" = "yes" ; then
+ if test -z "$PRESET_LDFLAGS"; then
+ LDFLAGS="$LDFLAGS -pthread"
+ fi
+ fi
+ ;;
esac
fi
-AC_SUBST(ASYNC_OBJS)
AC_SUBST(SQUID_PTHREAD_LIB)
+AC_ARG_ENABLE(storeio,
+[ --enable-storeio=\"list of modules\"
+ Build support for the list of store I/O modules.
+ The default is only to build the "ufs" module.
+ See src/fs for a list of available modules, or
+ Programmers Guide section <not yet written>
+ for details on how to build your custom store module],
+[ case $enableval in
+ yes)
+ for module in $srcdir/src/fs/*; do
+ if test -f $module/Makefile.in; then
+ STORE_MODULES="$STORE_MODULES `basename $module`"
+ fi
+ done
+ ;;
+ no)
+ ;;
+ *) STORE_MODULES="`echo $enableval| sed -e 's/,/ /g;s/ */ /g'`"
+ ;;
+ esac
+],
+[ if test -z "$STORE_MODULES"; then
+ STORE_MODULES="ufs"
+ fi
+])
+echo "Store moules built: $STORE_MODULES"
+AC_SUBST(STORE_MODULES)
+STORE_OBJS="fs/`echo $STORE_MODULES|sed -e's% %.a fs/%g'`.a"
+AC_SUBST(STORE_OBJS)
+
AC_ARG_ENABLE(icmp,
[ --enable-icmp Enable ICMP pinging],
[ if test "$enableval" = "yes" ; then
],[ERR_LANGUAGE="English"])
AC_SUBST(ERR_LANGUAGE)
+dnl Size of COSS memory buffer
+AC_ARG_WITH(coss-membuf-size,
+[ --with-coss-membuf-size COSS membuf size (default 1048576 bytes) ],
+[ if test "$with_coss_membuf_size"; then
+ echo "Setting COSS membuf size to $with_coss_membuf_size bytes"
+ AC_DEFINE_UNQUOTED(COSS_MEMBUF_SZ, $with_coss_membuf_size)
+ fi
+])
+
dnl Enable poll()
AC_ARG_ENABLE(poll,
[ --enable-poll Enable poll() instead of select(). Normally poll
[ --enable-heap-replacement
This option allows you to use various cache
replacement algorithms, instead of the standard
- LRU algorithm.
- ],
+ LRU algorithm.],
[ if test "$enableval" = "yes" ; then
echo "Enabling HEAP_REPLACEMENT"
AC_DEFINE(HEAP_REPLACEMENT, 1)
fi
AC_SUBST(AUTH_MODULES)
-dnl Enable "diskd" code
-AC_ARG_ENABLE(diskd,
-[ --enable-diskd Uses shared memory, message queues, and external
- processes for disk I/O.],
-[ if test "$enableval" = "yes" ; then
- echo "DISKD enabled"
- AC_DEFINE(USE_DISKD, 1)
- OPT_DISKD_EXE='$(DISKD_EXE)'
- AC_SUBST(OPT_DISKD_EXE)
- DISKD_OBJS='diskd.o'
- AC_SUBST(DISKD_OBJS)
- fi
-])
-
# Force some compilers to use ANSI features
#
case "$host" in
mktime \
mstats \
poll \
+ pthread_attr_setscope \
+ pthread_setschedparam \
+ pthread_attr_setschedparam \
putenv \
random \
regcomp \
vsnprintf \
)
-if test "$async_io" = "yes" ; then
- AC_CHECK_FUNCS(\
- pthread_attr_setscope \
- pthread_setschedparam \
- pthread_attr_setschedparam \
- )
-fi
-
dnl Yay! Another Linux brokenness. Its not good enough
dnl to know that setresuid() exists, because RedHat 5.0 declares
dnl setresuid() but doesn't implement it.
./scripts/RunCache \
./scripts/RunAccel \
./src/Makefile \
+ ./src/fs/Makefile \
+ ./src/fs/ufs/Makefile \
+ ./src/fs/aufs/Makefile \
+ ./src/fs/coss/Makefile \
+ ./src/fs/diskd/Makefile \
./contrib/Makefile \
$SNMP_MAKEFILE \
./icons/Makefile \
<article>
<title>Squid Programmers Guide</title>
<author>Duane Wessels, Squid Developers
-<date>$Id: prog-guide.sgml,v 1.26 2000/04/27 07:58:09 adrian Exp $</date>
+<date>$Id: prog-guide.sgml,v 1.27 2000/05/03 17:15:40 adrian Exp $</date>
<abstract>
Squid is a WWW Cache application developed by the National Laboratory
Squid can quickly locate cached objects because it keeps
(in memory) a hash table of all <em/StoreEntry/'s. The
keys for the hash table are MD5 checksums of the objects
- URI. In addition there is also a doubly-linked list of
- <em/StoreEntry/'s used for the LRU replacement algorithm.
- When an entry is accessed, it is moved to the head of the
- LRU list. When Squid needs to replace cached objects, it
- takes objects from the tail of the LRU list.
+ URI. In addition there is also a storage policy such
+ as LRU that keeps track of the objects and determines
+ the removal order when space needs to be reclaimed.
+ For the LRU policy this is implemented as a doubly linked
+ list.
<P>
- Objects are saved to disk in a two-level directory structure.
- For each object the <em/StoreEntry/ includes a 4-byte
- <em/fileno/ field. This file number is converted to a disk
- pathname by a simple algorithm which evenly distributes
- the files across all cache directories. A cache swap file
- consists of two parts: the cache metadata, and the object
- data. Note the object data includes the full HTTP
- reply---headers and body. The HTTP reply headers are not
- the same as the cache metadata.
+ For each object the <em/StoreEntry/ maps to a cache_dir
+ and location via sdirn and sfilen. For the "ufs" store
+ this file number (sfilen) is converted to a disk pathname
+ by a simple modulo of L2 and L1, but other storage drivers may
+ map sfilen in other ways. A cache swap file consists
+ of two parts: the cache metadata, and the object data.
+ Note the object data includes the full HTTP reply---headers
+ and body. The HTTP reply headers are not the same as the
+ cache metadata.
<P>
Client-side requests register themselves with a <em/StoreEntry/
<!-- %%%% Chapter : CLIENT REQUEST PROCESSING %%%% -->
<sect>Processing Client Requests
+ <P>
+ To be written...
+
<!-- %%%% Chapter : STORAGE MANAGER %%%% -->
<sect>Storage Manager
-<!-- %%%% Chapter : FILESYSTEM INTERFACE %%%% -->
-<sect>Filesystem Interface
+<sect1>Introduction
+
+ <P>
+ The Storage Manager is the glue between client and server
+ sides. Every object saved in the cache is allocated a
+ <em/StoreEntry/ structure. While the object is being
+ accessed, it also has a <em/MemObject/ structure.
+
+ <P>
+ Squid can quickly locate cached objects because it keeps
+ (in memory) a hash table of all <em/StoreEntry/'s. The
+ keys for the hash table are MD5 checksums of the objects
+ URI. In addition there is also a storage policy such
+ as LRU that keeps track of the objects and determines
+ the removal order when space needs to be reclaimed.
+ For the LRU policy this is implemented as a doubly linked
+ list.
+
+ <P>
+ For each object the <em/StoreEntry/ maps to a cache_dir
+ and location via sdirn and sfilen. For the "ufs" store
+ this file number (sfilen) is converted to a disk pathname
+ by a simple modulo of L2 and L1, but other storage drivers may
+ map sfilen in other ways. A cache swap file consists
+ of two parts: the cache metadata, and the object data.
+ Note the object data includes the full HTTP reply---headers
+ and body. The HTTP reply headers are not the same as the
+ cache metadata.
+
+ <P>
+ Client-side requests register themselves with a <em/StoreEntry/
+ to be notified when new data arrives. Multiple clients
+ may receive data via a single <em/StoreEntry/. For POST
+ and PUT request, this process works in reverse. Server-side
+ functions are notified when additional data is read from
+ the client.
+
+<sect1>Object storage
+
+ <P>
+ To be written...
+
+<sect1>Object retreival
+
+ <P>
+ To be written...
+
+<!-- %%%% Chapter : STORAGE INTERFACE %%%% -->
+<sect>Storage Interface
<sect1>Introduction
We want to try out our own, customized filesystems with Squid.
In order to do that, we need a well-defined interface
for the bits of Squid that access the permanent storage
- devices.
+ devices. We also require tigher control of the replacement
+ policy by each storage module, rather than a single global
+ replacement policy.
+
+<sect1>Build structure
+
+ <P>
+ The storage types live in squid/src/fs/ . Each subdirectory corresponds
+ to the name of the storage type. When a new storage type is implemented
+ configure.in must be updated to autogenerate a Makefile in
+ squid/src/fs/$type/ from a Makefile.in file.
-<sect1>The Interface
+ <P>
+ configure will take a list of storage types through the
+ <em/--enable-store-io/ parameter. This parameter takes a list of
+ space seperated storage types. For example,
+ --enable-store-io="ufs coss" .
-<sect2>Data Structures
+ <P>
+ Each storage type must create an archive file
+ <tt/in squid/src/fs/$type/.a . This file is automatically linked into
+ squid at compile time.
-<sect3><em/storeIOState/
+ <P>
+ Each storefs must export a function named <tt/storeFsSetup_$type()/.
+ This function is called at runtime to initialise each storage type.
+ The list of storage types is passed through <tt/store_modules.sh/
+ to generate the initialisation function <tt/storeFsSetup()/. This
+ function lives in <tt/store_modules.c/.
<P>
- Every cache object that is ``opened'' for reading or writing
- will have a <em/storeIOState/ data structure associated with
- it. Currently, this structure looks like:
+ An example of the automatically generated file:
+
<verb>
- struct _storeIOState {
- sfileno swap_file_number;
- mode_t mode;
- size_t st_size; /* do stat(2) after read open */
- off_t offset; /* current offset pointer */
- STIOCB *callback;
- void *callback_data;
- struct {
- STRCB *callback;
- void *callback_data;
- } read;
- struct {
- unsigned int closing:1; /* debugging aid */
- } flags;
- union {
- struct {
- int fd;
- struct {
- unsigned int close_request:1;
- unsigned int reading:1;
- unsigned int writing:1;
- } flags;
- } ufs;
- } type;
- };
+ /* automatically generated by ./store_modules.sh ufs coss
+ * do not edit
+ */
+ #include "squid.h"
+
+ extern STSETUP storeFsSetup_ufs;
+ extern STSETUP storeFsSetup_coss;
+ void storeFsSetup(void)
+ {
+ storeFsAdd("ufs", storeFsSetup_ufs);
+ storeFsAdd("coss", storeFsSetup_coss);
+ }
</verb>
- <em/swap_file_number/ is the 32-bit swap file number for the
- object, taken from the <em/StoreEntry/.
- <em/mode/ is either O_RDONLY or O_WRONLY.
+<sect1>Initialisation of a storage type
+
+ <P>
+ Each storage type initialises through the <tt/storeFsSetup_$type()/
+ function. The <tt/storeFsSetup_$type()/ function takes a single
+ argument - a <tt/storefs_entry_t/ pointer. This pointer references
+ the storefs_entry to initialise. A typical setup function is as
+ follows:
+
+<verb>
+ void
+ storeFsSetup_ufs(storefs_entry_t *storefs)
+ {
+ assert(!ufs_initialised);
+ storefs->parsefunc = storeUfsDirParse;
+ storefs->reconfigurefunc = storeUfsDirReconfigure;
+ storefs->donefunc = storeUfsDirDone;
+ ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t));
+ ufs_initialised = 1;
+ }
+</verb>
+
+ <P>
+ There are five function pointers in the storefs_entry which require
+ initialising. In this example, some protection is made against the
+ setup function being called twice, and a memory pool is initialised
+ for use inside the storage module.
- <em/offset/ represents the file (byte) offset after the
- last operation completed. For example, after a read operation,
- <em/offset/ must be incremented by the number of bytes read.
- The same goes for write operations. This means that the
- filesystem layer needs explicit (callback) notification
- for writes. It is wrong to increment <em/offset/ before
- an I/O operation has been known to succeed.
+ <P>
+ Each function will be covered below.
- <em/st_size/ is filled in with the object's on-disk size
- after an object is opened for reading. This allows
- the upper layers to double-check that the disk object
- actually belongs to the StoreEntry.
- Note that there are two callback functions. The first,
- <em/callback/, of type <em/STIOCB/ (store I/O callback),
- is callback for the <em/storeIOState/ as a whole. This
- callback is used to indicate success or failure of accessing
- the object, whether its for reading or writing.
- There are no callbacks for open and write operations,
- unless they fail.
+<sect2>done
- The second, <em/read.callback/, of type <em/STRCB/ (store
- read callback) is used for every read operation.
+ <P>
+<verb>
+ typedef void
+ STFSSHUTDOWN(void);
+</verb>
- The ugly union is used to hold filesystem-specific state
- information.
+ <P>
+ This function is called whenever the storage system is to be shut down.
+ It should take care of deallocating any resources currently allocated.
- <em/storeIOState/ structures are allocated by calling
- <tt/storeOpen()/, and will be deallocated by the
- filesystem layer after
- <tt/storeClose()/ is called.
-<sect2>External Functions
+<verb>
+ typedef void STFSPARSE(SwapDir *SD, int index, char *path);
+ typedef void STFSRECONFIGURE(SwapDir *SD, int index, char *path);
+</verb>
-<sect3>Object I/O
+ <P>
+ These functions handle configuring and reconfiguring a storage
+ directory. Additional arguments from the cache_dir configuration
+ line can be retrieved through calls to strtok() and GetInteger().
<P>
- These functions all relate to per-object I/O tasks: opening,
- closing, reading, writing, and unlinking objects on disk.
- These functions can all be found in <tt/store_io.c/.
+ <em/STFSPARSE/ has the task of initialising a new swapdir. It should
+ parse the remaining arguments on the cache_dir line, initialise the
+ relevant function pointers and data structures, and choose the
+ replacement policy. <em/STFSRECONFIGURE/ deals with reconfiguring an
+ active swapdir. It should parse the remaining arguments on the
+ cache_dir line and change any active configuration parameters. The
+ actual storage initialisation is done through the <em/STINIT/ function
+ pointer in the SwapDir.
<P>
- Note that the underlying storage system functions are
- accessed through function pointers, kept in the
- <em/SwapDir/ structure:
<verb>
- struct _SwapDir {
- ....
- struct {
- STOBJOPEN *open;
- STOBJCLOSE *close;
- STOBJREAD *read;
- STOBJWRITE *write;
- STOBJUNLINK *unlink;
- } obj;
- ....
- };
+ struct _SwapDir {
+ char *type; /* Pointer to the store dir type string */
+ int cur_size; /* Current swapsize in kb */
+ int low_size; /* ?? */
+ int max_size; /* Maximum swapsize in kb */
+ char *path; /* Path to store */
+ int index; /* This entry's index into the swapDir array */
+ int suggest; /* Suggestion for UFS style stores (??) */
+ size_t max_objsize; /* Maximum object size for this store */
+ union { /* Replacement policy-specific fields */
+ #ifdef HEAP_REPLACEMENT
+ struct {
+ heap *heap;
+ } heap;
+ #endif
+ struct {
+ dlink_list list;
+ dlink_node *walker;
+ } lru;
+ } repl;
+ int removals;
+ int scanned;
+ struct {
+ unsigned int selected:1; /* Currently selected for write */
+ unsigned int read_only:1; /* This store is read only */
+ } flags;
+ STINIT *init; /* Initialise the fs */
+ STNEWFS *newfs; /* Create a new fs */
+ STDUMP *dump; /* Dump fs config snippet */
+ STFREE *freefs; /* Free the fs data */
+ STDBLCHECK *dblcheck; /* Double check the obj integrity */
+ STSTATFS *statfs; /* Dump fs statistics */
+ STMAINTAINFS *maintainfs; /* Replacement maintainence */
+ STCHECKOBJ *checkob; /* Check if the fs will store an object, and get the FS load */
+ /* These two are notifications */
+ STREFOBJ *refobj; /* Reference this object */
+ STUNREFOBJ *unrefobj; /* Unreference this object */
+ STCALLBACK *callback; /* Handle pending callbacks */
+ STSYNC *sync; /* Sync the directory */
+ struct {
+ STOBJCREATE *create; /* Create a new object */
+ STOBJOPEN *open; /* Open an existing object */
+ STOBJCLOSE *close; /* Close an open object */
+ STOBJREAD *read; /* Read from an open object */
+ STOBJWRITE *write; /* Write to a created object */
+ STOBJUNLINK *unlink; /* Remove the given object */
+ } obj;
+ struct {
+ STLOGOPEN *open; /* Open the log */
+ STLOGCLOSE *close; /* Close the log */
+ STLOGWRITE *write; /* Write to the log */
+ struct {
+ STLOGCLEANOPEN *open; /* Open a clean log */
+ STLOGCLEANWRITE *write; /* Write to the log */
+ void *state; /* Current state */
+ } clean;
+ } log;
+ void *fsdata; /* FS-specific data */
+ };
+</verb>
+
+
+
+<sect1>Operation of a storage module
+
+ <P>
+ Squid understands the concept of multiple diverse storage directories.
+ Each storage directory provides a caching object store, with object
+ storage, retrieval, indexing and replacement.
+
+ <P>
+ Each open object has associated with it a <em/storeIOState/ object. The
+ <em/storeIOState/ object is used to record the state of the current
+ object. Each <em/storeIOState/ can have a storage module specific data
+ structure containing information private to the storage module.
+
+ <P>
+<verb>
+ struct _storeIOState {
+ sdirno swap_dirn; /* SwapDir index */
+ sfileno swap_filen; /* Unique file index number */
+ StoreEntry *e; /* Pointer to parent StoreEntry */
+ mode_t mode; /* Mode - O_RDONLY or O_WRONLY */
+ size_t st_size; /* Size of the object if known */
+ off_t offset; /* current _on-disk_ offset pointer */
+ STFNCB *file_callback; /* called on delayed sfileno assignments */
+ STIOCB *callback; /* IO Error handler callback */
+ void *callback_data; /* IO Error handler callback data */
+ struct {
+ STRCB *callback; /* Read completion callback */
+ void *callback_data; /* Read complation callback data */
+ } read;
+ struct {
+ unsigned int closing:1; /* debugging aid */
+ } flags;
+ void *fsstate; /* pointer to private fs state */
+ };
+</verb>
+
+ <P>
+ Each <em/SwapDir/ has the concept of a maximum object size. This is used
+ as a basic hint to the storage layer in first choosing a suitable
+ <em/SwapDir/. The checkobj function is then called for suitable
+ candidate <em/SwapDirs/ to find out whether it wants to store a
+ given <em/StoreEntry/. A <em/maxobjsize/ of -1 means 'any size'.
+
+ <P>
+ The specific filesystem operations listed in the SwapDir object are
+ covered below.
+
+<sect2>initfs
+
+ <P>
+<verb>
+ typedef void
+ STINIT(SwapDir *SD);
+</verb>
+
+ <P>
+ Initialise the given <em/SwapDir/. Operations such as verifying and
+ rebuilding the storage and creating any needed bitmaps are done
+ here.
+
+
+<sect2>newfs
+
+ <P>
+<verb>
+ typedef void
+ STNEWFS(SwapDir *SD);
+</verb>
+
+ <P>
+ Called for each configured <em/SwapDir/ to perform filesystem
+ initialisation. This happens when '-z' is given to squid on the
+ command line.
+
+
+<sect2>dumpfs
+
+ <P>
+<verb>
+ typedef void
+ STDUMP(StoreEntry *e, const char *path, SwapDir *SD);
+</verb>
+
+ <P>
+ Dump the configuration of the current <em/SwapDir/ to the given
+ <em/StoreEntry/. Used to grab a configuration file dump from th
+ <em/cachemgr/ interface. 'const char *' refers to the path of the
+ given <em/Swapdir/, and is redundant.
+
+
+<sect2>freefs
+
+ <P>
+<verb>
+ typedef void
+ STFREE(SwapDir *SD);
+</verb>
+
+ <P>
+ Free the <em/SwapDir/ filesystem information. This routine should
+ deallocate <em/SD->fsdata/.
+
+
+<sect2>doublecheckfs
+
+ <P>
+<verb>
+ typedef int
+ STDBLCHECK(SwapDir *SD, StoreEntry *e);
+</verb>
+
+ <P>
+ Double-check the given object for validity. Called during rebuild if
+ the '-S' flag is given to squid on the command line. Returns 1 if the
+ object is indeed valid, and 0 if the object is found invalid.
+
+
+<sect2>statfs
+
+ <P>
+<verb>
+ typedef void
+ STSTATFS(SwapDir *SD, StoreEntry *e);
+</verb>
+
+ <P>
+ Called to retrieve filesystem statistics, such as usage, load and
+ errors. The information should be appended to the passed
+ <em/StoreEntry/ e.
+
+
+<sect2>maintainfs
+
+ <P>
+<verb>
+ typedef void
+ STMAINTAINFS(SwapDir *SD);
+</verb>
+
+ <P>
+ Called periodically to replace objects. The active replacement policy
+ should be used to timeout unused objects in order to make room for
+ new objects.
+
+<sect2>callback
+
+ <P>
+<verb>
+ typedef void
+ STCALLBACK(SwapDir *SD);
+</verb>
+
+ <P>
+ This function is called inside the comm_select/comm_poll loop to handle
+ any callbacks pending.
+
+
+<sect2>sync
+
+ <P>
+<verb>
+ typedef void
+ STSYNC(SwapDir *SD);
+</verb>
+
+ <P>
+ This function is called whenever a sync to disk is required. This
+ function should not return until all pending data has been flushed to
+ disk.
+
+
+<sect2>parse/reconfigure
+
+ <P>
+
+<sect2>checkobj
+
+ <P>
+<verb>
+ typedef int
+ STCHECKOBJ(SwapDir *SD, const StoreEntry *e);
+</verb>
+
+ <P>
+ Called by <tt/storeDirSelectSwapDir()/ to determine whether the
+ <em/SwapDir/ will store the given <em/StoreEntry/ object. If the
+ <em/SwapDir/ is not willing to store the given <em/StoreEntry/
+ -1 should be returned. Otherwise, a value between 0 and 1000 should
+ be returned indicating the current IO load. A value of 1000 indicates
+ the <em/SwapDir/ has an IO load of 100%. This is used by
+ <tt/storeDirSelectSwapDir()/ to choose the <em/SwapDir/ with the
+ lowest IO load.
+
+
+<sect2>referenceobj
+
+ <P>
+<verb>
+ typedef void
+ STREFOBJ(SwapDir *SD, StoreEntry *e);
+</verb>
+
+ <P>
+ Called whenever an object is locked by <tt/storeLockObject()/.
+ It is typically used to update the objects position in the replacement
+ policy.
+
+
+<sect2>unreferenceobj
+
+ <P>
+<verb>
+ typedef void
+ STUNREFOBJ(SwapDir *SD, StoreEntry *e);
+</verb>
+
+ <P>
+ Called whenever the object is unlocked by <tt/storeUnlockObject()/
+ and the lock count reaches 0. It is also typically used to update the
+ objects position in the replacement policy.
+
+
+<sect2>createobj
+
+ <P>
+<verb>
+ typedef storeIOState *
+ STOBJCREATE(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data);
+</verb>
+
+ <P>
+ Create an object in the <em/SwapDir/ *SD. <em/file_callback/ is called
+ whenever the filesystem allocates or reallocates the <em/swap_filen/.
+ Note - <em/STFNCB/ is called with a generic cbdata pointer, which
+ points to the <em/StoreEntry/ e. The <em/StoreEntry/ should not be
+ modified EXCEPT for the replacement policy fields.
+
+ <P>
+ The IO callback should be called when an error occurs and when the
+ object is closed. Once the IO callback is called, the <em/storeIOState/
+ becomes invalid.
+
+ <P>
+ <em/STOBJCREATE/ returns a <em/storeIOState/ suitable for writing on
+ sucess, or NULL if an error occurs.
+
+
+<sect2>openobj
+
+ <P>
+<verb>
+ typedef storeIOState *
+ STOBJOPEN(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *io_callback, void *io_callback_data);
+</verb>
+
+ <P>
+ Open the <em/StoreEntry/ in <em/SwapDir/ *SD for reading. Much the
+ same is applicable from <em/STOBJCREATE/, the major difference being
+ that the data passed to <em/file_callback/ is the relevant
+ <em/store_client/ .
+
+
+<sect2>closeobj
+
+ <P>
+<verb>
+ typedef void
+ STOBJCLOSE(SwapDir *SD, storeIOState *sio);
+</verb>
+
+ <P>
+ Close an opened object. The <em/STIOCB/ callback should be called at
+ the end of this routine.
+
+
+<sect2>readobj
+
+ <P>
+<verb>
+ typedef void
+ STOBJREAD(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *read_callback, void *read_callback_data);
</verb>
<P>
- Thus, a storage system must do something like this
- when initializing its <em/SwapDir/ structure:
+ Read part of the object of into <em/buf/. It is safe to request a read
+ when there are other pending reads or writes. <em/STRCB/ is called at
+ completion.
+
+ <P>
+ If a read operation fails, the filesystem layer notifies the
+ calling module by calling the <em/STIOCB/ callback with an
+ error status code.
+
+
+<sect2>writeobj
+
+ <P>
+<verb>
+ typedef void
+ STOBJWRITE(SwapDir *SD, storeIOState *sio, char *buf, size_t size, off_t offset, FREE *freefunc);
+</verb>
+
+ <P>
+ Write the given block of data to the given store object. <em/buf/ is
+ allocated by the caller. When the write is complete, the data is freed
+ through <em/free_func/.
+
+ <P>
+ If a write operation fails, the filesystem layer notifies the
+ calling module by calling the <em/STIOCB/ callback with an
+ error status code.
+
+
+<sect2>unlinkobj
+
+ <P>
<verb>
- SwapDir->obj.open = storeFooOpen;
- SwapDir->obj.close = storeFooClose;
- SwapDir->obj.read = storeFooRead;
- SwapDir->obj.write = storeFooWrite;
- SwapDir->obj.unlink = storeFooUnlink;
+ typedef void STOBJUNLINK(SwapDir *, StoreEntry *);
</verb>
-<sect4><tt/storeOpen()/
+ <P>
+ Remove the <em/StoreEntry/ e from the <em/SwapDir/ SD and the
+ replacement policy.
+
+
+
+<sect1>Store IO calls
+
+ <P>
+ These routines are used inside the storage manager to create and
+ retrieve objects from a storage directory.
+
+<sect2>storeCreate()
<P>
<verb>
storeIOState *
- storeOpen(sfileno f, mode_t mode, STIOCB *callback, void *callback_data)
+ storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void * callback_data)
</verb>
<P>
- <tt/storeOpen()/
- submits a request to open a cache object for reading or writing.
- <tt/f/ is the 32-bit swap file number of the cached object.
- <tt/mode/ should be either <tt/O_RDONLY/ or <tt/O_WRONLY/.
+ <tt/storeCreate/ is called to store the given <em/StoreEntry/ in
+ a storage directory.
<P>
<tt/callback/ is a function that will be called either when
or writing immediately.
<P>
- <tt/storeOpen()/ may return NULL if the requested object
- can not be openeed. In this case the <tt/callback/ function
+ <tt/storeCreate()/ may return NULL if the requested object
+ can not be created. In this case the <tt/callback/ function
will not be called.
-<sect4><tt/storeClose()/
+
+<sect2>storeOpen()
<P>
<verb>
- void
- storeClose(storeIOState *sio)
+ storeIOState *
+ storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback, void *callback_data)
</verb>
<P>
- <tt/storeClose()/
- submits a request to close the cache object. It is safe to request
- a close even if there are read or write operations pending.
- When the underlying filesystem actually closes the object,
- the <em/STIOCB/ callback (registered with <tt/storeOpen()/) will
- be called.
+ <tt/storeOpen/ is called to open the given <em/StoreEntry/ from
+ the storage directory it resides on.
-<sect4><tt/storeRead()/
+ <P>
+ <tt/callback/ is a function that will be called either when
+ an error is encountered, or when the object is closed (by
+ calling <tt/storeClose()/). If the open request is
+ successful, there is no callback. The calling module must
+ assume the open request will succeed, and may begin reading
+ or writing immediately.
<P>
+ <tt/storeOpen()/ may return NULL if the requested object
+ can not be openeed. In this case the <tt/callback/ function
+ will not be called.
+
+
+<sect2>storeRead()
+
+ <P>
<verb>
void
storeRead(storeIOState *sio, char *buf, size_t size, off_t offset, STRCB *callback, void *callback_data)
<P>
<tt/storeRead()/ is more complicated than the other functions
because it requires its own callback function to notify the
- caller when the requested data has actually been read.
+ caller when the requested data has actually been read.
<em/buf/ must be a valid memory buffer of at least <em/size/
bytes. <em/offset/ specifies the byte offset where the
read should begin. Note that with the Swap Meta Headers
the offset into the actual object data.
<P>
- The caller is responsible for allocating and freeing <em/buf/
+ The caller is responsible for allocating and freeing <em/buf/ .
-<sect4><tt/storeWrite()/
- <P>
+<sect2>storeWrite()
+
+ <P>
<verb>
void
storeWrite(storeIOState *sio, char *buf, size_t size, off_t offset, FREE *free_func)
<P>
<tt/storeWrite()/ submits a request to write a block
- of data to the disk store.
+ of data to the disk store.
The caller is responsible for allocating <em/buf/, but since
there is no per-write callback, this memory must be freed by
the lower filesystem implementation. Therefore, the caller
calling module by calling the <em/STIOCB/ callback with an
error status code.
-<sect4><tt/storeUnlink()/
- <P>
+<sect2>storeUnlink()
+
+ <P>
<verb>
- void
- storeUnlink(sfileno f)
+ void
+ storeUnlink(StoreEntry *e)
</verb>
<P>
does not need to be opened first. The filesystem
layer will remove the object if it exists on the disk.
-<sect4><tt/storeOffset()/
- <P>
+<sect2>storeOffset()
+
+ <P>
<verb>
- off_t
- storeOffset(storeIOState *sio)
+ off_t storeOffset(storeIOState *sio)
</verb>
+
+
<P>
- Returns the current byte-offset of the cache object on
- disk. For read-objects, this is the offset after the last
- successful disk read operation. For write-objects, it is
- the offset of the last successful disk write operation.
+ <tt/storeOffset()/ returns the current _ondisk_ offset. This is used to
+ determine how much of an objects memory can be freed to make way for
+ other in-transit and cached objects. You must make sure that the
+ <em/storeIOState->offset/ refers to the ondisk offset, or undefined
+ results will occur. For reads, this returns the current offset of
+ sucessfully read data, not including queued reads.
+
-<sect4><em/STIOCB/ callback
+<sect1>Callbacks
+
+<sect2><em/STIOCB/ callback
<P>
<verb>
Once the The <em/stiocb/ function has been called,
the <em/sio/ structure should not be accessed further.
-<sect4><em/STRCB/ callback
+<sect2><em/STRCB/ callback
<P>
<verb>
called if the read operation is successful. If it fails,
then the <em/STIOCB/ callback will be called instead.
-<sect3>Config file parsing
-
- <P>
- There are three functions relating to the Squid configuration
- file: parsing, dumping, and freeing.
-
- <P>
- The parse function is called at startup, and during a reconfigure,
- for a <em/cache_dir/ line. The first keyword after the <em/cache_dir/
- keyword will be a filesystem type (such as "ufs"). A switch
- statement in <tt/parse_cachedir/ will call the appropriate
- filesystem-dependent parsing function. The parsing function
- may use <tt/strtok()/ to continue reading keywords after the
- filesystem type on the <em/cache_dir/ line.
-
-
- <P>
- The ``dump'' function is used to output a configuration
- file from the in-memory configuration structure. It is
- called with a <em/SwapDir/ argument, and must append one
- line to the <em/StoreEntry/ that is holding the configuration
- file being generated.
-
- <P>
- The free function is called during a reconfigure (and at
- exit) to free up (or un-initialize) any memory or structures
- associated with the configuration line. The <em/SwapDir/
- structure includes common and private sections. The
- <tt/free_cachedir()/ function will handle freeing anything
- in the common section, and relies on a filesystem-dependent
- function to free, or un-initialize private members.
-
-<sect3>Filesystem Startup, Initialization, and State Logging
+<sect1>State Logging
<P>
- These functions deal with initializing, state
- logging, and related tasks for a squid storage system.
+ These functions deal with state
+ logging and related tasks for a squid storage system.
These functions are used (called) in <tt/store_dir.c/.
<P>
};
</verb>
-<sect4><tt/init()/
-
- <P>
-<verb>
- void
- STINIT(SwapDir *);
-</verb>
-
- <P>
- The <tt/init/ function, of type <em/STINIT/ is called by
- <tt/storeDirInit()/ when Squid first starts up. The
- <tt/init/ function should probably start the process of
- reading saved state information from disk (aka the "rebuild"
- procedure).
-
-<sect4><tt/newfs()/
-
- <P>
-<verb>
- void
- STNEWFS(SwapDir *);
-</verb>
-
- <P>
- The <tt/newfs/ function, of type <em/STNEWFS/, is used to
- prepare a cache_dir for use by squid. It is called when
- the user runs <em/squid -z/. For the Unix file system,
- the <tt/newfs/ function makes all the two-layer subdirectories.
-
-<sect4><tt/log.open()/
+<sect2><tt/log.open()/
<P>
<verb>
the state log and then re-opens them. A <em/squid -k reconfigure/
does the same.
-<sect4><tt/log.close()/
+<sect2><tt/log.close()/
<P>
<verb>
the open state-holding log files (if any) for the storage
system.
-<sect4><tt/log.write()/
+<sect2><tt/log.write()/
<P>
<verb>
This feature may not be required by some storage systems
and can be implemented as a null-function (no-op).
-<sect4><tt/log.clean.open()/
+<sect2><tt/log.clean.open()/
<P>
<verb>
keep state information for the clean-writing process, but
should not be accessed by upper layers.
-<sect4><tt/log.clean.write()/
+<sect2><tt/log.clean.write()/
<P>
<verb>
to become the official state-holding log.
+<sect1>Replacement policy implementation
+
+<P>
+The replacement policy can be updated during STOBJREAD/STOBJWRITE/STOBJOPEN/
+STOBJCLOSE as well as STREFOBJ and STUNREFOBJ. Care should be taken to
+only modify the relevant replacement policy entries in the StoreEntry.
+The responsibility of replacement policy maintainence has been moved into
+each SwapDir so that the storage code can have tight control of the
+replacement policy. Cyclic filesystems such as COSS require this tight
+coupling between the storage layer and the replacement policy.
+
+
+<sect1>Future removal policy
+
+ <P>
+ (replaces the above Replace policy)
+
+ <P>
+ The removal policy is responsible for determining in which order
+ objects are deleted when Squid needs to reclaim space for new objects.
+ Such a policy is used by a object storage for maintaining the stored
+ objects and determining what to remove to reclaim space for new objects.
+ (together they implements a replacement policy)
+
+<sect2>API
+ <P>
+ It is implemented as a modular API where a storage directory or
+ memory creates a policy of choice for maintaining it's objects,
+ and modules registerering to be used by this API.
+
+<sect3>createRemovalPolicy()
+
+<P>
+<verb>
+ RemovalPolicy policy = createRemovalPolicy(cons char *type, cons char *args)
+</verb>
+
+ <P>
+ Creates a removal policy instance where object priority can be
+ maintained
+
+ <P>
+ The returned RemovalPolicy instance is cbdata registered
+
+<sect3>policy.Free()
+
+ <P>
+<verb>
+ policy->Free(RemovalPolicy *policy)
+</verb>
+
+<P>
+ Destroys the policy instance and frees all related memory.
+
+<sect3>policy.Add()
+
+<P>
+<verb>
+ policy->Add(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node)
+</verb>
+
+ <P>
+ Adds a StoreEntry to the policy instance.
+
+ <P>
+ datap is a pointer to where policy specific data can be stored
+ for the store entry, currently the size of one (void *) pointer.
+
+<sect3>policy.Remove()
+
+<P>
+<verb>
+ policy->Remove(RemovalPolicy *policy, StoreEntry *, RemovalPolicyNode *node)
+</verb>
+
+ <P>
+ Removes a StoreEntry from the policy instance out of
+ policy order. For example when an object is replaced
+ by a newer one or is manually purged from the store.
+
+ <P>
+ datap is a pointer to where policy specific data is stored
+ for the store entry, currently the size of one (void *) pointer.
+
+<sect3>policy.Referenced()
+
+<P>
+<verb>
+ policy->Referenced(RemovalPolicy *policy, const StoreEntry *, RemovalPolicyNode *node)
+</verb>
+
+ <P>
+ Tells the policy that a StoreEntry has been referenced.
+
+ <P>
+ datap is a pointer to where policy specific data is stored
+ for the store entry, currently the size of one (void *) pointer.
+
+<sect3>policy.WalkInit()
+
+<P>
+<verb>
+ RemovalPolicyWalker walker = policy->WalkInit(RemovalPolicy *policy)
+</verb>
+
+ <P>
+ Initiates a walk of all objects in the policy instance.
+ The objects is returned in an order suitable for using
+ as reinsertion order when rebuilding the policy.
+
+ <P>
+ The returned RemovalPolicyWalker instance is cbdata registered
+
+ <P>
+ Note: The walk must be performed as an atomic operation
+ with no other policy actions interveaning, or the outcome
+ will be undefined.
+
+<sect3>walker.Next()
+
+ <P>
+<verb>
+ const StoreEntry *entry = walker->Next(RemovalPolicyWalker *walker)
+</verb>
+
+<P>
+ Gets the next object in the walk chain
+
+ <P>
+ Return NULL when there is no further objects
+
+<sect3>walker.Done()
+
+<P>
+<verb>
+ walker->Done(RemovalPolicyWalker *walker)
+</verb>
+
+ <P>
+ Finishes a walk of the maintaned objects, destroys
+ walker.
+
+<sect3>policy.PurgeInit()
+
+<P>
+<verb>
+ RemovalPurgeWalker purgewalker = policy->PurgeInit(RemovalPolicy *policy, int max_scan)
+</verb>
+
+ <P>
+ Initiates a search for removal candidates. Seach depth is indicated
+ by max_scan.
+
+ <P>
+ The returned RemovalPurgeWalker instance is cbdata registered
+
+ <P>
+ Note: The walk must be performed as an atomic operation
+ with no other policy actions interveaning, or the outcome
+ will be undefined.
+
+<sect3>purgewalker.Next()
+
+<P>
+<verb>
+ StoreEntry *entry = purgewalker->Next(RemovalPurgeWalker *purgewalker)
+</verb>
+
+ <P>
+ Gets the next object to purge. The purgewalker will remove each
+ returned object from the policy.
+
+ <P>It is the policys responsibility to verify that the object
+ isn't locked or otherwise prevented from being removed. What this
+ means is that the policy must not return objects where
+ storeEntryLocked() is true.
+
+ <P>
+ Return NULL when there is no further purgeable objects in the policy.
+
+<sect3>purgewalker.Done()
+
+<P>
+<verb>
+ purgewalker->Done(RemovalPurgeWalker *purgewalker)
+</verb>
+
+ <P>
+ Finishes a walk of the maintaned objects, destroys
+ walker and restores the policy to it's normal state.
+
+<sect2>Future removal policy implementation
+
+<sect3>Source layout
+
+<P>
+ Policy implementations resides in src/repl/<name>/, and a make in
+ such a directory must result in a object archive src/repl/<name>.a
+ containing all the objects implementing the policy.
+
+<sect3>Internal structures
+
+<sect4>RemovalPolicy
+
+<P>
+<verb>
+ typedef struct _RemovalPolicy RemovalPolicy;
+ struct _RemovalPolicy {
+ char *_type;
+ void *_data;
+ void (*add)(RemovalPolicy *policy, StoreEntry *);
+ ... /* see the API definition above */
+ };
+</verb>
+
+<P>
+ The _type member is mainly for debugging and diagnostics purposes, and
+ should be a pointer to the name of the policy (same name as used for
+ creation)
+
+<P>
+ The _data member is for storing policy specific information.
+
+<sect4>RemovalPolicyWalker
+
+<P>
+<verb>
+ typedef struct _RemovalPolicyWalker RemovalPolicyWalker;
+ struct _RemovalPolicyWalker {
+ RemovalPolicy *_policy;
+ void *_data;
+ StoreEntry *(*next)(RemovalPolicyWalker *);
+ ... /* see the API definition above */
+ };
+</verb>
+
+<sect4>RemovalPolicyNode
+
+<P>
+<verb>
+ typedef struct _RemovalPolicyNode RemovalPolicyNode;
+ struct _RemovalPolicyNode {
+ void *data;
+ };
+</verb>
+
+ Stores policy specific information about a entry. Currently
+ there is only space for a single pointer, but plans are to
+ maybe later provide more space here to allow simple policies
+ to store all their data "inline" to preserve some memory.
+
+<sect3>Policy registration
+
+<P>
+ Policies are automatically registered in the Squid binary from the
+ policy selection made by the user building Squid. In the future this
+ might get extended to support loadable modules. All registered
+ policies are available to object stores which wishes to use them.
+
+<sect3>Policy instance creation
+
+<P>
+ Each policy must implement a "create/new" function "<tt/RemovalPolicy *
+ createRemovalPolicy_<name>(char *arguments)/". This function
+ creates the policy instance and populates it with at least the API
+ methods supported. Currently all API calls are mandatory, but the
+ policy implementation must make sure to NULL fill the structure prior
+ to populating it in order to assure future API compability.
+
+<P>
+ It should also populate the _data member with a pointer to policy
+ specific data.
+
+<P>
+ Prior to returning the created instance must be registered as
+ callback-data by calling cbdataAdd().
+
+<sect3>Walker
+
+<P>
+ When a walker is created the policy populates it with at least the API
+ methods supported. Currently all API calls are mandatory, but the
+ policy implementation must make sure to NULL fill the structure prior
+ to populating it in order to assure future API compability.
+
+<P>
+ Prior to returning the created instance must be registered as
+ callback-data by calling cbdataAdd().
+
<!-- %%%% Chapter : FORWARDING SELECTION %%%% -->
<sect>Forwarding Selection
+ <P>
+ To be written...
+
<!-- %%%% Chapter : IP/FQDN CACHE %%%% -->
<sect>IP Cache and FQDN Cache
<!-- %%%% Chapter : SERVER PROTOCOLS %%%% -->
<sect>Server Protocols
<sect1>HTTP
+
+ <P>
+ To be written...
+
<sect1>FTP
+
+ <P>
+ To be written...
+
<sect1>Gopher
+
+ <P>
+ To be written...
+
<sect1>Wais
+
+ <P>
+ To be written...
+
<sect1>SSL
+
+ <P>
+ To be written...
+
<sect1>Passthrough
+ <P>
+ To be written...
+
<!-- %%%% Chapter : TIMEOUTS %%%% -->
<sect>Timeouts
+ <P>
+ To be written...
+
<!-- %%%% Chapter : EVENTS %%%% -->
<sect>Events
+ <P>
+ To be written...
+
<!-- %%%% Chapter : ACCESS CONTROLS %%%% -->
<sect>Access Controls
+ <P>
+ To be written...
+
<!-- %%%% Chapter : ICP %%%% -->
<sect>ICP
+ <P>
+ To be written...
+
<!-- %%%% Chapter : NETDB %%%% -->
<sect>Network Measurement Database
+ <P>
+ To be written...
+
<!-- %%%% Chapter : Error Pages %%%% -->
<sect>Error Pages
+ <P>
+ To be written...
+
+<!-- %%%% Chapter : Callback Data Base %%%% -->
<sect>Callback Data Database
<P>
<!-- %%%% Chapter : CACHE MANAGER %%%% -->
<sect>Cache Manager
+ <P>
+ To be written...
+
<!-- %%%% Chapter : HTTP Headers %%%% -->
<sect>HTTP Headers
#undef FORW_VIA_DB
-/* Define to use async disk I/O operations */
-#undef USE_ASYNC_IO
-
/* Defines how many threads to use for async I/O */
-#undef NUMTHREADS
+#undef ASYNC_IO_THREADS
/*
* If you want to use Squid's ICMP features (highly recommended!) then
*/
#undef HEAP_REPLACEMENT
-/*
- * Use DISKD
- */
-#undef USE_DISKD
-
/*
* message type for message queues
*/
#undef mtyp_t
+/* Define if you want to set the COSS membuf size */
+#undef COSS_MEMBUF_SZ
+
/* The number of bytes in a int. */
#undef SIZEOF_INT
/* Define if you have the srandom function. */
#undef HAVE_SRANDOM
+/* Define if you have the statfs function. */
+#undef HAVE_STATFS
+
/* Define if you have the strerror function. */
#undef HAVE_STRERROR
/* Define if you have the <sys/ioctl.h> header file. */
#undef HAVE_SYS_IOCTL_H
-/* Define if you have the <sys/msg.h> header file. */
-#undef HAVE_SYS_MSG_H
-
/* Define if you have the <sys/mount.h> header file. */
#undef HAVE_SYS_MOUNT_H
+/* Define if you have the <sys/msg.h> header file. */
+#undef HAVE_SYS_MSG_H
+
/* Define if you have the <sys/ndir.h> header file. */
#undef HAVE_SYS_NDIR_H
#
# Makefile for the Squid Object Cache server
#
-# $Id: Makefile.in,v 1.184 2000/05/02 21:12:42 hno Exp $
+# $Id: Makefile.in,v 1.185 2000/05/03 17:15:40 adrian Exp $
#
# Uncomment and customize the following to suit your needs:
#
srcdir = @srcdir@
VPATH = @srcdir@
+SUBDIRS = fs
+
# Gotta love the DOS legacy
#
SQUID_EXE = squid$(exec_suffix)
UNLINKD_EXE = unlinkd$(exec_suffix)
PINGER_EXE = pinger$(exec_suffix)
CACHEMGR_EXE = cachemgr$(cgi_suffix)
-DISKD_EXE = diskd$(exec_suffix)
-OPT_DISKD_EXE = @OPT_DISKD_EXE@
DEFAULT_PREFIX = $(prefix)
DEFAULT_CONFIG_FILE = $(sysconfdir)/squid.conf
LDFLAGS = @LDFLAGS@
XTRA_LIBS = @XTRA_LIBS@
XTRA_OBJS = @XTRA_OBJS@
+STORE_OBJS = @STORE_OBJS@
+STORE_MODULES = @STORE_MODULES@
MV = @MV@
RM = @RM@
SHELL = /bin/sh
STD_APP_LIBS = -L../lib -lmiscutil $(XTRA_LIBS)
PROGS = $(SQUID_EXE) $(CLIENT_EXE)
-UTILS = $(DNSSERVER_EXE) $(UNLINKD_EXE) $(OPT_DISKD_EXE)
+UTILS = $(DNSSERVER_EXE) $(UNLINKD_EXE) $(DISKD_EXE)
SUID_UTILS = $(PINGER_EXE)
CGIPROGS = $(CACHEMGR_EXE)
OBJS = \
access_log.o \
acl.o \
asn.o \
- @ASYNC_OBJS@ \
authenticate.o \
cache_cf.o \
CacheDigest.o \
stmem.o \
store.o \
store_io.o \
- store_io_ufs.o \
- @DISKD_OBJS@ \
store_client.o \
store_digest.o \
store_dir.o \
- store_dir_ufs.o \
store_key_md5.o \
store_log.o \
+ store_modules.o \
store_rebuild.o \
store_swapin.o \
store_swapmeta.o \
store_swapout.o \
+ store_heap_replacement.o \
string_arrays.o \
tools.o \
unlinkd.o \
DELAY_OBJS = delay_pools.o
-ASYNC_OBJS = \
- aiops.o \
- async_io.o \
- store_io_asyncufs.o
-
LEAKFINDER_OBJS = \
leakfinder.o
DEFAULTS = \
-DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\"
-all: squid.conf $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS)
+all: squid.conf store_modules $(PROGS) $(UTILS) $(SUID_UTILS) $(CGIPROGS)
$(OBJS): $(top_srcdir)/include/version.h
$(SNMP_OBJS): ../snmplib/libsnmp.a $(top_srcdir)/include/cache_snmp.h
-$(SQUID_EXE): $(OBJS)
- $(CC) -o $@ $(LDFLAGS) $(OBJS) $(SQUID_LIBS)
+$(SQUID_EXE): $(OBJS) $(STORE_OBJS)
+ $(CC) -o $@ $(LDFLAGS) $(OBJS) $(STORE_OBJS) $(SQUID_LIBS)
globals.o: globals.c Makefile
- $(CC) -c $< $(CFLAGS) -I$(srcdir) $(DEFAULTS)
+ $(CC) -c globals.c $(CFLAGS) -I$(srcdir) $(DEFAULTS)
globals.c: globals.h mk-globals-c.pl
$(PERL) $(srcdir)/mk-globals-c.pl < $(srcdir)/globals.h > $@
$(DNSSERVER_EXE): dnsserver.o
$(CC) -o $@ $(LDFLAGS) dnsserver.o $(DNSSERVER_LIBS)
-$(DISKD_EXE): diskd-daemon.o
- $(CC) -o $@ $(LDFLAGS) diskd-daemon.o $(DISKD_LIBS)
-
-diskd-daemon.o: diskd.c
- $(CC) -c $(CFLAGS) -DDISKD_DAEMON=1 $(srcdir)/diskd.c -o $@
-
$(CACHEMGR_EXE): cachemgr.o
$(CC) -o $@ $(LDFLAGS) cachemgr.o $(CLIENT_LIBS)
s%@DEFAULT_PREFIX@%$(DEFAULT_PREFIX)%g;"\
< $(srcdir)/cf.data.pre >$@
+store_modules.c: store_modules.sh Makefile
+ @sh $(srcdir)/store_modules.sh $(STORE_MODULES) >store_modules.c
+
+store_modules.o: store_modules.c
+ $(CC) -c store_modules.c $(CFLAGS) -I$(srcdir)
+
+$(STORE_OBJS): fs/stamp
+
+store_modules fs/stamp:
+ @sh -c "cd fs && $(MAKE) all"
+
install-mkdirs:
-@if test ! -d $(prefix); then \
echo "mkdir $(prefix)"; \
# Michael Lupp <mike@nemesis.saar.de> wants to know about additions
# to the install target.
install: all install-mkdirs
+ @for dir in $(SUBDIRS); do \
+ echo "Making $@ in $$dir..."; \
+ (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \
+ done
@for f in $(PROGS); do \
if test -f $(bindir)/$$f; then \
echo $(MV) $(bindir)/$$f $(bindir)/-$$f; \
clean:
-rm -rf *.o *pure_* core $(PROGS) $(UTILS) $(CGIPROGS) $(SUID_UTILS)
-rm -f cf_gen cf_gen_defines.h cf_parser.c cf.data globals.c string_arrays.c
+ -rm -f store_modules.c
+ @for dir in $(SUBDIRS); do \
+ echo "Making $@ in $$dir..."; \
+ (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \
+ done
distclean: clean
-rm -f Makefile squid.conf squid.conf.pre
-rm -f Makefile.bak
+ -rm -f tags
+ @for dir in $(SUBDIRS); do \
+ echo "Making $@ in $$dir..."; \
+ (cd $$dir ; $(MAKE) $(MFLAGS) prefix="$(prefix)" $@) || exit 1; \
+ done
tags:
ctags *.[ch] ../include/*.h ../lib/*.[ch]
-depend:
- $(MAKEDEPEND) -I../include -I. -fMakefile *.c
+depend: cf_parser.c
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile $(srcdir)/*.c
/*
- * $Id: cache_cf.cc,v 1.342 2000/05/02 20:58:30 hno Exp $
+ * $Id: cache_cf.cc,v 1.343 2000/05/03 17:15:41 adrian Exp $
*
* DEBUG: section 3 Configuration File Parsing
* AUTHOR: Harvest Derived
static int http_header_first;
static int http_header_allowed = 0;
+static void update_maxobjsize(void);
static void configDoConfigure(void);
static void parse_refreshpattern(refresh_t **);
static int parseTimeUnits(const char *unit);
return i;
}
+static void
+update_maxobjsize(void)
+{
+ int i;
+ size_t ms = -1;
+
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ if (Config.cacheSwap.swapDirs[i].max_objsize > ms)
+ ms = Config.cacheSwap.swapDirs[i].max_objsize;
+ }
+ store_maxobjsize = ms;
+}
+
int
parseConfigFile(const char *file_name)
{
#if USE_DNSSERVERS
requirePathnameExists("cache_dns_program", Config.Program.dnsserver);
#endif
-#if USE_UNLINKD
requirePathnameExists("unlinkd_program", Config.Program.unlinkd);
-#endif
if (Config.Program.redirect)
requirePathnameExists("redirect_program", Config.Program.redirect->key);
if (Config.Program.authenticate)
int i;
for (i = 0; i < swap.n_configured; i++) {
s = swap.swapDirs + i;
- switch (s->type) {
- case SWAPDIR_UFS:
- case SWAPDIR_ASYNCUFS:
- storeUfsDirDump(entry, name, s);
- break;
- default:
- debug(0, 0) ("dump_cachedir doesn't know about type %d\n",
- (int) s->type);
- break;
- }
+ s->dump(entry, name, s);
}
}
}
}
+static int
+find_fstype(char *type)
+{
+ int i;
+ for (i = 0; storefs_list[i].typestr != NULL; i++) {
+ if (strcasecmp(type, storefs_list[i].typestr) == 0) {
+ return i;
+ }
+ }
+ return (-1);
+}
+
static void
parse_cachedir(cacheSwap * swap)
{
char *type_str;
+ char *path_str;
+ SwapDir *sd;
+ int i;
+ int fs;
+ size_t maxobjsize;
+
if ((type_str = strtok(NULL, w_space)) == NULL)
self_destruct();
- if (0 == strcasecmp(type_str, "ufs")) {
- storeUfsDirParse(swap);
-#if USE_ASYNC_IO
- } else if (0 == strcasecmp(type_str, "asyncufs")) {
- storeAufsDirParse(swap);
-#endif
-#if USE_DISKD
- } else if (0 == strcasecmp(type_str, "diskd")) {
- storeDiskdDirParse(swap);
-#endif
- } else {
- fatalf("Unknown cache_dir type '%s'\n", type_str);
+
+ maxobjsize = (size_t)GetInteger();
+
+ if ((path_str = strtok(NULL, w_space)) == NULL)
+ self_destruct();
+
+ /*
+ * This bit of code is a little strange.
+ * See, if we find a path and type match for a given line, then
+ * as long as we're reconfiguring, we can just call its reconfigure
+ * function. No harm there.
+ *
+ * Trouble is, if we find a path match, but not a type match, we have
+ * a dilemma - we could gracefully shut down the fs, kill it, and
+ * create a new one of a new type in its place, BUT at this stage the
+ * fs is meant to be the *NEW* one, and so things go very strange. :-)
+ *
+ * So, we'll assume the person isn't going to change the fs type for now,
+ * and XXX later on we will make sure that its picked up.
+ *
+ * (moving around cache_dir lines will be looked at later in a little
+ * more sane detail..)
+ */
+
+ for (i = 0; i < swap->n_configured; i++) {
+ if (0 == strcasecmp(path_str, swap->swapDirs[i].path)) {
+ /* This is a little weird, you'll appreciate it later */
+ fs = find_fstype(type_str);
+ if (fs < 0) {
+ fatalf("Unknown cache_dir type '%s'\n", type_str);
+ }
+ sd = swap->swapDirs + i;
+ storefs_list[fs].reconfigurefunc(sd, i, path_str);
+ sd->max_objsize = maxobjsize;
+ update_maxobjsize();
+ return;
+ }
+ }
+
+ fs = find_fstype(type_str);
+ if (fs < 0) {
+ /* If we get here, we didn't find a matching cache_dir type */
+ fatalf("Unknown cache_dir type '%s'\n", type_str);
}
+
+ allocate_new_swapdir(swap);
+ sd = swap->swapDirs + swap->n_configured;
+ storefs_list[fs].parsefunc(sd, swap->n_configured, path_str);
+ /* XXX should we dupe the string here, in case it gets trodden on? */
+ sd->type = storefs_list[fs].typestr;
+ sd->max_objsize = maxobjsize;
+ swap->n_configured++;
+ /* Update the max object size */
+ update_maxobjsize();
}
static void
return;
for (i = 0; i < swap->n_configured; i++) {
s = swap->swapDirs + i;
- switch (s->type) {
- case SWAPDIR_UFS:
- case SWAPDIR_ASYNCUFS:
- storeUfsDirFree(s);
- break;
- default:
- debug(0, 0) ("dump_cachedir doesn't know about type %d\n",
- (int) s->type);
- break;
- }
+ s->freefs(s);
xfree(s->path);
- filemapFreeMemory(s->map);
}
safe_free(swap->swapDirs);
swap->swapDirs = NULL;
#
-# $Id: cf.data.pre,v 1.178 2000/05/02 21:38:12 hno Exp $
+# $Id: cf.data.pre,v 1.179 2000/05/03 17:15:41 adrian Exp $
#
#
# SQUID Internet Object Cache http://squid.nlanr.net/Squid/
NAME: cache_dir
TYPE: cachedir
DEFAULT: none
-DEFAULT_IF_NONE: ufs @DEFAULT_SWAP_DIR@ 100 16 256
+DEFAULT_IF_NONE: ufs -1 @DEFAULT_SWAP_DIR@ 100 16 256
LOC: Config.cacheSwap
DOC_START
Usage:
- cache_dir Type Directory-Name Mbytes Level-1 Level2
+ cache_dir Type Maxobjsize Directory-Name Mbytes Level-1 Level2
You can specify multiple cache_dir lines to spread the
cache among different disk partitions.
want to try "asyncufs" as the type. Async IO support may be
buggy, however, so beware.
+ Maxobjsize refers to the max object size this storedir supports.
+ It is used to initially choose the storedir to dump the object.
+ -1 means 'any size'.
+
'Directory' is a top-level directory where cache swap
files will be stored. If you want to use an entire disk
for caching, then this can be the mount-point directory.
default is `0' which disables sending the announcement
messages.
+ To enable announcing your cache, just uncomment the line
+ below.
+
NOCOMMENT_START
#To enable announcing your cache, just uncomment the line below.
#announce_period 1 day
/*
- * $Id: comm_select.cc,v 1.41 2000/03/06 16:23:30 wessels Exp $
+ * $Id: comm_select.cc,v 1.42 2000/05/03 17:15:41 adrian Exp $
*
* DEBUG: section 5 Socket Functions
*
#include "squid.h"
-#if USE_ASYNC_IO
-#define MAX_POLL_TIME 10
-#elif USE_DISKD
-#define MAX_POLL_TIME 10
-#else
-#define MAX_POLL_TIME 1000
-#endif
+static int MAX_POLL_TIME = 1000; /* see also comm_quick_poll_required() */
#ifndef howmany
#define howmany(x, y) (((x)+((y)-1))/(y))
getCurrentTime();
start = current_dtime;
#endif
-#if USE_ASYNC_IO
- aioCheckCallbacks();
-#endif
-#if USE_DISKD
- storeDiskdReadQueue();
-#endif
+ /* Handle any fs callbacks that need doing */
+ storeDirCallback();
#if DELAY_POOLS
FD_ZERO(&slowfds);
#endif
fd_set slowfds;
#endif
PF *hdl = NULL;
+ SwapDir *SD;
int fd;
int maxfd;
int num;
struct timeval poll_time;
double timeout = current_dtime + (msec / 1000.0);
fde *F;
+ int i;
do {
#if !ALARM_UPDATES_TIME
getCurrentTime();
#endif
-#if USE_ASYNC_IO
- aioCheckCallbacks();
-#endif
-#if USE_DISKD
- storeDiskdReadQueue();
-#endif
#if DELAY_POOLS
FD_ZERO(&slowfds);
#endif
+ /* Handle any fs callbacks that need doing */
+ storeDirCallback();
if (commCheckICPIncoming)
comm_select_icp_incoming();
if (commCheckDNSIncoming)
nwritefds--;
}
}
+
+/* Called by async-io or diskd to speed up the polling */
+void
+comm_quick_poll_required(void)
+{
+ MAX_POLL_TIME = 10;
+}
/*
- * $Id: defines.h,v 1.79 2000/05/02 20:58:30 hno Exp $
+ * $Id: defines.h,v 1.80 2000/05/03 17:15:41 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
#define LOG_DISABLE 0
#define SM_PAGE_SIZE 4096
-#define DISK_PAGE_SIZE 8192
#define EBIT_SET(flag, bit) ((void)((flag) |= ((1L<<(bit)))))
#define EBIT_CLR(flag, bit) ((void)((flag) &= ~((1L<<(bit)))))
#define _PATH_DEVNULL "/dev/null"
#endif
-#if USE_ASYNC_IO
-#ifndef NUMTHREADS
-#define NUMTHREADS 16
-#endif
-#endif
-
-#if USE_ASYNC_IO
-#undef USE_UNLINKD
-#else
-#define USE_UNLINKD 1
-#endif
-
/*
- * $Id: disk.cc,v 1.147 2000/03/06 16:23:30 wessels Exp $
+ * $Id: disk.cc,v 1.148 2000/05/03 17:15:41 adrian Exp $
*
* DEBUG: section 6 Disk I/O Routines
* AUTHOR: Harvest Derived
debug(6, 3) ("diskHandleWrite: FD %d writing %d bytes\n",
fd, (int) (fdd->write_q->len - fdd->write_q->buf_offset));
errno = 0;
+ if (fdd->write_q->file_offset != -1)
+ lseek(fd, fdd->write_q->file_offset, SEEK_SET);
len = write(fd,
fdd->write_q->buf + fdd->write_q->buf_offset,
fdd->write_q->len - fdd->write_q->buf_offset);
/*
- * $Id: enums.h,v 1.166 2000/05/02 21:38:12 hno Exp $
+ * $Id: enums.h,v 1.167 2000/05/03 17:15:41 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
MEM_ACL_NAME_LIST,
MEM_ACL_PROXY_AUTH_USER,
MEM_ACL_TIME_DATA,
- MEM_AIO_RESULT_T,
MEM_CACHEMGR_PASSWD,
#if USE_CACHE_DIGESTS
MEM_CACHE_DIGEST,
#if USE_CACHE_DIGESTS
MEM_DIGEST_FETCH_STATE,
#endif
- MEM_DISK_BUF,
MEM_DLINK_LIST,
MEM_DLINK_NODE,
MEM_DNSSERVER_T,
NETDB_EX_RTT,
NETDB_EX_HOPS
};
-
-typedef enum {
- SWAPDIR_UFS,
- SWAPDIR_ASYNCUFS,
-#if USE_DISKD
- SWAPDIR_DISKD,
-#endif
- SWAPDIR_MAX
-} swapdir_t;
/*
- * $Id: fd.cc,v 1.38 2000/03/06 16:23:31 wessels Exp $
+ * $Id: fd.cc,v 1.39 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 51 Filedescriptor Functions
* AUTHOR: Duane Wessels
{
fde *F = &fd_table[fd];
assert(fd >= 0);
-#if USE_ASYNC_IO
- if (F->flags.closing) {
- /* Reuse of a closed FD before we have noticed it is closed */
- fd_close(fd);
- }
-#endif
if (F->flags.open) {
debug(51, 1) ("WARNING: Closing open FD %4d\n", fd);
fd_close(fd);
--- /dev/null
+# Makefile for storage modules in the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/05/03 17:15:46 adrian Exp $
+#
+
+SUBDIRS = @STORE_MODULES@
+
+all:
+ @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \
+ sh -c "cd $$dir && $(MAKE) all" || exit 1; \
+ done; \
+ if [ ! -f stamp ]; then \
+ touch stamp; \
+ fi
+
+clean:
+ -rm -f *.a stamp
+ @for dir in *; do \
+ if [ -f $$dir/Makefile ]; then \
+ sh -c "cd $$dir && $(MAKE) $@" || exit 1;\
+ fi; \
+ done
+
+distclean:
+ -rm -f *.a Makefile
+ @for dir in *; do \
+ if [ -f $$dir/Makefile ]; then \
+ sh -c "cd $$dir && $(MAKE) distclean"; \
+ fi; \
+ done
+
+.DEFAULT:
+ @test -z "$(SUBDIRS)" || for dir in $(SUBDIRS); do \
+ sh -c "cd $$dir && $(MAKE) $@" || exit 1; \
+ done
--- /dev/null
+#
+# Makefile for the AUFS storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/05/03 17:15:46 adrian Exp $
+#
+
+FS = aufs
+
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(FS).a
+
+OBJS = \
+ aiops.o \
+ async_io.o \
+ store_dir_aufs.o \
+ store_io_aufs.o
+
+
+all: $(OUT)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) $(CFLAGS) -c $<
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+install:
+
+tags:
+ ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
--- /dev/null
+/*
+ * $Id: aiops.cc,v 1.1 2000/05/03 17:15:46 adrian Exp $
+ *
+ * DEBUG: section 43 AIOPS
+ * AUTHOR: Stewart Forster <slf@connect.com.au>
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "store_asyncufs.h"
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <errno.h>
+#include <dirent.h>
+#include <signal.h>
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+#define RIDICULOUS_LENGTH 4096
+
+#if defined(_SQUID_LINUX_)
+/* Linux requires proper use of mutexes or it will segfault deep in the
+ * thread libraries. Observed on Alpha SMP Linux 2.2.10-ac12.
+ */
+#define AIO_PROPER_MUTEX 1
+#endif
+
+enum _aio_thread_status {
+ _THREAD_STARTING = 0,
+ _THREAD_WAITING,
+ _THREAD_BUSY,
+ _THREAD_FAILED,
+ _THREAD_DONE
+};
+
+enum _aio_request_type {
+ _AIO_OP_NONE = 0,
+ _AIO_OP_OPEN,
+ _AIO_OP_READ,
+ _AIO_OP_WRITE,
+ _AIO_OP_CLOSE,
+ _AIO_OP_UNLINK,
+ _AIO_OP_OPENDIR,
+ _AIO_OP_STAT
+};
+
+typedef struct aio_request_t {
+ enum _aio_request_type request_type;
+ int cancelled;
+ char *path;
+ int oflag;
+ mode_t mode;
+ int fd;
+ char *bufferp;
+ char *tmpbufp;
+ int buflen;
+ off_t offset;
+ int whence;
+ int ret;
+ int err;
+ struct stat *tmpstatp;
+ struct stat *statp;
+ aio_result_t *resultp;
+ struct aio_request_t *next;
+} aio_request_t;
+
+
+typedef struct aio_thread_t {
+ pthread_t thread;
+ enum _aio_thread_status status;
+ pthread_mutex_t mutex; /* Mutex for testing condition variable */
+ pthread_cond_t cond; /* Condition variable */
+ struct aio_request_t *volatile req; /* set by main, cleared by thread */
+ struct aio_request_t *processed_req; /* reminder to main */
+ struct aio_thread_t *next;
+} aio_thread_t;
+
+
+int aio_cancel(aio_result_t *);
+int aio_open(const char *, int, mode_t, aio_result_t *);
+int aio_read(int, char *, int, off_t, int, aio_result_t *);
+int aio_write(int, char *, int, off_t, int, aio_result_t *);
+int aio_close(int, aio_result_t *);
+int aio_unlink(const char *, aio_result_t *);
+int aio_opendir(const char *, aio_result_t *);
+aio_result_t *aio_poll_done();
+int aio_sync(void);
+
+static void aio_init(void);
+static void aio_queue_request(aio_request_t *);
+static void aio_process_request_queue(void);
+static void aio_cleanup_request(aio_request_t *);
+static void *aio_thread_loop(void *);
+static void aio_do_open(aio_request_t *);
+static void aio_do_read(aio_request_t *);
+static void aio_do_write(aio_request_t *);
+static void aio_do_close(aio_request_t *);
+static void aio_do_stat(aio_request_t *);
+static void aio_do_unlink(aio_request_t *);
+#if AIO_OPENDIR
+static void *aio_do_opendir(aio_request_t *);
+#endif
+static void aio_debug(aio_request_t *);
+static void aio_poll_threads(void);
+
+static aio_thread_t *threads;
+static int aio_initialised = 0;
+
+static int request_queue_len = 0;
+static MemPool *aio_request_pool = NULL;
+static aio_request_t *request_queue_head = NULL;
+static aio_request_t *request_queue_tail = NULL;
+static aio_request_t *request_done_head = NULL;
+static aio_request_t *request_done_tail = NULL;
+static aio_thread_t *wait_threads = NULL;
+static aio_thread_t *busy_threads_head = NULL;
+static aio_thread_t *busy_threads_tail = NULL;
+static pthread_attr_t globattr;
+static struct sched_param globsched;
+static pthread_t main_thread;
+
+static void
+aio_init(void)
+{
+ int i;
+ aio_thread_t *threadp;
+
+ if (aio_initialised)
+ return;
+
+ pthread_attr_init(&globattr);
+#if HAVE_PTHREAD_ATTR_SETSCOPE
+ pthread_attr_setscope(&globattr, PTHREAD_SCOPE_SYSTEM);
+#endif
+ globsched.sched_priority = 1;
+ main_thread = pthread_self();
+#if HAVE_PTHREAD_SETSCHEDPARAM
+ pthread_setschedparam(main_thread, SCHED_OTHER, &globsched);
+#endif
+ globsched.sched_priority = 2;
+#if HAVE_PTHREAD_ATTR_SETSCHEDPARAM
+ pthread_attr_setschedparam(&globattr, &globsched);
+#endif
+
+ /* Create threads and get them to sit in their wait loop */
+ threads = xcalloc(NUMTHREADS, sizeof(aio_thread_t));
+
+ for (i = 0; i < NUMTHREADS; i++) {
+ threadp = &threads[i];
+ threadp->status = _THREAD_STARTING;
+ if (pthread_mutex_init(&(threadp->mutex), NULL)) {
+ threadp->status = _THREAD_FAILED;
+ continue;
+ }
+ if (pthread_cond_init(&(threadp->cond), NULL)) {
+ threadp->status = _THREAD_FAILED;
+ continue;
+ }
+ threadp->req = NULL;
+ threadp->processed_req = NULL;
+ if (pthread_create(&threadp->thread, &globattr, aio_thread_loop, threadp)) {
+ fprintf(stderr, "Thread creation failed\n");
+ threadp->status = _THREAD_FAILED;
+ continue;
+ }
+ threadp->next = wait_threads;
+ wait_threads = threadp;
+#if AIO_PROPER_MUTEX
+ pthread_mutex_lock(&threadp->mutex);
+#endif
+ }
+
+ /* Create request pool */
+ aio_request_pool = memPoolCreate("aio_request", sizeof(aio_request_t));
+
+ aio_initialised = 1;
+}
+
+
+static void *
+aio_thread_loop(void *ptr)
+{
+ aio_thread_t *threadp = ptr;
+ aio_request_t *request;
+ sigset_t new;
+#if !AIO_PROPER_MUTEX
+ struct timespec wait_time;
+#endif
+
+ /*
+ * Make sure to ignore signals which may possibly get sent to
+ * the parent squid thread. Causes havoc with mutex's and
+ * condition waits otherwise
+ */
+
+ sigemptyset(&new);
+ sigaddset(&new, SIGPIPE);
+ sigaddset(&new, SIGCHLD);
+#ifdef _SQUID_LINUX_THREADS_
+ sigaddset(&new, SIGQUIT);
+ sigaddset(&new, SIGTRAP);
+#else
+ sigaddset(&new, SIGUSR1);
+ sigaddset(&new, SIGUSR2);
+#endif
+ sigaddset(&new, SIGHUP);
+ sigaddset(&new, SIGTERM);
+ sigaddset(&new, SIGINT);
+ sigaddset(&new, SIGALRM);
+ pthread_sigmask(SIG_BLOCK, &new, NULL);
+
+ pthread_mutex_lock(&threadp->mutex);
+ while (1) {
+#if AIO_PROPER_MUTEX
+ while (threadp->req == NULL) {
+ threadp->status = _THREAD_WAITING;
+ pthread_cond_wait(&threadp->cond, &threadp->mutex);
+ }
+#else
+ /* The timeout is used to unlock the race condition where
+ * ->req is set between the check and pthread_cond_wait.
+ * The thread steps it's own clock on each timeout, to avoid a CPU
+ * spin situation if the main thread is suspended (paging), and
+ * squid_curtime is not being updated timely.
+ */
+ wait_time.tv_sec = squid_curtime + 1; /* little quicker first time */
+ wait_time.tv_nsec = 0;
+ while (threadp->req == NULL) {
+ threadp->status = _THREAD_WAITING;
+ pthread_cond_timedwait(&threadp->cond, &threadp->mutex, &wait_time);
+ wait_time.tv_sec += 3; /* then wait 3 seconds between each check */
+ }
+#endif
+ request = threadp->req;
+ errno = 0;
+ if (!request->cancelled) {
+ switch (request->request_type) {
+ case _AIO_OP_OPEN:
+ aio_do_open(request);
+ break;
+ case _AIO_OP_READ:
+ aio_do_read(request);
+ break;
+ case _AIO_OP_WRITE:
+ aio_do_write(request);
+ break;
+ case _AIO_OP_CLOSE:
+ aio_do_close(request);
+ break;
+ case _AIO_OP_UNLINK:
+ aio_do_unlink(request);
+ break;
+#if AIO_OPENDIR /* Opendir not implemented yet */
+ case _AIO_OP_OPENDIR:
+ aio_do_opendir(request);
+ break;
+#endif
+ case _AIO_OP_STAT:
+ aio_do_stat(request);
+ break;
+ default:
+ request->ret = -1;
+ request->err = EINVAL;
+ break;
+ }
+ } else { /* cancelled */
+ request->ret = -1;
+ request->err = EINTR;
+ }
+ threadp->req = NULL; /* tells main thread that we are done */
+ } /* while */
+ return NULL;
+} /* aio_thread_loop */
+
+static void
+aio_do_request(aio_request_t * requestp)
+{
+ if (wait_threads == NULL && busy_threads_head == NULL) {
+ fprintf(stderr, "PANIC: No threads to service requests with!\n");
+ exit(-1);
+ }
+ aio_queue_request(requestp);
+} /* aio_do_request */
+
+
+static void
+aio_queue_request(aio_request_t * requestp)
+{
+ aio_request_t *rp;
+ static int last_warn = 0;
+ static int high_start = 0;
+ static int queue_high, queue_low;
+ int i;
+
+ /* Mark it as not executed (failing result, no error) */
+ requestp->ret = -1;
+ requestp->err = 0;
+ /* Queue it on the request queue */
+ if (request_queue_head == NULL) {
+ request_queue_head = requestp;
+ request_queue_tail = requestp;
+ } else {
+ request_queue_tail->next = requestp;
+ request_queue_tail = requestp;
+ }
+ requestp->next = NULL;
+ request_queue_len += 1;
+ /* Poll done threads if needed */
+ if (wait_threads == NULL)
+ aio_poll_threads();
+ /* Kick it rolling */
+ aio_process_request_queue();
+ /* Warn if out of threads */
+ if (request_queue_len > (NUMTHREADS >> 1)) {
+ if (high_start == 0) {
+ high_start = squid_curtime;
+ queue_high = request_queue_len;
+ queue_low = request_queue_len;
+ }
+ if (request_queue_len > queue_high)
+ queue_high = request_queue_len;
+ if (request_queue_len < queue_low)
+ queue_low = request_queue_len;
+ if (squid_curtime >= (last_warn + 15) &&
+ squid_curtime >= (high_start + 3)) {
+ debug(43, 1) ("aio_queue_request: WARNING - Running out of I/O threads\n");
+ debug(43, 2) ("aio_queue_request: Queue Length: current=%d, high=%d, low=%d, duration=%d\n",
+ request_queue_len, queue_high, queue_low, squid_curtime - high_start);
+ debug(43, 1) ("aio_queue_request: Perhaps you should increase NUMTHREADS\n");
+ debug(43, 1) ("aio_queue_request: Or install more disks to share the load\n");
+ debug(43, 3) ("aio_queue_request: First %d items on request queue\n", NUMTHREADS);
+ rp = request_queue_head;
+ for (i = 1; i <= NUMTHREADS; i++) {
+ switch (rp->request_type) {
+ case _AIO_OP_OPEN:
+ debug(43, 3) ("aio_queue_request: %d : open -> %s\n", i, rp->path);
+ break;
+ case _AIO_OP_READ:
+ debug(43, 3) ("aio_queue_request: %d : read -> FD = %d\n", i, rp->fd);
+ break;
+ case _AIO_OP_WRITE:
+ debug(43, 3) ("aio_queue_request: %d : write -> FD = %d\n", i, rp->fd);
+ break;
+ case _AIO_OP_CLOSE:
+ debug(43, 3) ("aio_queue_request: %d : close -> FD = %d\n", i, rp->fd);
+ break;
+ case _AIO_OP_UNLINK:
+ debug(43, 3) ("aio_queue_request: %d : unlink -> %s\n", i, rp->path);
+ break;
+ case _AIO_OP_STAT:
+ debug(43, 3) ("aio_queue_request: %d : stat -> %s\n", i, rp->path);
+ break;
+ default:
+ debug(43, 1) ("aio_queue_request: %d : Unimplemented request type: %d\n", i, rp->request_type);
+ break;
+ }
+ if ((rp = rp->next) == NULL)
+ break;
+ }
+ last_warn = squid_curtime;
+ }
+ } else {
+ high_start = 0;
+ }
+ if (request_queue_len > RIDICULOUS_LENGTH) {
+ debug(43, 0) ("aio_queue_request: Async request queue growing uncontrollably!\n");
+ debug(43, 0) ("aio_queue_request: Syncing pending I/O operations.. (blocking)\n");
+ aio_sync();
+ debug(43, 0) ("aio_queue_request: Synced\n");
+ }
+} /* aio_queue_request */
+
+
+static void
+aio_process_request_queue(void)
+{
+ aio_thread_t *threadp;
+ aio_request_t *requestp;
+
+ for (;;) {
+ if (wait_threads == NULL || request_queue_head == NULL)
+ return;
+
+ requestp = request_queue_head;
+ if ((request_queue_head = requestp->next) == NULL)
+ request_queue_tail = NULL;
+ requestp->next = NULL;
+ request_queue_len--;
+
+ if (requestp->cancelled) {
+ aio_cleanup_request(requestp);
+ continue;
+ }
+ threadp = wait_threads;
+ wait_threads = threadp->next;
+ threadp->next = NULL;
+
+ if (busy_threads_head != NULL)
+ busy_threads_tail->next = threadp;
+ else
+ busy_threads_head = threadp;
+ busy_threads_tail = threadp;
+
+ threadp->status = _THREAD_BUSY;
+ threadp->req = threadp->processed_req = requestp;
+ pthread_cond_signal(&(threadp->cond));
+#if AIO_PROPER_MUTEX
+ pthread_mutex_unlock(&threadp->mutex);
+#endif
+ }
+} /* aio_process_request_queue */
+
+
+static void
+aio_cleanup_request(aio_request_t * requestp)
+{
+ aio_result_t *resultp = requestp->resultp;
+ int cancelled = requestp->cancelled;
+
+ /* Free allocated structures and copy data back to user space if the */
+ /* request hasn't been cancelled */
+ switch (requestp->request_type) {
+ case _AIO_OP_STAT:
+ if (!cancelled && requestp->ret == 0)
+ xmemcpy(requestp->statp, requestp->tmpstatp, sizeof(struct stat));
+ xfree(requestp->tmpstatp);
+ case _AIO_OP_OPEN:
+ if (cancelled && requestp->ret >= 0)
+ /* The open() was cancelled but completed */
+ close(requestp->ret);
+ xfree(requestp->path);
+ break;
+ case _AIO_OP_CLOSE:
+ if (cancelled && requestp->ret < 0)
+ /* The close() was cancelled and never got executed */
+ close(requestp->fd);
+ break;
+ case _AIO_OP_UNLINK:
+ case _AIO_OP_OPENDIR:
+ xfree(requestp->path);
+ break;
+ case _AIO_OP_READ:
+ if (!cancelled && requestp->ret > 0)
+ xmemcpy(requestp->bufferp, requestp->tmpbufp, requestp->ret);
+ case _AIO_OP_WRITE:
+ xfree(requestp->tmpbufp);
+ break;
+ default:
+ break;
+ }
+ if (resultp != NULL && !cancelled) {
+ resultp->aio_return = requestp->ret;
+ resultp->aio_errno = requestp->err;
+ }
+ memPoolFree(aio_request_pool, requestp);
+} /* aio_cleanup_request */
+
+
+int
+aio_cancel(aio_result_t * resultp)
+{
+ aio_thread_t *threadp;
+ aio_request_t *requestp;
+
+ for (threadp = busy_threads_head; threadp != NULL; threadp = threadp->next)
+ if (threadp->processed_req->resultp == resultp) {
+ threadp->processed_req->cancelled = 1;
+ threadp->processed_req->resultp = NULL;
+ return 0;
+ }
+ for (requestp = request_queue_head; requestp != NULL; requestp = requestp->next)
+ if (requestp->resultp == resultp) {
+ requestp->cancelled = 1;
+ requestp->resultp = NULL;
+ return 0;
+ }
+ for (requestp = request_done_head; requestp != NULL; requestp = requestp->next)
+ if (requestp->resultp == resultp) {
+ requestp->cancelled = 1;
+ requestp->resultp = NULL;
+ return 0;
+ }
+ return 1;
+} /* aio_cancel */
+
+
+int
+aio_open(const char *path, int oflag, mode_t mode, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+ int len;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ len = strlen(path) + 1;
+ if ((requestp->path = (char *) xmalloc(len)) == NULL) {
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ strncpy(requestp->path, path, len);
+ requestp->oflag = oflag;
+ requestp->mode = mode;
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_OPEN;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_open(aio_request_t * requestp)
+{
+ requestp->ret = open(requestp->path, requestp->oflag, requestp->mode);
+ requestp->err = errno;
+}
+
+
+int
+aio_read(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ requestp->fd = fd;
+ requestp->bufferp = bufp;
+ if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) {
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ requestp->buflen = bufs;
+ requestp->offset = offset;
+ requestp->whence = whence;
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_READ;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_read(aio_request_t * requestp)
+{
+ lseek(requestp->fd, requestp->offset, requestp->whence);
+ requestp->ret = read(requestp->fd, requestp->tmpbufp, requestp->buflen);
+ requestp->err = errno;
+}
+
+
+int
+aio_write(int fd, char *bufp, int bufs, off_t offset, int whence, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ requestp->fd = fd;
+ if ((requestp->tmpbufp = (char *) xmalloc(bufs)) == NULL) {
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ xmemcpy(requestp->tmpbufp, bufp, bufs);
+ requestp->buflen = bufs;
+ requestp->offset = offset;
+ requestp->whence = whence;
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_WRITE;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_write(aio_request_t * requestp)
+{
+ requestp->ret = write(requestp->fd, requestp->tmpbufp, requestp->buflen);
+ requestp->err = errno;
+}
+
+
+int
+aio_close(int fd, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ requestp->fd = fd;
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_CLOSE;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_close(aio_request_t * requestp)
+{
+ requestp->ret = close(requestp->fd);
+ requestp->err = errno;
+}
+
+
+int
+aio_stat(const char *path, struct stat *sb, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+ int len;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ len = strlen(path) + 1;
+ if ((requestp->path = (char *) xmalloc(len)) == NULL) {
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ strncpy(requestp->path, path, len);
+ requestp->statp = sb;
+ if ((requestp->tmpstatp = (struct stat *) xmalloc(sizeof(struct stat))) == NULL) {
+ xfree(requestp->path);
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_STAT;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_stat(aio_request_t * requestp)
+{
+ requestp->ret = stat(requestp->path, requestp->tmpstatp);
+ requestp->err = errno;
+}
+
+
+int
+aio_unlink(const char *path, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+ int len;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ len = strlen(path) + 1;
+ if ((requestp->path = (char *) xmalloc(len)) == NULL) {
+ memPoolFree(aio_request_pool, requestp);
+ errno = ENOMEM;
+ return -1;
+ }
+ strncpy(requestp->path, path, len);
+ requestp->resultp = resultp;
+ requestp->request_type = _AIO_OP_UNLINK;
+ requestp->cancelled = 0;
+
+ aio_do_request(requestp);
+ return 0;
+}
+
+
+static void
+aio_do_unlink(aio_request_t * requestp)
+{
+ requestp->ret = unlink(requestp->path);
+ requestp->err = errno;
+}
+
+
+#if AIO_OPENDIR
+/* XXX aio_opendir NOT implemented? */
+
+int
+aio_opendir(const char *path, aio_result_t * resultp)
+{
+ aio_request_t *requestp;
+ int len;
+
+ if (!aio_initialised)
+ aio_init();
+ if ((requestp = memPoolAlloc(aio_request_pool)) == NULL) {
+ errno = ENOMEM;
+ return -1;
+ }
+ return -1;
+}
+
+static void
+aio_do_opendir(aio_request_t * requestp)
+{
+ /* NOT IMPLEMENTED */
+}
+
+#endif
+
+
+void
+aio_poll_threads(void)
+{
+ aio_thread_t *prev;
+ aio_thread_t *threadp;
+ aio_request_t *requestp;
+
+ do { /* while found completed thread */
+ prev = NULL;
+ threadp = busy_threads_head;
+ while (threadp) {
+ debug(43, 9) ("aio_poll_threads: %p: request type %d -> status %d\n",
+ threadp,
+ threadp->processed_req->request_type,
+ threadp->status);
+#if AIO_PROPER_MUTEX
+ if (threadp->req == NULL)
+ if (pthread_mutex_trylock(&threadp->mutex) == 0)
+ break;
+#else
+ if (threadp->req == NULL)
+ break;
+#endif
+ prev = threadp;
+ threadp = threadp->next;
+ }
+ if (threadp == NULL)
+ break;
+
+ if (prev == NULL)
+ busy_threads_head = busy_threads_head->next;
+ else
+ prev->next = threadp->next;
+
+ if (busy_threads_tail == threadp)
+ busy_threads_tail = prev;
+
+ requestp = threadp->processed_req;
+ threadp->processed_req = NULL;
+
+ threadp->next = wait_threads;
+ wait_threads = threadp;
+
+ if (request_done_tail != NULL)
+ request_done_tail->next = requestp;
+ else
+ request_done_head = requestp;
+ request_done_tail = requestp;
+ } while (threadp);
+
+ aio_process_request_queue();
+} /* aio_poll_threads */
+
+aio_result_t *
+aio_poll_done(void)
+{
+ aio_request_t *requestp;
+ aio_result_t *resultp;
+ int cancelled;
+
+ AIO_REPOLL:
+ aio_poll_threads();
+ if (request_done_head == NULL) {
+ return NULL;
+ }
+ requestp = request_done_head;
+ request_done_head = requestp->next;
+ if (!request_done_head)
+ request_done_tail = NULL;
+
+ resultp = requestp->resultp;
+ cancelled = requestp->cancelled;
+ aio_debug(requestp);
+ debug(43, 5) ("DONE: %d -> %d\n", requestp->ret, requestp->err);
+ aio_cleanup_request(requestp);
+ if (cancelled)
+ goto AIO_REPOLL;
+ return resultp;
+} /* aio_poll_done */
+
+int
+aio_operations_pending(void)
+{
+ return request_queue_len + (request_done_head != NULL) + (busy_threads_head != NULL);
+}
+
+int
+aio_overloaded(void)
+{
+ static time_t last_warn = 0;
+ if (aio_operations_pending() > RIDICULOUS_LENGTH / 4) {
+ if (squid_curtime >= (last_warn + 15)) {
+ debug(43, 0) ("Warning: Async-IO overloaded\n");
+ last_warn = squid_curtime;
+ }
+ return 1;
+ }
+ return 0;
+}
+
+int
+aio_sync(void)
+{
+ int loop_count = 0;
+ do {
+ aio_poll_threads();
+ assert(++loop_count < 10);
+ } while (request_queue_len > 0);
+ return aio_operations_pending();
+}
+
+int
+aio_get_queue_len(void)
+{
+ return request_queue_len;
+}
+
+static void
+aio_debug(aio_request_t * requestp)
+{
+ switch (requestp->request_type) {
+ case _AIO_OP_OPEN:
+ debug(43, 5) ("OPEN of %s to FD %d\n", requestp->path, requestp->ret);
+ break;
+ case _AIO_OP_READ:
+ debug(43, 5) ("READ on fd: %d\n", requestp->fd);
+ break;
+ case _AIO_OP_WRITE:
+ debug(43, 5) ("WRITE on fd: %d\n", requestp->fd);
+ break;
+ case _AIO_OP_CLOSE:
+ debug(43, 5) ("CLOSE of fd: %d\n", requestp->fd);
+ break;
+ case _AIO_OP_UNLINK:
+ debug(43, 5) ("UNLINK of %s\n", requestp->path);
+ break;
+ default:
+ break;
+ }
+}
--- /dev/null
+
+/*
+ * $Id: async_io.cc,v 1.1 2000/05/03 17:15:46 adrian Exp $
+ *
+ * DEBUG: section 32 Asynchronous Disk I/O
+ * AUTHOR: Pete Bentley <pete@demon.net>
+ * AUTHOR: Stewart Forster <slf@connect.com.au>
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "store_asyncufs.h"
+
+#define _AIO_OPEN 0
+#define _AIO_READ 1
+#define _AIO_WRITE 2
+#define _AIO_CLOSE 3
+#define _AIO_UNLINK 4
+#define _AIO_OPENDIR 5
+#define _AIO_STAT 6
+
+typedef struct aio_ctrl_t {
+ struct aio_ctrl_t *next;
+ int fd;
+ int operation;
+ AIOCB *done_handler;
+ void *done_handler_data;
+ aio_result_t result;
+} aio_ctrl_t;
+
+struct {
+ int open;
+ int close;
+ int cancel;
+ int write;
+ int read;
+ int stat;
+ int unlink;
+ int check_callback;
+} aio_counts;
+
+typedef struct aio_unlinkq_t {
+ char *path;
+ struct aio_unlinkq_t *next;
+} aio_unlinkq_t;
+
+static aio_ctrl_t *used_list = NULL;
+static int initialised = 0;
+static OBJH aioStats;
+static MemPool *aio_ctrl_pool;
+static void aioFDWasClosed(int fd);
+
+MemPool * aio_state_pool;
+
+static void
+aioFDWasClosed(int fd)
+{
+ if (fd_table[fd].flags.closing)
+ fd_close(fd);
+}
+
+void
+aioInit(void)
+{
+ if (initialised)
+ return;
+ aio_ctrl_pool = memPoolCreate("aio_ctrl", sizeof(aio_ctrl_t));
+ aio_state_pool = memPoolCreate("Async UFS IO State data", sizeof(aiostate_t));
+ cachemgrRegister("aio_counts", "Async IO Function Counters",
+ aioStats, 0, 1);
+ initialised = 1;
+ comm_quick_poll_required();
+}
+
+void
+aioDone(void)
+{
+ memPoolDestroy(aio_ctrl_pool);
+ memPoolDestroy(aio_state_pool);
+ initialised = 0;
+}
+
+void
+aioOpen(const char *path, int oflag, mode_t mode, AIOCB * callback, void *callback_data)
+{
+ aio_ctrl_t *ctrlp;
+ int ret;
+
+ assert(initialised);
+ aio_counts.open++;
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = -2;
+ ctrlp->done_handler = callback;
+ ctrlp->done_handler_data = callback_data;
+ ctrlp->operation = _AIO_OPEN;
+ cbdataLock(callback_data);
+ if (aio_open(path, oflag, mode, &ctrlp->result) < 0) {
+ ret = open(path, oflag, mode);
+ if (callback)
+ (callback) (ctrlp->fd, callback_data, ret, errno);
+ cbdataUnlock(callback_data);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ return;
+ }
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ return;
+}
+
+void
+aioClose(int fd)
+{
+ aio_ctrl_t *ctrlp;
+
+ assert(initialised);
+ aio_counts.close++;
+ aioCancel(fd);
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = fd;
+ ctrlp->done_handler = NULL;
+ ctrlp->done_handler_data = NULL;
+ ctrlp->operation = _AIO_CLOSE;
+ if (aio_close(fd, &ctrlp->result) < 0) {
+ close(fd); /* Can't create thread - do a normal close */
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ aioFDWasClosed(fd);
+ return;
+ }
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ return;
+}
+
+void
+aioCancel(int fd)
+{
+ aio_ctrl_t *curr;
+ aio_ctrl_t *prev;
+ aio_ctrl_t *next;
+ AIOCB *done_handler;
+ void *their_data;
+
+ assert(initialised);
+ aio_counts.cancel++;
+ prev = NULL;
+ curr = used_list;
+ for (curr = used_list;; curr = next) {
+ while (curr != NULL) {
+ if (curr->fd == fd)
+ break;
+ prev = curr;
+ curr = curr->next;
+ }
+ if (curr == NULL)
+ break;
+
+ aio_cancel(&curr->result);
+
+ if ((done_handler = curr->done_handler)) {
+ their_data = curr->done_handler_data;
+ curr->done_handler = NULL;
+ curr->done_handler_data = NULL;
+ debug(0, 0) ("this be aioCancel\n");
+ if (cbdataValid(their_data))
+ done_handler(fd, their_data, -2, -2);
+ cbdataUnlock(their_data);
+ }
+ next = curr->next;
+ if (prev == NULL)
+ used_list = next;
+ else
+ prev->next = next;
+
+ memPoolFree(aio_ctrl_pool, curr);
+ }
+}
+
+
+void
+aioWrite(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data, FREE * free_func)
+{
+ aio_ctrl_t *ctrlp;
+ int seekmode;
+
+ assert(initialised);
+ aio_counts.write++;
+ for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next)
+ if (ctrlp->fd == fd)
+ break;
+ if (ctrlp != NULL) {
+ debug(0, 0) ("aioWrite: EWOULDBLOCK\n");
+ errno = EWOULDBLOCK;
+ if (callback)
+ (callback) (fd, callback_data, -1, errno);
+ if (free_func)
+ free_func(bufp);
+ return;
+ }
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = fd;
+ ctrlp->done_handler = callback;
+ ctrlp->done_handler_data = callback_data;
+ ctrlp->operation = _AIO_WRITE;
+ if (offset >= 0)
+ seekmode = SEEK_SET;
+ else {
+ seekmode = SEEK_END;
+ offset = 0;
+ }
+ cbdataLock(callback_data);
+ if (aio_write(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) {
+ if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL)
+ errno = EWOULDBLOCK;
+ if (callback)
+ (callback) (fd, callback_data, -1, errno);
+ cbdataUnlock(callback_data);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ } else {
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ }
+ /*
+ * aio_write copies the buffer so we can free it here
+ */
+ if (free_func)
+ free_func(bufp);
+} /* aioWrite */
+
+
+void
+aioRead(int fd, int offset, char *bufp, int len, AIOCB * callback, void *callback_data)
+{
+ aio_ctrl_t *ctrlp;
+ int seekmode;
+
+ assert(initialised);
+ aio_counts.read++;
+ for (ctrlp = used_list; ctrlp != NULL; ctrlp = ctrlp->next)
+ if (ctrlp->fd == fd)
+ break;
+ if (ctrlp != NULL) {
+ errno = EWOULDBLOCK;
+ if (callback)
+ (callback) (fd, callback_data, -1, errno);
+ return;
+ }
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = fd;
+ ctrlp->done_handler = callback;
+ ctrlp->done_handler_data = callback_data;
+ ctrlp->operation = _AIO_READ;
+ if (offset >= 0)
+ seekmode = SEEK_SET;
+ else {
+ seekmode = SEEK_CUR;
+ offset = 0;
+ }
+ cbdataLock(callback_data);
+ if (aio_read(fd, bufp, len, offset, seekmode, &ctrlp->result) < 0) {
+ if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL)
+ errno = EWOULDBLOCK;
+ if (callback)
+ (callback) (fd, callback_data, -1, errno);
+ cbdataUnlock(callback_data);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ return;
+ }
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ return;
+} /* aioRead */
+
+void
+aioStat(char *path, struct stat *sb, AIOCB * callback, void *callback_data)
+{
+ aio_ctrl_t *ctrlp;
+
+ assert(initialised);
+ aio_counts.stat++;
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = -2;
+ ctrlp->done_handler = callback;
+ ctrlp->done_handler_data = callback_data;
+ ctrlp->operation = _AIO_STAT;
+ cbdataLock(callback_data);
+ if (aio_stat(path, sb, &ctrlp->result) < 0) {
+ if (errno == ENOMEM || errno == EAGAIN || errno == EINVAL)
+ errno = EWOULDBLOCK;
+ if (callback)
+ (callback) (ctrlp->fd, callback_data, -1, errno);
+ cbdataUnlock(callback_data);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ return;
+ }
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ return;
+} /* aioStat */
+
+void
+aioUnlink(const char *pathname, AIOCB * callback, void *callback_data)
+{
+ aio_ctrl_t *ctrlp;
+ char *path;
+ assert(initialised);
+ aio_counts.unlink++;
+ ctrlp = memPoolAlloc(aio_ctrl_pool);
+ ctrlp->fd = -2;
+ ctrlp->done_handler = callback;
+ ctrlp->done_handler_data = callback_data;
+ ctrlp->operation = _AIO_UNLINK;
+ path = xstrdup(pathname);
+ cbdataLock(callback_data);
+ if (aio_unlink(path, &ctrlp->result) < 0) {
+ int ret = unlink(path);
+ (callback) (ctrlp->fd, callback_data, ret, errno);
+ cbdataUnlock(callback_data);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ xfree(path);
+ return;
+ }
+ ctrlp->next = used_list;
+ used_list = ctrlp;
+ xfree(path);
+} /* aioUnlink */
+
+
+void
+aioCheckCallbacks(SwapDir *SD)
+{
+ aio_result_t *resultp;
+ aio_ctrl_t *ctrlp;
+ aio_ctrl_t *prev;
+ AIOCB *done_handler;
+ void *their_data;
+
+ assert(initialised);
+ aio_counts.check_callback++;
+ for (;;) {
+ if ((resultp = aio_poll_done()) == NULL)
+ break;
+ prev = NULL;
+ for (ctrlp = used_list; ctrlp != NULL; prev = ctrlp, ctrlp = ctrlp->next)
+ if (&ctrlp->result == resultp)
+ break;
+ if (ctrlp == NULL)
+ continue;
+ if (prev == NULL)
+ used_list = ctrlp->next;
+ else
+ prev->next = ctrlp->next;
+ if ((done_handler = ctrlp->done_handler)) {
+ their_data = ctrlp->done_handler_data;
+ ctrlp->done_handler = NULL;
+ ctrlp->done_handler_data = NULL;
+ if (cbdataValid(their_data))
+ done_handler(ctrlp->fd, their_data,
+ ctrlp->result.aio_return, ctrlp->result.aio_errno);
+ cbdataUnlock(their_data);
+ }
+ if (ctrlp->operation == _AIO_CLOSE)
+ aioFDWasClosed(ctrlp->fd);
+ memPoolFree(aio_ctrl_pool, ctrlp);
+ }
+}
+
+void
+aioStats(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "ASYNC IO Counters:\n");
+ storeAppendPrintf(sentry, "open\t%d\n", aio_counts.open);
+ storeAppendPrintf(sentry, "close\t%d\n", aio_counts.close);
+ storeAppendPrintf(sentry, "cancel\t%d\n", aio_counts.cancel);
+ storeAppendPrintf(sentry, "write\t%d\n", aio_counts.write);
+ storeAppendPrintf(sentry, "read\t%d\n", aio_counts.read);
+ storeAppendPrintf(sentry, "stat\t%d\n", aio_counts.stat);
+ storeAppendPrintf(sentry, "unlink\t%d\n", aio_counts.unlink);
+ storeAppendPrintf(sentry, "check_callback\t%d\n", aio_counts.check_callback);
+ storeAppendPrintf(sentry, "queue\t%d\n", aio_get_queue_len());
+}
+
+/* Flush all pending I/O */
+void
+aioSync(SwapDir *SD)
+{
+ if (!initialised)
+ return; /* nothing to do then */
+ /* Flush all pending operations */
+ debug(32, 1) ("aioSync: flushing pending I/O operations\n");
+ do {
+ aioCheckCallbacks(SD);
+ } while (aio_sync());
+ debug(32, 1) ("aioSync: done\n");
+}
+
+int
+aioQueueSize(void)
+{
+ return memPoolInUseCount(aio_ctrl_pool);
+}
--- /dev/null
+/*
+ * store_aufs.h
+ *
+ * Internal declarations for the aufs routines
+ */
+
+#ifndef __STORE_ASYNCUFS_H__
+#define __STORE_ASYNCUFS_H__
+
+#ifdef ASYNC_IO_THREADS
+#define NUMTHREADS ASYNC_IO_THREADS
+#else
+#define NUMTHREADS 16
+#endif
+
+#define MAGIC1 (NUMTHREADS/Config.cacheSwap.n_configured/2)
+
+struct _aio_result_t {
+ int aio_return;
+ int aio_errno;
+};
+
+typedef struct _aio_result_t aio_result_t;
+
+typedef void AIOCB(int fd, void *, int aio_return, int aio_errno);
+
+int aio_cancel(aio_result_t *);
+int aio_open(const char *, int, mode_t, aio_result_t *);
+int aio_read(int, char *, int, off_t, int, aio_result_t *);
+int aio_write(int, char *, int, off_t, int, aio_result_t *);
+int aio_close(int, aio_result_t *);
+int aio_stat(const char *, struct stat *, aio_result_t *);
+int aio_unlink(const char *, aio_result_t *);
+int aio_opendir(const char *, aio_result_t *);
+aio_result_t *aio_poll_done(void);
+int aio_operations_pending(void);
+int aio_overloaded(void);
+int aio_sync(void);
+int aio_get_queue_len(void);
+
+void aioInit(void);
+void aioDone(void);
+void aioCancel(int);
+void aioOpen(const char *, int, mode_t, AIOCB *, void *);
+void aioClose(int);
+void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *);
+void aioRead(int, int offset, char *, int size, AIOCB *, void *);
+void aioStat(char *, struct stat *, AIOCB *, void *);
+void aioUnlink(const char *, AIOCB *, void *);
+void aioCheckCallbacks(SwapDir *);
+void aioSync(SwapDir *);
+int aioQueueSize(void);
+
+struct _aioinfo_t {
+ int swaplog_fd;
+ int l1;
+ int l2;
+ fileMap *map;
+ int suggest;
+};
+
+struct _aiostate_t {
+ int fd;
+ struct {
+ unsigned int close_request:1;
+ unsigned int reading:1;
+ unsigned int writing:1;
+ unsigned int opening:1;
+ } flags;
+ const char *read_buf;
+ link_list *pending_writes;
+ link_list *pending_reads;
+};
+
+typedef struct _aioinfo_t aioinfo_t;
+typedef struct _aiostate_t aiostate_t;
+
+/* The aio_state memory pool */
+extern MemPool * aio_state_pool;
+
+extern void storeAufsDirMapBitReset(SwapDir *, sfileno);
+extern int storeAufsDirMapBitAllocate(SwapDir *);
+
+extern char * storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath);
+extern void storeAufsDirUnlinkFile(SwapDir *, sfileno);
+extern void storeAufsDirReplAdd(SwapDir *SD, StoreEntry *);
+extern void storeAufsDirReplRemove(StoreEntry *);
+
+/*
+ * Store IO stuff
+ */
+extern STOBJCREATE storeAufsCreate;
+extern STOBJOPEN storeAufsOpen;
+extern STOBJCLOSE storeAufsClose;
+extern STOBJREAD storeAufsRead;
+extern STOBJWRITE storeAufsWrite;
+extern STOBJUNLINK storeAufsUnlink;
+
+#endif
--- /dev/null
+
+/*
+ * $Id: store_dir_aufs.cc,v 1.1 2000/05/03 17:15:46 adrian Exp $
+ *
+ * DEBUG: section 47 Store Directory Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#if HAVE_STATVFS
+#if HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+#endif
+
+#include "store_asyncufs.h"
+
+#define DefaultLevelOneDirs 16
+#define DefaultLevelTwoDirs 256
+#define STORE_META_BASYNCUFSZ 4096
+
+typedef struct _RebuildState RebuildState;
+struct _RebuildState {
+ SwapDir *sd;
+ int n_read;
+ FILE *log;
+ int speed;
+ int curlvl1;
+ int curlvl2;
+ struct {
+ unsigned int need_to_validate:1;
+ unsigned int clean:1;
+ unsigned int init:1;
+ } flags;
+ int done;
+ int in_dir;
+ int fn;
+ struct dirent *entry;
+ DIR *td;
+ char fullpath[SQUID_MAXPATHLEN];
+ char fullfilename[SQUID_MAXPATHLEN];
+ struct _store_rebuild_data counts;
+};
+
+static int n_asyncufs_dirs = 0;
+static int *asyncufs_dir_index = NULL;
+
+static char *storeAufsDirSwapSubDir(SwapDir *, int subdirn);
+static int storeAufsDirCreateDirectory(const char *path, int);
+static int storeAufsDirVerifyCacheDirs(SwapDir *);
+static int storeAufsDirVerifyDirectory(const char *path);
+static void storeAufsDirCreateSwapSubDirs(SwapDir *);
+static char *storeAufsDirSwapLogFile(SwapDir *, const char *);
+static EVH storeAufsDirRebuildFromDirectory;
+static EVH storeAufsDirRebuildFromSwapLog;
+static int storeAufsDirGetNextFile(RebuildState *, int *sfileno, int *size);
+static StoreEntry *storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean);
+static void storeAufsDirRebuild(SwapDir * sd);
+static void storeAufsDirCloseTmpSwapLog(SwapDir * sd);
+static FILE *storeAufsDirOpenTmpSwapLog(SwapDir *, int *, int *);
+static STLOGOPEN storeAufsDirOpenSwapLog;
+static STINIT storeAufsDirInit;
+static STFREE storeAufsDirFree;
+static STLOGCLEANOPEN storeAufsDirWriteCleanOpen;
+static void storeAufsDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANWRITE storeAufsDirWriteCleanEntry;
+static STLOGCLOSE storeAufsDirCloseSwapLog;
+static STLOGWRITE storeAufsDirSwapLog;
+static STNEWFS storeAufsDirNewfs;
+static STDUMP storeAufsDirDump;
+static STMAINTAINFS storeAufsDirMaintain;
+static STCHECKOBJ storeAufsDirCheckObj;
+static STREFOBJ storeAufsDirRefObj;
+static STUNREFOBJ storeAufsDirUnrefObj;
+static QS rev_int_sort;
+static int storeAufsDirClean(int swap_index);
+static EVH storeAufsDirCleanEvent;
+static int storeAufsDirIs(SwapDir * sd);
+static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2);
+static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static void storeAufsDirStats(SwapDir *, StoreEntry *);
+static void storeAufsDirInitBitmap(SwapDir *);
+static int storeAufsDirValidFileno(SwapDir *, sfileno);
+static int storeAufsDirCheckExpired(SwapDir *, StoreEntry *);
+#if !HEAP_REPLACEMENT
+static time_t storeAufsDirExpiredReferenceAge(SwapDir *);
+#endif
+
+/*
+ * These functions were ripped straight out of the heart of store_dir.c.
+ * They assume that the given filenum is on a asyncufs partiton, which may or
+ * may not be true..
+ * XXX this evilness should be tidied up at a later date!
+ */
+
+int
+storeAufsDirMapBitTest(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ aioinfo_t *aioinfo;
+ aioinfo = (aioinfo_t *)SD->fsdata;
+ return file_map_bit_test(aioinfo->map, filn);
+}
+
+void
+storeAufsDirMapBitSet(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ aioinfo_t *aioinfo;
+ aioinfo = (aioinfo_t *)SD->fsdata;
+ file_map_bit_set(aioinfo->map, filn);
+}
+
+void
+storeAufsDirMapBitReset(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ aioinfo_t *aioinfo;
+ aioinfo = (aioinfo_t *)SD->fsdata;
+ file_map_bit_reset(aioinfo->map, filn);
+}
+
+int
+storeAufsDirMapBitAllocate(SwapDir *SD)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata;
+ int fn;
+ fn = file_map_allocate(aioinfo->map, aioinfo->suggest);
+ file_map_bit_set(aioinfo->map, fn);
+ aioinfo->suggest = fn + 1;
+ return fn;
+}
+
+
+/*
+ * Initialise the asyncufs bitmap
+ *
+ * If there already is a bitmap, and the numobjects is larger than currently
+ * configured, we allocate a new bitmap and 'grow' the old one into it.
+ */
+static void
+storeAufsDirInitBitmap(SwapDir *sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+
+ if (aioinfo->map == NULL) {
+ /* First time */
+ aioinfo->map = file_map_create();
+ } else if (aioinfo->map->max_n_files) {
+ /* it grew, need to expand */
+ /* XXX We don't need it anymore .. */
+ }
+ /* else it shrunk, and we leave the old one in place */
+}
+
+static char *
+storeAufsDirSwapSubDir(SwapDir * sd, int subdirn)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ assert(0 <= subdirn && subdirn < aioinfo->l1);
+ snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
+ return fullfilename;
+}
+
+static int
+storeAufsDirCreateDirectory(const char *path, int should_exist)
+{
+ int created = 0;
+ struct stat st;
+ getCurrentTime();
+ if (0 == stat(path, &st)) {
+ if (S_ISDIR(st.st_mode)) {
+ debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
+ } else {
+ fatalf("Swap directory %s is not a directory.", path);
+ }
+ } else if (0 == mkdir(path, 0755)) {
+ debug(20, should_exist ? 1 : 3) ("%s created\n", path);
+ created = 1;
+ } else {
+ fatalf("Failed to make swap directory %s: %s",
+ path, xstrerror());
+ }
+ return created;
+}
+
+static int
+storeAufsDirVerifyDirectory(const char *path)
+{
+ struct stat sb;
+ if (stat(path, &sb) < 0) {
+ debug(20, 0) ("%s: %s\n", path, xstrerror());
+ return -1;
+ }
+ if (S_ISDIR(sb.st_mode) == 0) {
+ debug(20, 0) ("%s is not a directory\n", path);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * This function is called by storeAufsDirInit(). If this returns < 0,
+ * then Squid exits, complains about swap directories not
+ * existing, and instructs the admin to run 'squid -z'
+ */
+static int
+storeAufsDirVerifyCacheDirs(SwapDir * sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ int j;
+ const char *path = sd->path;
+
+ if (storeAufsDirVerifyDirectory(path) < 0)
+ return -1;
+ for (j = 0; j < aioinfo->l1; j++) {
+ path = storeAufsDirSwapSubDir(sd, j);
+ if (storeAufsDirVerifyDirectory(path) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void
+storeAufsDirCreateSwapSubDirs(SwapDir * sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ int i, k;
+ int should_exist;
+ LOCAL_ARRAY(char, name, MAXPATHLEN);
+ for (i = 0; i < aioinfo->l1; i++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
+ if (storeAufsDirCreateDirectory(name, 0))
+ should_exist = 0;
+ else
+ should_exist = 1;
+ debug(47, 1) ("Making directories in %s\n", name);
+ for (k = 0; k < aioinfo->l2; k++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
+ storeAufsDirCreateDirectory(name, should_exist);
+ }
+ }
+}
+
+static char *
+storeAufsDirSwapLogFile(SwapDir * sd, const char *ext)
+{
+ LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, digit, 32);
+ char *pathtmp2;
+ if (Config.Log.swap) {
+ xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
+ while (index(pathtmp,'/'))
+ *index(pathtmp,'/')='.';
+ while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.')
+ pathtmp[strlen(pathtmp)-1]= '\0';
+ for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
+ snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2);
+ if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
+ strcat(path, ".");
+ snprintf(digit, 32, "%02d", sd->index);
+ strncat(path, digit, 3);
+ }
+ } else {
+ xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
+ strcat(path, "/swap.state");
+ }
+ if (ext)
+ strncat(path, ext, 16);
+ return path;
+}
+
+static void
+storeAufsDirOpenSwapLog(SwapDir * sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ char *path;
+ int fd;
+ path = storeAufsDirSwapLogFile(sd, NULL);
+ fd = file_open(path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", path, xstrerror());
+ fatal("storeAufsDirOpenSwapLog: Failed to open swap log.");
+ }
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+ aioinfo->swaplog_fd = fd;
+ if (0 == n_asyncufs_dirs)
+ assert(NULL == asyncufs_dir_index);
+ n_asyncufs_dirs++;
+ assert(n_asyncufs_dirs <= Config.cacheSwap.n_configured);
+}
+
+static void
+storeAufsDirCloseSwapLog(SwapDir * sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ if (aioinfo->swaplog_fd < 0) /* not open */
+ return;
+ file_close(aioinfo->swaplog_fd);
+ debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
+ sd->index, aioinfo->swaplog_fd);
+ aioinfo->swaplog_fd = -1;
+ n_asyncufs_dirs--;
+ assert(n_asyncufs_dirs >= 0);
+ if (0 == n_asyncufs_dirs)
+ safe_free(asyncufs_dir_index);
+}
+
+static void
+storeAufsDirInit(SwapDir * sd)
+{
+ static int started_clean_event = 0;
+ static const char *errmsg =
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
+ storeAufsDirInitBitmap(sd);
+ if (storeAufsDirVerifyCacheDirs(sd) < 0)
+ fatal(errmsg);
+ storeAufsDirOpenSwapLog(sd);
+ storeAufsDirRebuild(sd);
+ if (!started_clean_event) {
+ eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL, 15.0, 1);
+ started_clean_event = 1;
+ }
+}
+
+static void
+storeAufsDirRebuildFromDirectory(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
+ StoreEntry *e = NULL;
+ StoreEntry tmpe;
+ cache_key key[MD5_DIGEST_CHARS];
+ int sfileno = 0;
+ int count;
+ int size;
+ struct stat sb;
+ int swap_hdr_len;
+ int fd = -1;
+ tlv *tlv_list;
+ tlv *t;
+ assert(rb != NULL);
+ debug(20, 3) ("storeAufsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
+ for (count = 0; count < rb->speed; count++) {
+ assert(fd == -1);
+ fd = storeAufsDirGetNextFile(rb, &sfileno, &size);
+ if (fd == -2) {
+ debug(20, 1) ("Done scanning %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ store_dirs_rebuilding--;
+ storeAufsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ } else if (fd < 0) {
+ continue;
+ }
+ assert(fd > -1);
+ /* lets get file stats here */
+ if (fstat(fd, &sb) < 0) {
+ debug(20, 1) ("storeAufsDirRebuildFromDirectory: fstat(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %s %7d files opened so far.\n",
+ rb->sd->path, rb->counts.scancount);
+ debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
+ Counter.syscalls.disk.reads++;
+ if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
+ debug(20, 1) ("storeAufsDirRebuildFromDirectory: read(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ swap_hdr_len = 0;
+#if USE_TRUNCATE
+ if (sb.st_size == 0)
+ continue;
+#endif
+ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
+ if (tlv_list == NULL) {
+ debug(20, 1) ("storeAufsDirRebuildFromDirectory: failed to get meta data\n");
+ /* XXX shouldn't this be a call to storeAufsUnlink ? */
+ storeAufsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ debug(20, 3) ("storeAufsDirRebuildFromDirectory: successful swap meta unpacking\n");
+ memset(key, '\0', MD5_DIGEST_CHARS);
+ memset(&tmpe, '\0', sizeof(StoreEntry));
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_KEY:
+ assert(t->length == MD5_DIGEST_CHARS);
+ xmemcpy(key, t->value, MD5_DIGEST_CHARS);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ default:
+ break;
+ }
+ }
+ storeSwapTLVFree(tlv_list);
+ tlv_list = NULL;
+ if (storeKeyNull(key)) {
+ debug(20, 1) ("storeAufsDirRebuildFromDirectory: NULL key\n");
+ storeAufsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ tmpe.key = key;
+ /* check sizes */
+ if (tmpe.swap_file_sz == 0) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
+ tmpe.swap_file_sz, (int) sb.st_size);
+ storeAufsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
+ storeAufsDirUnlinkFile(SD, sfileno);
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(key);
+ if (e && e->lastref >= tmpe.lastref) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (NULL != e) {
+ /* URL already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeRelease(e); /* release old entry */
+ rb->counts.dupcount++;
+ }
+ rb->counts.objcount++;
+ storeEntryDump(&tmpe, 5);
+ e = storeAufsDirAddDiskRestore(SD, key,
+ sfileno,
+ tmpe.swap_file_sz,
+ tmpe.expires,
+ tmpe.timestamp,
+ tmpe.lastref,
+ tmpe.lastmod,
+ tmpe.refcount, /* refcount */
+ tmpe.flags, /* flags */
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeAufsDirRebuildFromDirectory, rb, 0.0, 1);
+}
+
+static void
+storeAufsDirRebuildFromSwapLog(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogData s;
+ size_t ss = sizeof(storeSwapLogData);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ debug(20, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ store_dirs_rebuilding--;
+ storeAufsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ }
+ rb->n_read++;
+ if (s.op <= SWAP_LOG_NOP)
+ continue;
+ if (s.op >= SWAP_LOG_MAX)
+ continue;
+ debug(20, 3) ("storeAufsDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ if ((e = storeGet(s.key)) != NULL) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeAufsDirReplRemove(e);
+ storeRelease(e);
+ if (e->swap_filen > -1) {
+ /* Fake a unlink here, this is a bad hack :( */
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %7d %s Entries read so far.\n",
+ rb->counts.scancount, rb->sd->path);
+ if (!storeAufsDirValidFileno(SD, s.swap_filen)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeAufsDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+#if HEAP_REPLACEMENT
+ storeHeapPositionUpdate(e, SD);
+ storeAufsDirUnrefObj(SD, e);
+#endif
+ } else {
+ debug_trap("storeAufsDirRebuildFromSwapLog: bad condition");
+ debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
+ continue;
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeAufsDirReplRemove(e);
+ storeRelease(e);
+ if (e->swap_filen > -1) {
+ /* Fake a unlink here, this is a bad hack :( */
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeAufsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeAufsDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+static int
+storeAufsDirGetNextFile(RebuildState * rb, int *sfileno, int *size)
+{
+ SwapDir *SD = rb->sd;
+ aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata;
+ int fd = -1;
+ int used = 0;
+ int dirs_opened = 0;
+ debug(20, 3) ("storeAufsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
+ rb->flags.init,
+ rb->sd->index,
+ rb->curlvl1,
+ rb->curlvl2);
+ if (rb->done)
+ return -2;
+ while (fd < 0 && rb->done == 0) {
+ fd = -1;
+ if (0 == rb->flags.init) { /* initialize, open first file */
+ rb->done = 0;
+ rb->curlvl1 = 0;
+ rb->curlvl2 = 0;
+ rb->in_dir = 0;
+ rb->flags.init = 1;
+ assert(Config.cacheSwap.n_configured > 0);
+ }
+ if (0 == rb->in_dir) { /* we need to read in a new directory */
+ snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ rb->sd->path,
+ rb->curlvl1, rb->curlvl2);
+ if (rb->flags.init && rb->td != NULL)
+ closedir(rb->td);
+ rb->td = NULL;
+ if (dirs_opened)
+ return -1;
+ rb->td = opendir(rb->fullpath);
+ dirs_opened++;
+ if (rb->td == NULL) {
+ debug(50, 1) ("storeAufsDirGetNextFile: opendir: %s: %s\n",
+ rb->fullpath, xstrerror());
+ } else {
+ rb->entry = readdir(rb->td); /* skip . and .. */
+ rb->entry = readdir(rb->td);
+ if (rb->entry == NULL && errno == ENOENT)
+ debug(20, 1) ("storeAufsDirGetNextFile: directory does not exist!.\n");
+ debug(20, 3) ("storeAufsDirGetNextFile: Directory %s\n", rb->fullpath);
+ }
+ }
+ if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
+ rb->in_dir++;
+ if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
+ debug(20, 3) ("storeAufsDirGetNextFile: invalid %s\n",
+ rb->entry->d_name);
+ continue;
+ }
+ if (!storeAufsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
+ debug(20, 3) ("storeAufsDirGetNextFile: %08X does not belong in %d/%d/%d\n",
+ rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
+ continue;
+ }
+ used = storeAufsDirMapBitTest(SD, rb->fn);
+ if (used) {
+ debug(20, 3) ("storeAufsDirGetNextFile: Locked, continuing with next.\n");
+ continue;
+ }
+ snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
+ rb->fullpath, rb->entry->d_name);
+ debug(20, 3) ("storeAufsDirGetNextFile: Opening %s\n", rb->fullfilename);
+ fd = file_open(rb->fullfilename, O_RDONLY);
+ if (fd < 0)
+ debug(50, 1) ("storeAufsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
+ else
+ store_open_disk_fd++;
+ continue;
+ }
+ rb->in_dir = 0;
+ if (++rb->curlvl2 < aioinfo->l2)
+ continue;
+ rb->curlvl2 = 0;
+ if (++rb->curlvl1 < aioinfo->l1)
+ continue;
+ rb->curlvl1 = 0;
+ rb->done = 1;
+ }
+ *sfileno = rb->fn;
+ return fd;
+}
+
+/* Add a new object to the cache with empty memory copy and pointer to disk
+ * use to rebuild store from disk. */
+static StoreEntry *
+storeAufsDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean)
+{
+ StoreEntry *e = NULL;
+ debug(20, 5) ("storeAufsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
+ /* if you call this you'd better be sure file_number is not
+ * already in use! */
+ e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ e->store_status = STORE_OK;
+ storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->swap_status = SWAPOUT_DONE;
+ e->swap_filen = file_number;
+ e->swap_dirn = SD->index;
+ e->swap_file_sz = swap_file_sz;
+ e->lock_count = 0;
+#if !HEAP_REPLACEMENT
+ e->refcount = 0;
+#endif
+ e->lastref = lastref;
+ e->timestamp = timestamp;
+ e->expires = expires;
+ e->lastmod = lastmod;
+ e->refcount = refcount;
+ e->flags = flags;
+ EBIT_SET(e->flags, ENTRY_CACHABLE);
+ EBIT_CLR(e->flags, RELEASE_REQUEST);
+ EBIT_CLR(e->flags, KEY_PRIVATE);
+ e->ping_status = PING_NONE;
+ EBIT_CLR(e->flags, ENTRY_VALIDATED);
+ storeAufsDirMapBitSet(SD, e->swap_filen);
+ storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
+ storeAufsDirReplAdd(SD, e);
+ return e;
+}
+
+static void
+storeAufsDirRebuild(SwapDir * sd)
+{
+ RebuildState *rb = xcalloc(1, sizeof(*rb));
+ int clean = 0;
+ int zero = 0;
+ FILE *fp;
+ EVH *func = NULL;
+ rb->sd = sd;
+ rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ /*
+ * If the swap.state file exists in the cache_dir, then
+ * we'll use storeAufsDirRebuildFromSwapLog(), otherwise we'll
+ * use storeAufsDirRebuildFromDirectory() to open up each file
+ * and suck in the meta data.
+ */
+ fp = storeAufsDirOpenTmpSwapLog(sd, &clean, &zero);
+ if (fp == NULL || zero) {
+ if (fp != NULL)
+ fclose(fp);
+ func = storeAufsDirRebuildFromDirectory;
+ } else {
+ func = storeAufsDirRebuildFromSwapLog;
+ rb->log = fp;
+ rb->flags.clean = (unsigned int) clean;
+ }
+ if (!clean)
+ rb->flags.need_to_validate = 1;
+ debug(20, 1) ("Rebuilding storage in %s (%s)\n",
+ sd->path, clean ? "CLEAN" : "DIRTY");
+ store_dirs_rebuilding++;
+ cbdataAdd(rb, cbdataXfree, 0);
+ eventAdd("storeRebuild", func, rb, 0.0, 1);
+}
+
+static void
+storeAufsDirCloseTmpSwapLog(SwapDir * sd)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
+ char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new"));
+ int fd;
+ file_close(aioinfo->swaplog_fd);
+#ifdef _SQUID_OS2_
+ if (unlink(swaplog_path) < 0) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeAufsDirCloseTmpSwapLog: unlink failed");
+ }
+#endif
+ if (xrename(new_path, swaplog_path) < 0) {
+ fatal("storeAufsDirCloseTmpSwapLog: rename failed");
+ }
+ fd = file_open(swaplog_path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeAufsDirCloseTmpSwapLog: Failed to open swap log.");
+ }
+ safe_free(swaplog_path);
+ safe_free(new_path);
+ aioinfo->swaplog_fd = fd;
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+}
+
+static FILE *
+storeAufsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
+ char *clean_path = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
+ char *new_path = xstrdup(storeAufsDirSwapLogFile(sd, ".new"));
+ struct stat log_sb;
+ struct stat clean_sb;
+ FILE *fp;
+ int fd;
+ if (stat(swaplog_path, &log_sb) < 0) {
+ debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return NULL;
+ }
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
+ /* close the existing write-only FD */
+ if (aioinfo->swaplog_fd >= 0)
+ file_close(aioinfo->swaplog_fd);
+ /* open a write-only FD for the new log */
+ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
+ }
+ aioinfo->swaplog_fd = fd;
+ /* open a read-only stream of the old log */
+ fp = fopen(swaplog_path, "r");
+ if (fp == NULL) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("Failed to open swap log for reading");
+ }
+ memset(&clean_sb, '\0', sizeof(struct stat));
+ if (stat(clean_path, &clean_sb) < 0)
+ *clean_flag = 0;
+ else if (clean_sb.st_mtime < log_sb.st_mtime)
+ *clean_flag = 0;
+ else
+ *clean_flag = 1;
+ safeunlink(clean_path, 1);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return fp;
+}
+
+struct _clean_state {
+ char *cur;
+ char *new;
+ char *cln;
+ char *outbuf;
+ off_t outbuf_offset;
+ int fd;
+};
+
+#define CLEAN_BUF_SZ 16384
+/*
+ * Begin the process to write clean cache state. For ASYNCUFS this means
+ * opening some log files and allocating write buffers. Return 0 if
+ * we succeed, and assign the 'func' and 'data' return pointers.
+ */
+static int
+storeAufsDirWriteCleanOpen(SwapDir * sd)
+{
+ struct _clean_state *state = xcalloc(1, sizeof(*state));
+ struct stat sb;
+ sd->log.clean.write = NULL;
+ sd->log.clean.state = NULL;
+ state->cur = xstrdup(storeAufsDirSwapLogFile(sd, NULL));
+ state->new = xstrdup(storeAufsDirSwapLogFile(sd, ".clean"));
+ state->cln = xstrdup(storeAufsDirSwapLogFile(sd, ".last-clean"));
+ state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
+ state->outbuf_offset = 0;
+ unlink(state->new);
+ unlink(state->cln);
+ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
+ if (state->fd < 0)
+ return -1;
+ debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
+ state->new, state->fd);
+#if HAVE_FCHMOD
+ if (stat(state->cur, &sb) == 0)
+ fchmod(state->fd, sb.st_mode);
+#endif
+ sd->log.clean.write = storeAufsDirWriteCleanEntry;
+ sd->log.clean.state = state;
+ return 0;
+}
+
+/*
+ * "write" an entry to the clean log file.
+ */
+static void
+storeAufsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+{
+ storeSwapLogData s;
+ static size_t ss = sizeof(storeSwapLogData);
+ struct _clean_state *state = sd->log.clean.state;
+ if (NULL == e) {
+ storeAufsDirWriteCleanClose(sd);
+ return;
+ }
+ memset(&s, '\0', ss);
+ s.op = (char) SWAP_LOG_ADD;
+ s.swap_filen = e->swap_filen;
+ s.timestamp = e->timestamp;
+ s.lastref = e->lastref;
+ s.expires = e->expires;
+ s.lastmod = e->lastmod;
+ s.swap_file_sz = e->swap_file_sz;
+ s.refcount = e->refcount;
+ s.flags = e->flags;
+ xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
+ xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
+ state->outbuf_offset += ss;
+ /* buffered write */
+ if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+ }
+ state->outbuf_offset = 0;
+ }
+}
+
+static void
+storeAufsDirWriteCleanClose(SwapDir * sd)
+{
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->fd < 0)
+ return;
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile "
+ "not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ }
+ safe_free(state->outbuf);
+ /*
+ * You can't rename open files on Microsoft "operating systems"
+ * so we have to close before renaming.
+ */
+ storeAufsDirCloseSwapLog(sd);
+ /* rename */
+ if (state->fd >= 0) {
+#ifdef _SQUID_OS2_
+ file_close(state->fd);
+ state->fd = -1;
+ if (unlink(cur) < 0)
+ debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
+ xstrerror(), cur);
+#endif
+ xrename(state->new, state->cur);
+ }
+ /* touch a timestamp file if we're not still validating */
+ if (store_dirs_rebuilding)
+ (void) 0;
+ else if (state->fd < 0)
+ (void) 0;
+ else
+ file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC));
+ /* close */
+ safe_free(state->cur);
+ safe_free(state->new);
+ safe_free(state->cln);
+ if (state->fd >= 0)
+ file_close(state->fd);
+ state->fd = -1;
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+}
+
+static void
+storeAufsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)sd->fsdata;
+ storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData));
+ s->op = (char) op;
+ s->swap_filen = e->swap_filen;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->swap_file_sz = e->swap_file_sz;
+ s->refcount = e->refcount;
+ s->flags = e->flags;
+ xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
+ file_write(aioinfo->swaplog_fd,
+ -1,
+ s,
+ sizeof(storeSwapLogData),
+ NULL,
+ NULL,
+ xfree);
+}
+
+static void
+storeAufsDirNewfs(SwapDir * sd)
+{
+ debug(47, 3) ("Creating swap space in %s\n", sd->path);
+ storeAufsDirCreateDirectory(sd->path, 0);
+ storeAufsDirCreateSwapSubDirs(sd);
+}
+
+static int
+rev_int_sort(const void *A, const void *B)
+{
+ const int *i1 = A;
+ const int *i2 = B;
+ return *i2 - *i1;
+}
+
+static int
+storeAufsDirClean(int swap_index)
+{
+ DIR *dp = NULL;
+ struct dirent *de = NULL;
+ LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
+ LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
+#if USE_TRUNCATE
+ struct stat sb;
+#endif
+ int files[20];
+ int swapfileno;
+ int fn; /* same as swapfileno, but with dirn bits set */
+ int n = 0;
+ int k = 0;
+ int N0, N1, N2;
+ int D0, D1, D2;
+ SwapDir *SD;
+ aioinfo_t *aioinfo;
+ N0 = n_asyncufs_dirs;
+ D0 = asyncufs_dir_index[swap_index % N0];
+ SD = &Config.cacheSwap.swapDirs[D0];
+ aioinfo = (aioinfo_t *)SD->fsdata;
+ N1 = aioinfo->l1;
+ D1 = (swap_index / N0) % N1;
+ N2 = aioinfo->l2;
+ D2 = ((swap_index / N0) / N1) % N2;
+ snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ Config.cacheSwap.swapDirs[D0].path, D1, D2);
+ debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
+ dp = opendir(p1);
+ if (dp == NULL) {
+ if (errno == ENOENT) {
+ debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
+ if (mkdir(p1, 0777) == 0)
+ return 0;
+ }
+ debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
+ safeunlink(p1, 1);
+ return 0;
+ }
+ while ((de = readdir(dp)) != NULL && k < 20) {
+ if (sscanf(de->d_name, "%X", &swapfileno) != 1)
+ continue;
+ fn = swapfileno; /* XXX should remove this cruft ! */
+ if (storeAufsDirValidFileno(SD, fn))
+ if (storeAufsDirMapBitTest(SD, fn))
+ if (storeAufsFilenoBelongsHere(fn, D0, D1, D2))
+ continue;
+#if USE_TRUNCATE
+ if (!stat(de->d_name, &sb))
+ if (sb.st_size == 0)
+ continue;
+#endif
+ files[k++] = swapfileno;
+ }
+ closedir(dp);
+ if (k == 0)
+ return 0;
+ qsort(files, k, sizeof(int), rev_int_sort);
+ if (k > 10)
+ k = 10;
+ for (n = 0; n < k; n++) {
+ debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
+ snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
+#if USE_TRUNCATE
+ truncate(p2, 0);
+#else
+ safeunlink(p2, 0);
+#endif
+ Counter.swap_files_cleaned++;
+ }
+ debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
+ return k;
+}
+
+static void
+storeAufsDirCleanEvent(void *unused)
+{
+ static int swap_index = 0;
+ int i;
+ int j = 0;
+ int n = 0;
+ /*
+ * Assert that there are ASYNCUFS cache_dirs configured, otherwise
+ * we should never be called.
+ */
+ assert(n_asyncufs_dirs);
+ if (NULL == asyncufs_dir_index) {
+ SwapDir *sd;
+ aioinfo_t *aioinfo;
+ /*
+ * Initialize the little array that translates ASYNCUFS cache_dir
+ * number into the Config.cacheSwap.swapDirs array index.
+ */
+ asyncufs_dir_index = xcalloc(n_asyncufs_dirs, sizeof(*asyncufs_dir_index));
+ for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
+ sd = &Config.cacheSwap.swapDirs[i];
+ if (!storeAufsDirIs(sd))
+ continue;
+ asyncufs_dir_index[n++] = i;
+ aioinfo = (aioinfo_t *)sd->fsdata;
+ j += (aioinfo->l1 * aioinfo->l2);
+ }
+ assert(n == n_asyncufs_dirs);
+ /*
+ * Start the storeAufsDirClean() swap_index with a random
+ * value. j equals the total number of ASYNCUFS level 2
+ * swap directories
+ */
+ swap_index = (int) (squid_random() % j);
+ }
+ if (0 == store_dirs_rebuilding) {
+ n = storeAufsDirClean(swap_index);
+ swap_index++;
+ }
+ eventAdd("storeDirClean", storeAufsDirCleanEvent, NULL,
+ 15.0 * exp(-0.25 * n), 1);
+}
+
+static int
+storeAufsDirIs(SwapDir * sd)
+{
+ if (strncmp(sd->type, "aufs", 3) == 0)
+ return 1;
+ return 0;
+}
+
+/*
+ * Does swapfile number 'fn' belong in cachedir #F0,
+ * level1 dir #F1, level2 dir #F2?
+ */
+static int
+storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2)
+{
+ int D1, D2;
+ int L1, L2;
+ int filn = fn;
+ aioinfo_t *aioinfo;
+ assert(F0 < Config.cacheSwap.n_configured);
+ aioinfo = (aioinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata;
+ L1 = aioinfo->l1;
+ L2 = aioinfo->l2;
+ D1 = ((filn / L2) / L2) % L1;
+ if (F1 != D1)
+ return 0;
+ D2 = (filn / L2) % L2;
+ if (F2 != D2)
+ return 0;
+ return 1;
+}
+
+int
+storeAufsDirValidFileno(SwapDir *SD, sfileno filn)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata;
+ if (filn < 0)
+ return 0;
+ if (filn > aioinfo->map->max_n_files)
+ return 0;
+ return 1;
+}
+
+void
+storeAufsDirMaintain(SwapDir *SD)
+{
+ StoreEntry *e = NULL;
+ int scanned = 0;
+ int locked = 0;
+ int expired = 0;
+ int max_scan;
+ int max_remove;
+ double f;
+ static time_t last_warn_time = 0;
+#if !HEAP_REPLACEMENT
+ dlink_node *m;
+ dlink_node *prev = NULL;
+#else
+ heap_key age;
+ heap_key min_age = 0.0;
+ link_list *locked_entries = NULL;
+#if HEAP_REPLACEMENT_DEBUG
+ if (!verify_heap_property(SD->repl.heap.heap)) {
+ debug(20, 1) ("Heap property violated!\n");
+ }
+#endif
+#endif
+ /* We can't delete objects while rebuilding swap */
+ if (store_dirs_rebuilding) {
+ return;
+ } else {
+ f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
+ f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
+ max_scan = (int) (f * 400.0 + 100.0);
+ max_remove = (int) (f * 70.0 + 10.0);
+ /*
+ * This is kinda cheap, but so we need this priority hack?
+ */
+#if 0
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
+#endif
+ }
+ debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
+#if HEAP_REPLACEMENT
+ while (heap_nodes(SD->repl.heap.heap) > 0) {
+ if (store_swap_size < store_swap_low)
+ break;
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ age = heap_peepminkey(SD->repl.heap.heap);
+ e = heap_extractmin(SD->repl.heap.heap);
+ e->repl.node = NULL; /* no longer in the heap */
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * Entry is in use ... put it in a linked list to ignore it.
+ */
+ if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ /*
+ * If this was a "SPECIAL" do not add it back into the heap.
+ * It will always be "SPECIAL" and therefore never removed.
+ */
+ debug(20, 4) ("storeAufsDirMaintain: locked url %s\n",
+ (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
+key));
+ linklistPush(&locked_entries, e);
+ }
+ locked++;
+ continue;
+ } else if (storeAufsDirCheckExpired(SD, e)) {
+ /*
+ * Note: This will not check the reference age ifdef
+ * HEAP_REPLACEMENT, but it does some other useful
+ * checks...
+ */
+ expired++;
+ debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
+ age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
+ min_age = age;
+ storeRelease(e);
+ } else {
+ /*
+ * Did not expire the object so we need to add it back
+ * into the heap!
+ */
+ debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
+ storeKeyText(e->key));
+ linklistPush(&locked_entries, e);
+ continue;
+ }
+ if (store_swap_size < store_swap_low)
+ break;
+ else if (expired >= max_remove)
+ break;
+ else if (scanned >= max_scan)
+ break;
+ }
+ /*
+ * Bump the heap age factor.
+ */
+ if (min_age > 0.0)
+ SD->repl.heap.heap->age = min_age;
+ /*
+ * Reinsert all bumped locked entries back into heap...
+ */
+ while ((e = linklistShift(&locked_entries)))
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+#else
+ for (m = SD->repl.lru.list.tail; m; m = prev) {
+ prev = m->prev;
+ e = m->data;
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * If there is a locked entry at the tail of the LRU list,
+ * move it to the beginning to get it out of the way.
+ * Theoretically, we might have all locked objects at the
+ * tail, and then we'll never remove anything here and the
+ * LRU age will go to zero.
+ */
+ if (memInUse(MEM_STOREENTRY) > max_scan) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+ locked++;
+
+ } else if (storeAufsDirCheckExpired(SD, e)) {
+ expired++;
+ storeRelease(e);
+ }
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ }
+#endif
+ debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l
+ocked %d f=%.03f\n",
+ scanned, max_scan, expired, max_remove, locked, f);
+ debug(20, 3) ("storeMaintainSwapSpace stats:\n");
+ debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
+ debug(20, 3) (" %6d were scanned\n", scanned);
+ debug(20, 3) (" %6d were locked\n", locked);
+ debug(20, 3) (" %6d were expired\n", expired);
+ if (store_swap_size < Config.Swap.maxSize)
+ return;
+ if (squid_curtime - last_warn_time < 10)
+ return;
+ debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
+ store_swap_size, Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
+}
+
+/*
+ * storeAufsDirCheckObj
+ *
+ * This routine is called by storeDirSelectSwapDir to see if the given
+ * object is able to be stored on this filesystem. AUFS filesystems will
+ * happily store anything as long as the LRU time isn't too small.
+ */
+int
+storeAufsDirCheckObj(SwapDir *SD, const StoreEntry *e)
+{
+ int loadav;
+ int ql;
+
+#if !HEAP_REPLACEMENT
+ if (storeAufsDirExpiredReferenceAge(SD) < 300) {
+ debug(20, 3) ("storeAufsDirCheckObj: NO: LRU Age = %d\n",
+ storeAufsDirExpiredReferenceAge(SD));
+ /* store_check_cachable_hist.no.lru_age_too_low++; */
+ return -1;
+ }
+#endif
+ ql = aioQueueSize();
+ if (ql == 0)
+ loadav = 0;
+ else if (ql >= MAGIC1) /* Queue is too long, don't even consider it */
+ loadav = -1;
+ else
+ loadav = MAGIC1 * 1000 / ql;
+ return loadav;
+}
+
+/*
+ * storeAufsDirRefObj
+ *
+ * This routine is called whenever an object is referenced, so we can
+ * maintain replacement information within the storage fs.
+ */
+void
+storeAufsDirRefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeAufsDirRefObj: referencing %d/%d\n", e->swap_dirn,
+ e->swap_filen);
+#if HEAP_REPLACEMENT
+ /* Nothing to do here */
+#else
+ /* Reference the object */
+ if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
+ !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+#endif
+}
+
+/*
+ * storeAufsDirUnrefObj
+ * This routine is called whenever the last reference to an object is
+ * removed, to maintain replacement information within the storage fs.
+ */
+void
+storeAufsDirUnrefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeAufsDirUnrefObj: referencing %d/%d\n", e->swap_dirn,
+ e->swap_filen);
+#if HEAP_REPLACEMENT
+ if (e->repl.node)
+ heap_update(SD->repl.heap.heap, e->repl.node, e);
+#endif
+}
+
+/*
+ * storeAufsDirUnlinkFile
+ *
+ * This routine unlinks a file and pulls it out of the bitmap.
+ * It used to be in storeAufsUnlink(), however an interface change
+ * forced this bit of code here. Eeek.
+ */
+void
+storeAufsDirUnlinkFile(SwapDir *SD, sfileno f)
+{
+ debug(79, 3) ("storeAufsDirUnlinkFile: unlinking fileno %08X\n", f);
+ storeAufsDirMapBitReset(SD, f);
+ aioUnlink(storeAufsDirFullPath(SD, f, NULL), NULL, NULL);
+}
+
+#if !HEAP_REPLACEMENT
+/*
+ * storeAufsDirExpiredReferenceAge
+ *
+ * The LRU age is scaled exponentially between 1 minute and
+ * Config.referenceAge , when store_swap_low < store_swap_size <
+ * store_swap_high. This keeps store_swap_size within the low and high
+ * water marks. If the cache is very busy then store_swap_size stays
+ * closer to the low water mark, if it is not busy, then it will stay
+ * near the high water mark. The LRU age value can be examined on the
+ * cachemgr 'info' page.
+ */
+static time_t
+storeAufsDirExpiredReferenceAge(SwapDir *SD)
+{
+ double x;
+ double z;
+ time_t age;
+ long store_high, store_low;
+
+ store_high = (long) (((float) SD->max_size *
+ (float) Config.Swap.highWaterMark) / (float) 100);
+ store_low = (long) (((float) SD->max_size *
+ (float) Config.Swap.lowWaterMark) / (float) 100);
+ debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
+
+ x = (double) (store_high - SD->cur_size) /
+ (store_high - store_low);
+ x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
+ z = pow((double) (Config.referenceAge / 60), x);
+ age = (time_t) (z * 60.0);
+ if (age < 60)
+ age = 60;
+ else if (age > Config.referenceAge)
+ age = Config.referenceAge;
+ return age;
+}
+#endif
+
+/*
+ * storeAufsDirCheckExpired
+ *
+ * Check whether the given object is expired or not
+ * It breaks layering a little by calling the upper layers to find
+ * out whether the object is locked or not, but we can't help this
+ * right now.
+ */
+static int
+storeAufsDirCheckExpired(SwapDir *SD, StoreEntry *e)
+{
+ if (storeEntryLocked(e))
+ return 0;
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ return 1;
+ if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
+ return 1;
+
+#if HEAP_REPLACEMENT
+ /*
+ * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
+ * controls the replacement of objects.
+ */
+ return 1;
+#else
+ if (squid_curtime - e->lastref > storeAufsDirExpiredReferenceAge(SD))
+ return 1;
+ return 0;
+#endif
+}
+
+/*
+ * Add and remove the given StoreEntry from the replacement policy in
+ * use.
+ */
+
+void
+storeAufsDirReplAdd(SwapDir *SD, StoreEntry *e)
+{
+ debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ (void) 0;
+ } else {
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+ debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node);
+ }
+#else
+ /* Shouldn't we not throw special objects into the lru ? */
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+void
+storeAufsDirReplRemove(StoreEntry *e)
+{
+ SwapDir *SD = INDEXSD(e->swap_dirn);
+ debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ /* And now, release the object from the replacement policy */
+ if (e->repl.node) {
+ debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n",
+ e->repl.node);
+ heap_delete(SD->repl.heap.heap, e->repl.node);
+ e->repl.node = NULL;
+ }
+#else
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
+
+void
+storeAufsDirStats(SwapDir *SD, StoreEntry * sentry)
+{
+ aioinfo_t *aioinfo;
+#if HAVE_STATVFS
+ struct statvfs sfs;
+#endif
+ aioinfo = (aioinfo_t *)SD->fsdata;
+ storeAppendPrintf(sentry, "First level subdirectories: %d\n", aioinfo->l1);
+ storeAppendPrintf(sentry, "Second level subdirectories: %d\n", aioinfo->l2);
+ storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
+ storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
+ storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
+ 100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
+ aioinfo->map->n_files_in_map, aioinfo->map->max_n_files,
+ percent(aioinfo->map->n_files_in_map, aioinfo->map->max_n_files));
+#if HAVE_STATVFS
+#define fsbtoblk(num, fsbs, bs) \
+ (((fsbs) != 0 && (fsbs) < (bs)) ? \
+ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
+ if (!statvfs(SD->path, &sfs)) {
+ storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
+ fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024),
+ fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024),
+ percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
+ storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
+ sfs.f_files - sfs.f_ffree, sfs.f_files,
+ percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
+ }
+#endif
+ storeAppendPrintf(sentry, "Flags:");
+ if (SD->flags.selected)
+ storeAppendPrintf(sentry, " SELECTED");
+ if (SD->flags.read_only)
+ storeAppendPrintf(sentry, " READ-ONLY");
+ storeAppendPrintf(sentry, "\n");
+#if !HEAP_REPLACEMENT
+ storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
+ (double) storeAufsDirExpiredReferenceAge(SD) / 86400.0);
+#else
+#if 0
+ storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
+ heap_peepminkey(sd.repl.heap.heap));
+#endif
+#endif
+}
+
+/*
+ * storeAufsDirReconfigure
+ *
+ * This routine is called when the given swapdir needs reconfiguring
+ */
+void
+storeAufsDirReconfigure(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ unsigned int read_only = 0;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeAufsDirReconfigure: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeAufsDirReconfigure: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeAufsDirReconfigure: invalid level 2 directories value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ /* just reconfigure it */
+ if (size == sd->max_size)
+ debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n",
+ path, size);
+ else
+ debug(3, 1) ("Cache dir '%s' size changed to %d KB\n",
+ path, size);
+ sd->max_size = size;
+ if (sd->flags.read_only != read_only)
+ debug(3, 1) ("Cache dir '%s' now %s\n",
+ path, read_only ? "Read-Only" : "Read-Write");
+ sd->flags.read_only = read_only;
+ return;
+}
+
+void
+storeAufsDirDump(StoreEntry * entry, const char *name, SwapDir * s)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata;
+ storeAppendPrintf(entry, "%s %s %s %d %d %d\n",
+ name,
+ "asyncufs",
+ s->path,
+ s->max_size >> 10,
+ aioinfo->l1,
+ aioinfo->l2);
+}
+
+/*
+ * Only "free" the filesystem specific stuff here
+ */
+static void
+storeAufsDirFree(SwapDir * s)
+{
+ aioinfo_t *aioinfo = (aioinfo_t *)s->fsdata;
+ if (aioinfo->swaplog_fd > -1) {
+ file_close(aioinfo->swaplog_fd);
+ aioinfo->swaplog_fd = -1;
+ }
+ filemapFreeMemory(aioinfo->map);
+ xfree(aioinfo);
+ s->fsdata = NULL; /* Will aid debugging... */
+
+}
+
+char *
+storeAufsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath)
+{
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ aioinfo_t *aioinfo = (aioinfo_t *)SD->fsdata;
+ int L1 = aioinfo->l1;
+ int L2 = aioinfo->l2;
+ if (!fullpath)
+ fullpath = fullfilename;
+ fullpath[0] = '\0';
+ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
+ SD->path,
+ ((filn / L2) / L2) % L1,
+ (filn / L2) % L2,
+ filn);
+ return fullpath;
+}
+
+/*
+ * storeAufsCleanupDoubleCheck
+ *
+ * This is called by storeCleanup() if -S was given on the command line.
+ */
+static int
+storeAufsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e)
+{
+ struct stat sb;
+
+ if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n");
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
+ storeAufsDirFullPath(sd, e->swap_filen, NULL));
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ if (e->swap_file_sz != sb.st_size) {
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n");
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
+ storeAufsDirFullPath(sd, e->swap_filen, NULL));
+ debug(20, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
+ e->swap_file_sz, (int) sb.st_size);
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * storeAufsDirParse
+ *
+ * Called when a *new* fs is being setup.
+ */
+void
+storeAufsDirParse(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ unsigned int read_only = 0;
+ aioinfo_t *aioinfo;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeAufsDirParse: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeAufsDirParse: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeAufsDirParse: invalid level 2 directories value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ aioinfo = xmalloc(sizeof(aioinfo_t));
+ if (aioinfo == NULL)
+ fatal("storeAufsDirParse: couldn't xmalloc() aioinfo_t!\n");
+
+ sd->index = index;
+ sd->path = xstrdup(path);
+ sd->max_size = size;
+ sd->fsdata = aioinfo;
+ aioinfo->l1 = l1;
+ aioinfo->l2 = l2;
+ aioinfo->swaplog_fd = -1;
+ aioinfo->map = NULL; /* Debugging purposes */
+ aioinfo->suggest = 0;
+ sd->flags.read_only = read_only;
+ sd->init = storeAufsDirInit;
+ sd->newfs = storeAufsDirNewfs;
+ sd->dump = storeAufsDirDump;
+ sd->freefs = storeAufsDirFree;
+ sd->dblcheck = storeAufsCleanupDoubleCheck;
+ sd->statfs = storeAufsDirStats;
+ sd->maintainfs = storeAufsDirMaintain;
+ sd->checkobj = storeAufsDirCheckObj;
+ sd->refobj = storeAufsDirRefObj;
+ sd->unrefobj = storeAufsDirUnrefObj;
+ sd->callback = aioCheckCallbacks;
+ sd->sync = aioSync;
+ sd->obj.create = storeAufsCreate;
+ sd->obj.open = storeAufsOpen;
+ sd->obj.close = storeAufsClose;
+ sd->obj.read = storeAufsRead;
+ sd->obj.write = storeAufsWrite;
+ sd->obj.unlink = storeAufsUnlink;
+ sd->log.open = storeAufsDirOpenSwapLog;
+ sd->log.close = storeAufsDirCloseSwapLog;
+ sd->log.write = storeAufsDirSwapLog;
+ sd->log.clean.open = storeAufsDirWriteCleanOpen;
+
+ /* Initialise replacement policy stuff */
+#if HEAP_REPLACEMENT
+ /*
+ * Create new heaps with cache replacement policies attached to them.
+ * The cache replacement policy is specified as either GDSF or LFUDA in
+ * the squid.conf configuration file. Note that the replacement policy
+ * applies only to the disk replacement algorithm. Memory replacement
+ * always uses GDSF since we want to maximize object hit rate.
+ */
+ if (Config.replPolicy) {
+ if (tolower(Config.replPolicy[0]) == 'g') {
+ debug(20, 1) ("Using GDSF disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ } else if (tolower(Config.replPolicy[0]) == 'l') {
+ if (tolower(Config.replPolicy[1]) == 'f') {
+ debug(20, 1) ("Using LFUDA disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
+ } else if (tolower(Config.replPolicy[1]) == 'r') {
+ debug(20, 1) ("Using LRU heap disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
+ }
+ } else {
+ debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+ } else {
+ debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+#else
+ sd->repl.lru.list.head = NULL;
+ sd->repl.lru.list.tail = NULL;
+#endif
+}
+
+void
+storeFsSetup_aufs(storefs_entry_t *storefs)
+{
+ storefs->parsefunc = storeAufsDirParse;
+ storefs->reconfigurefunc = storeAufsDirReconfigure;
+ storefs->donefunc = aioDone;
+ aioInit();
+}
--- /dev/null
+
+/*
+ * DEBUG 78
+ */
+
+#include "squid.h"
+#include "store_asyncufs.h"
+
+static AIOCB storeAufsReadDone;
+static AIOCB storeAufsWriteDone;
+static void storeAufsIOCallback(storeIOState * sio, int errflag);
+static AIOCB storeAufsOpenDone;
+static int storeAufsSomethingPending(storeIOState *);
+static int storeAufsKickWriteQueue(storeIOState * sio);
+static void storeAufsIOFreeEntry(void *, int);
+
+struct _queued_write {
+ char *buf;
+ size_t size;
+ off_t offset;
+ FREE *free_func;
+};
+
+struct _queued_read {
+ char *buf;
+ size_t size;
+ off_t offset;
+ STRCB *callback;
+ void *callback_data;
+};
+
+/* === PUBLIC =========================================================== */
+
+/* open for reading */
+storeIOState *
+storeAufsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback,
+ STIOCB * callback, void *callback_data)
+{
+ sfileno f = e->swap_filen;
+ char *path = storeAufsDirFullPath(SD, f, NULL);
+ storeIOState *sio;
+ debug(78, 3) ("storeAufsOpen: fileno %08X\n", f);
+ /*
+ * we should detect some 'too many files open' condition and return
+ * NULL here.
+ */
+ while (aioQueueSize() > MAGIC1)
+ return NULL;
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(aio_state_pool);
+ ((aiostate_t *)(sio->fsstate))->fd = -1;
+ ((aiostate_t *)(sio->fsstate))->flags.opening = 1;
+ sio->swap_filen = f;
+ sio->swap_dirn = SD->index;
+ sio->mode = O_RDONLY;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ sio->e = e;
+ cbdataLock(callback_data);
+ aioOpen(path, O_RDONLY, 0644, storeAufsOpenDone, sio);
+ Opening_FD++;
+ store_open_disk_fd++;
+ return sio;
+}
+
+/* open for creating */
+storeIOState *
+storeAufsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data)
+{
+ char *path ;
+ storeIOState *sio;
+ sfileno filn;
+ sdirno dirn;
+
+ /* Allocate a number */
+ dirn = SD->index;
+ filn = storeAufsDirMapBitAllocate(SD);
+ path = storeAufsDirFullPath(SD, filn, NULL);
+
+ debug(78, 3) ("storeAufsCreate: fileno %08X\n", filn);
+ /*
+ * we should detect some 'too many files open' condition and return
+ * NULL here.
+ */
+ while (aioQueueSize() > MAGIC1)
+ return NULL;
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeAufsIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(aio_state_pool);
+ ((aiostate_t *)(sio->fsstate))->fd = -1;
+ ((aiostate_t *)(sio->fsstate))->flags.opening = 1;
+ sio->swap_filen = filn;
+ sio->swap_dirn = dirn;
+ sio->mode = O_WRONLY;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ sio->e = (StoreEntry *) e;
+ cbdataLock(callback_data);
+ aioOpen(path, O_WRONLY | O_CREAT | O_TRUNC, 0644, storeAufsOpenDone, sio);
+ Opening_FD++;
+ store_open_disk_fd++;
+
+ /* now insert into the replacement policy */
+ storeAufsDirReplAdd(SD, e);
+ return sio;
+
+}
+
+
+
+/* Close */
+void
+storeAufsClose(SwapDir *SD, storeIOState * sio)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ debug(78, 3) ("storeAufsClose: dirno %d, fileno %08X, FD %d\n",
+ sio->swap_dirn, sio->swap_filen, aiostate->fd);
+ if (storeAufsSomethingPending(sio)) {
+ aiostate->flags.close_request = 1;
+ return;
+ }
+ storeAufsIOCallback(sio, 0);
+}
+
+
+/* Read */
+void
+storeAufsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ assert(sio->read.callback == NULL);
+ assert(sio->read.callback_data == NULL);
+ assert(!aiostate->flags.reading);
+ if (aiostate->fd < 0) {
+ struct _queued_read *q;
+ debug(78, 3) ("storeAufsRead: queueing read because FD < 0\n");
+ assert(aiostate->flags.opening);
+ assert(aiostate->pending_reads == NULL);
+ q = xcalloc(1, sizeof(*q));
+ q->buf = buf;
+ q->size = size;
+ q->offset = offset;
+ q->callback = callback;
+ q->callback_data = callback_data;
+ linklistPush(&(aiostate->pending_reads), q);
+ return;
+ }
+ sio->read.callback = callback;
+ sio->read.callback_data = callback_data;
+ aiostate->read_buf = buf;
+ cbdataLock(callback_data);
+ debug(78, 3) ("storeAufsRead: dirno %d, fileno %08X, FD %d\n",
+ sio->swap_dirn, sio->swap_filen, aiostate->fd);
+ sio->offset = offset;
+ aiostate->flags.reading = 1;
+ aioRead(aiostate->fd, offset, buf, size, storeAufsReadDone, sio);
+}
+
+
+/* Write */
+void
+storeAufsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ debug(78, 3) ("storeAufsWrite: dirno %d, fileno %08X, FD %d\n",
+ sio->swap_dirn, sio->swap_filen, aiostate->fd);
+ if (aiostate->fd < 0) {
+ /* disk file not opened yet */
+ struct _queued_write *q;
+ assert(aiostate->flags.opening);
+ q = xcalloc(1, sizeof(*q));
+ q->buf = buf;
+ q->size = size;
+ q->offset = offset;
+ q->free_func = free_func;
+ linklistPush(&(aiostate->pending_writes), q);
+ return;
+ }
+ if (aiostate->flags.writing) {
+ struct _queued_write *q;
+ debug(78, 3) ("storeAufsWrite: queuing write\n");
+ q = xcalloc(1, sizeof(*q));
+ q->buf = buf;
+ q->size = size;
+ q->offset = offset;
+ q->free_func = free_func;
+ linklistPush(&(aiostate->pending_writes), q);
+ return;
+ }
+ aiostate->flags.writing = 1;
+ /*
+ * XXX it might be nice if aioWrite() gave is immediate
+ * feedback here about EWOULDBLOCK instead of in the
+ * callback function
+ */
+ aioWrite(aiostate->fd, offset, buf, size, storeAufsWriteDone, sio,
+ free_func);
+}
+
+/* Unlink */
+void
+storeAufsUnlink(SwapDir *SD, StoreEntry *e)
+{
+ debug(78, 3) ("storeAufsUnlink: dirno %d, fileno %08X\n", SD->index, e->swap_filen);
+ storeAufsDirReplRemove(e);
+ storeAufsDirUnlinkFile(SD, e->swap_filen);
+}
+
+/* === STATIC =========================================================== */
+
+static int
+storeAufsKickWriteQueue(storeIOState * sio)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ struct _queued_write *q = linklistShift(&aiostate->pending_writes);
+ if (NULL == q)
+ return 0;
+ debug(78, 3) ("storeAufsKickWriteQueue: writing queued chunk of %d bytes\n",
+ q->size);
+ storeAufsWrite(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->free_func);
+ xfree(q);
+ return 1;
+}
+
+static int
+storeAufsKickReadQueue(storeIOState * sio)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ struct _queued_read *q = linklistShift(&(aiostate->pending_reads));
+ if (NULL == q)
+ return 0;
+ debug(78, 3) ("storeAufsKickReadQueue: reading queued request of %d bytes\n",
+ q->size);
+ storeAufsRead(INDEXSD(sio->swap_dirn), sio, q->buf, q->size, q->offset, q->callback, q->callback_data);
+ xfree(q);
+ return 1;
+}
+
+static void
+storeAufsOpenDone(int unused, void *my_data, int fd, int errflag)
+{
+ storeIOState *sio = my_data;
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ debug(78, 3) ("storeAufsOpenDone: FD %d, errflag %d\n", fd, errflag);
+ Opening_FD--;
+ aiostate->flags.opening = 0;
+ if (errflag || fd < 0) {
+ errno = errflag;
+ debug(78, 0) ("storeAufsOpenDone: %s\n", xstrerror());
+ debug(78, 1) ("\t%s\n", storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
+ storeAufsIOCallback(sio, errflag);
+ return;
+ }
+ aiostate->fd = fd;
+ commSetCloseOnExec(fd);
+ fd_open(fd, FD_FILE, storeAufsDirFullPath(INDEXSD(sio->swap_dirn), sio->swap_filen, NULL));
+ if (sio->mode == O_WRONLY)
+ storeAufsKickWriteQueue(sio);
+ else if (sio->mode == O_RDONLY)
+ storeAufsKickReadQueue(sio);
+ debug(78, 3) ("storeAufsOpenDone: exiting\n");
+}
+
+static void
+storeAufsReadDone(int fd, void *my_data, int len, int errflag)
+{
+ storeIOState *sio = my_data;
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ STRCB *callback = sio->read.callback;
+ void *their_data = sio->read.callback_data;
+ ssize_t rlen;
+ debug(78, 3) ("storeAufsReadDone: dirno %d, fileno %08X, FD %d, len %d\n",
+ sio->swap_dirn, sio->swap_filen, fd, len);
+ aiostate->flags.reading = 0;
+ if (errflag) {
+ debug(78, 3) ("storeAufsReadDone: got failure (%d)\n", errflag);
+ rlen = -1;
+ } else {
+ rlen = (ssize_t) len;
+ sio->offset += len;
+ }
+ assert(callback);
+ assert(their_data);
+ sio->read.callback = NULL;
+ sio->read.callback_data = NULL;
+ if (cbdataValid(their_data))
+ callback(their_data, aiostate->read_buf, rlen);
+ cbdataUnlock(their_data);
+ /*
+ * XXX is this safe? The above callback may have caused sio
+ * to be freed/closed already? Philip Guenther <guenther@gac.edu>
+ * says it fixes his FD leaks, with no side effects.
+ */
+ if (aiostate->flags.close_request)
+ storeAufsIOCallback(sio, DISK_OK);
+}
+
+/*
+ * XXX TODO
+ * if errflag == EWOULDBLOCK, then we'll need to re-queue the
+ * chunk at the beginning of the write_pending list and try
+ * again later.
+ */
+static void
+storeAufsWriteDone(int fd, void *my_data, int len, int errflag)
+{
+ static int loop_detect = 0;
+ storeIOState *sio = my_data;
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ debug(78, 3) ("storeAufsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n",
+ sio->swap_dirn, sio->swap_filen, fd, len);
+ assert(++loop_detect < 10);
+ aiostate->flags.writing = 0;
+ if (errflag) {
+ debug(78, 0) ("storeAufsWriteDone: got failure (%d)\n", errflag);
+ storeAufsIOCallback(sio, errflag);
+ loop_detect--;
+ return;
+ }
+ sio->offset += len;
+ if (storeAufsKickWriteQueue(sio))
+ (void) 0;
+ else if (aiostate->flags.close_request)
+ storeAufsIOCallback(sio, errflag);
+ loop_detect--;
+}
+
+static void
+storeAufsIOCallback(storeIOState * sio, int errflag)
+{
+ STIOCB *callback = sio->callback;
+ void *their_data = sio->callback_data;
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ int fd = aiostate->fd;
+ debug(78, 3) ("storeAufsIOCallback: errflag=%d\n", errflag);
+ sio->callback = NULL;
+ sio->callback_data = NULL;
+ debug(78, 3) ("%s:%d\n", __FILE__, __LINE__);
+ if (callback)
+ if (NULL == their_data || cbdataValid(their_data))
+ callback(their_data, errflag, sio);
+ debug(78, 3) ("%s:%d\n", __FILE__, __LINE__);
+ cbdataUnlock(their_data);
+ aiostate->fd = -1;
+ cbdataFree(sio);
+ if (fd < 0)
+ return;
+ debug(78, 3) ("%s:%d\n", __FILE__, __LINE__);
+ aioClose(fd);
+ fd_close(fd);
+ store_open_disk_fd--;
+ debug(78, 3) ("%s:%d\n", __FILE__, __LINE__);
+}
+
+
+static int
+storeAufsSomethingPending(storeIOState * sio)
+{
+ aiostate_t *aiostate = (aiostate_t *)sio->fsstate;
+ if (aiostate->flags.reading)
+ return 1;
+ if (aiostate->flags.writing)
+ return 1;
+ if (aiostate->flags.opening)
+ return 1;
+ return 0;
+}
+
+
+/*
+ * We can't pass memFree() as a free function here, because we need to free
+ * the fsstate variable ..
+ */
+static void
+storeAufsIOFreeEntry(void *sio, int foo)
+{
+ memPoolFree(aio_state_pool, ((storeIOState *)sio)->fsstate);
+ memFree(sio, MEM_STORE_IO);
+}
+
--- /dev/null
+#
+# Makefile for the COSS storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/05/03 17:15:47 adrian Exp $
+#
+
+FS = coss
+
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(FS).a
+
+OBJS = \
+ store_dir_coss.o \
+ store_io_coss.o
+
+
+all: $(OUT)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) $(CFLAGS) -c $<
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+install:
+
+tags:
+ ctags *.[ch] ../../*.[ch] ../../../include/*.h ../lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
--- /dev/null
+#ifndef __COSS_H__
+#define __COSS_H__
+
+#ifndef COSS_MEMBUF_SZ
+#define COSS_MEMBUF_SZ 1048576
+#endif
+
+#define COSS_ALLOC_NOTIFY 0
+#define COSS_ALLOC_ALLOCATE 1
+#define COSS_ALLOC_REALLOC 2
+
+struct _cossmembuf {
+ size_t diskstart;
+ size_t diskend;
+ SwapDir *SD;
+ int lockcount;
+ char buffer[COSS_MEMBUF_SZ];
+ struct _cossmembuf_flags {
+ unsigned int full:1;
+ unsigned int writing:1;
+ } flags;
+ struct _cossmembuf *next;
+};
+
+
+/* Per-storedir info */
+struct _cossinfo {
+ struct _cossmembuf *membufs;
+ struct _cossmembuf *current_membuf;
+ size_t current_offset;
+ int fd;
+ int swaplog_fd;
+ int numcollisions;
+};
+
+
+/* Per-storeiostate info */
+struct _cossstate {
+ char *readbuffer;
+ char *requestbuf;
+ size_t requestlen;
+ size_t requestoffset;
+ sfileno reqdiskoffset;
+ struct {
+ unsigned int reading:1;
+ unsigned int writing:1;
+ } flags;
+};
+
+typedef struct _cossmembuf CossMemBuf;
+typedef struct _cossinfo CossInfo;
+typedef struct _cossstate CossState;
+
+/* Whether the coss system has been setup or not */
+extern int coss_initialised;
+extern MemPool * coss_membuf_pool;
+extern MemPool * coss_state_pool;
+
+
+/*
+ * Store IO stuff
+ */
+extern STOBJCREATE storeCossCreate;
+extern STOBJOPEN storeCossOpen;
+extern STOBJCLOSE storeCossClose;
+extern STOBJREAD storeCossRead;
+extern STOBJWRITE storeCossWrite;
+extern STOBJUNLINK storeCossUnlink;
+
+extern off_t storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which);
+extern void storeCossFree(StoreEntry *e);
+extern void storeCossMaintainSwapSpace(SwapDir *SD);
+extern void storeCossDirStats(SwapDir *, StoreEntry *);
+extern void storeCossDirDump(StoreEntry * entry, const char *name, SwapDir* s);
+extern void storeCossDirFree(SwapDir *);
+extern SwapDir *storeCossDirPick(void);
+
+void storeFsSetup_ufs(storefs_entry_t *);
+
+
+
+#endif
--- /dev/null
+
+/*
+ * $Id: store_dir_coss.cc,v 1.1 2000/05/03 17:15:47 adrian Exp $
+ *
+ * DEBUG: section 81 Store COSS Directory Routines
+ * AUTHOR: Eric Stern
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+
+#include "store_coss.h"
+
+#define STORE_META_BUFSZ 4096
+
+extern void storeCossFlushMemBufs(SwapDir *SD);
+
+int n_coss_dirs = 0;
+/* static int last_coss_pick_index = -1; */
+int coss_initialised = 0;
+MemPool * coss_membuf_pool = NULL;
+MemPool * coss_state_pool = NULL;
+
+typedef struct _RebuildState RebuildState;
+struct _RebuildState {
+ SwapDir *sd;
+ int n_read;
+ FILE *log;
+ int speed;
+ int curlvl1;
+ int curlvl2;
+ struct {
+ unsigned int need_to_validate:1;
+ unsigned int clean:1;
+ unsigned int init:1;
+ } flags;
+ int done;
+ int in_dir;
+ int fn;
+ struct dirent *entry;
+ DIR *td;
+ char fullpath[SQUID_MAXPATHLEN];
+ char fullfilename[SQUID_MAXPATHLEN];
+ struct _store_rebuild_data counts;
+};
+
+static char *storeCossDirSwapLogFile(SwapDir *, const char *);
+static EVH storeCossRebuildFromSwapLog;
+static StoreEntry *storeCossAddDiskRestore(SwapDir *SD,const cache_key *key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean);
+static void storeCossDirRebuild(SwapDir * sd);
+static void storeCossDirCloseTmpSwapLog(SwapDir * sd);
+static FILE *storeCossDirOpenTmpSwapLog(SwapDir *, int *, int *);
+static STLOGOPEN storeCossDirOpenSwapLog;
+static STINIT storeCossDirInit;
+static STLOGCLEANOPEN storeCossDirWriteCleanOpen;
+static void storeCossDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANWRITE storeCossDirWriteCleanEntry;
+static STLOGCLOSE storeCossDirCloseSwapLog;
+static STLOGWRITE storeCossDirSwapLog;
+static STNEWFS storeCossDirNewfs;
+static STCHECKOBJ storeCossDirCheckObj;
+static void storeCossDirShutdown(SwapDir * sd);
+static void storeCossDirParse(SwapDir *, int, char *);
+static void storeCossDirReconfigure(SwapDir *, int, char *);
+
+static char *
+storeCossDirSwapLogFile(SwapDir * sd, const char *ext)
+{
+ LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, digit, 32);
+ char *pathtmp2;
+ if (Config.Log.swap) {
+ xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
+ while (index(pathtmp,'/'))
+ *index(pathtmp,'/')='.';
+ while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.')
+ pathtmp[strlen(pathtmp)-1]= '\0';
+ for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
+ snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2);
+ if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
+ strcat(path, ".");
+ snprintf(digit, 32, "%02d", sd->index);
+ strncat(path, digit, 3);
+ }
+ } else {
+ xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
+ strcat(path, "/swap.state");
+ }
+ if (ext)
+ strncat(path, ext, 16);
+ return path;
+}
+
+static void
+storeCossDirOpenSwapLog(SwapDir * sd)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ char *path;
+ int fd;
+ path = storeCossDirSwapLogFile(sd, NULL);
+ fd = file_open(path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(81, 1) ("%s: %s\n", path, xstrerror());
+ fatal("storeCossDirOpenSwapLog: Failed to open swap log.");
+ }
+ debug(81, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd);
+ cs->swaplog_fd = fd;
+}
+
+static void
+storeCossDirCloseSwapLog(SwapDir * sd)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ if (cs->swaplog_fd < 0) /* not open */
+ return;
+ file_close(cs->swaplog_fd);
+ debug(81, 3) ("Cache COSS Dir #%d log closed on FD %d\n",
+ sd->index, cs->swaplog_fd);
+ cs->swaplog_fd = -1;
+}
+
+static void
+storeCossDirInit(SwapDir * sd)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ storeCossDirOpenSwapLog(sd);
+ storeCossDirRebuild(sd);
+ cs->fd = file_open(sd->path, O_RDWR | O_CREAT);
+ n_coss_dirs++;
+}
+
+static void
+storeCossRebuildFromSwapLog(void *data)
+{
+ RebuildState *rb = data;
+ StoreEntry *e = NULL;
+ storeSwapLogData s;
+ size_t ss = sizeof(storeSwapLogData);
+ int count;
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ debug(81, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ store_dirs_rebuilding--;
+ storeCossDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ }
+ rb->n_read++;
+ if (s.op <= SWAP_LOG_NOP)
+ continue;
+ if (s.op >= SWAP_LOG_MAX)
+ continue;
+ debug(20, 3) ("storeCossRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ if ((e = storeGet(s.key)) != NULL) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ if (e->swap_filen > -1) {
+ e->swap_filen = -1;
+ }
+ storeRelease(e);
+ /* Fake an unlink here, this is a bad hack :( */
+ dlinkDelete(&e->repl.lru, &rb->sd->repl.lru.list);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %7d %s Entries read so far.\n",
+ rb->counts.scancount, rb->sd->path);
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ if (e) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeCossAddDiskRestore(rb->sd,s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+/* Add a new object to the cache with empty memory copy and pointer to disk
+ * use to rebuild store from disk. */
+static StoreEntry *
+storeCossAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean)
+{
+ StoreEntry *e = NULL;
+ debug(20, 5) ("storeCossAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
+ /* if you call this you'd better be sure file_number is not
+ * already in use! */
+ e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ e->store_status = STORE_OK;
+ e->swap_dirn = SD->index;
+ storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->swap_status = SWAPOUT_DONE;
+ e->swap_filen = file_number;
+ e->swap_file_sz = swap_file_sz;
+ e->lock_count = 0;
+#if !HEAP_REPLACEMENT
+ e->refcount = 0;
+#endif
+ e->lastref = lastref;
+ e->timestamp = timestamp;
+ e->expires = expires;
+ e->lastmod = lastmod;
+ e->refcount = refcount;
+ e->flags = flags;
+ EBIT_SET(e->flags, ENTRY_CACHABLE);
+ EBIT_CLR(e->flags, RELEASE_REQUEST);
+ EBIT_CLR(e->flags, KEY_PRIVATE);
+ e->ping_status = PING_NONE;
+ EBIT_CLR(e->flags, ENTRY_VALIDATED);
+ storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY);
+ return e;
+}
+
+static void
+storeCossDirRebuild(SwapDir * sd)
+{
+ RebuildState *rb = xcalloc(1, sizeof(*rb));
+ int clean = 0;
+ int zero = 0;
+ FILE *fp;
+ EVH *func = NULL;
+ rb->sd = sd;
+ rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ /*
+ * If the swap.state file exists in the cache_dir, then
+ * we'll use storeCossRebuildFromSwapLog().
+ */
+ fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero);
+ debug(20, 1) ("Rebuilding COSS storage in %s (%s)\n",
+ sd->path, clean ? "CLEAN" : "DIRTY");
+ if (!clean || fp == NULL) {
+ /* COSS cannot yet rebuild from a dirty state. If the log
+ * is dirty then the COSS contents is thrown away.
+ * Why? I guess it is because some contents will be lost,
+ * and COSS cannot verify this..
+ */
+ if (fp != NULL)
+ fclose(fp);
+ storeCossDirCloseTmpSwapLog(rb->sd);
+ /*
+ * XXX Make sure we don't trigger an assertion if this is the first
+ * storedir, since if we are, this call will cause storeRebuildComplete
+ * to prematurely complete the rebuild process, and then some other
+ * storedir will try to rebuild and eventually die.
+ */
+ store_dirs_rebuilding++;
+ storeRebuildComplete(&rb->counts);
+ store_dirs_rebuilding--;
+ xfree(rb);
+ return;
+ }
+ func = storeCossRebuildFromSwapLog;
+ rb->log = fp;
+ rb->flags.clean = (unsigned int) clean;
+ store_dirs_rebuilding++;
+ cbdataAdd(rb, cbdataXfree, 0);
+ eventAdd("storeCossRebuild", func, rb, 0.0, 1);
+}
+
+static void
+storeCossDirCloseTmpSwapLog(SwapDir * sd)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL));
+ char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new"));
+ int fd;
+ file_close(cs->swaplog_fd);
+#ifdef _SQUID_OS2_
+ if (unlink(swaplog_path) < 0) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeCossDirCloseTmpSwapLog: unlink failed");
+ }
+#endif
+ if (xrename(new_path, swaplog_path) < 0) {
+ fatal("storeCossDirCloseTmpSwapLog: rename failed");
+ }
+ fd = file_open(swaplog_path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeCossDirCloseTmpSwapLog: Failed to open swap log.");
+ }
+ safe_free(swaplog_path);
+ safe_free(new_path);
+ cs->swaplog_fd = fd;
+ debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd);
+}
+
+static FILE *
+storeCossDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL));
+ char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean"));
+ char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new"));
+ struct stat log_sb;
+ struct stat clean_sb;
+ FILE *fp;
+ int fd;
+ if (stat(swaplog_path, &log_sb) < 0) {
+ debug(47, 1) ("Cache COSS Dir #%d: No log file\n", sd->index);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return NULL;
+ }
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
+ /* close the existing write-only FD */
+ if (cs->swaplog_fd >= 0)
+ file_close(cs->swaplog_fd);
+ /* open a write-only FD for the new log */
+ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
+ }
+ cs->swaplog_fd = fd;
+ /* open a read-only stream of the old log */
+ fp = fopen(swaplog_path, "r");
+ if (fp == NULL) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("Failed to open swap log for reading");
+ }
+ memset(&clean_sb, '\0', sizeof(struct stat));
+ if (stat(clean_path, &clean_sb) < 0)
+ *clean_flag = 0;
+ else if (clean_sb.st_mtime < log_sb.st_mtime)
+ *clean_flag = 0;
+ else
+ *clean_flag = 1;
+ safeunlink(clean_path, 1);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return fp;
+}
+
+struct _clean_state {
+ char *cur;
+ char *new;
+ char *cln;
+ char *outbuf;
+ off_t outbuf_offset;
+ int fd;
+};
+
+#define CLEAN_BUF_SZ 16384
+/*
+ * Begin the process to write clean cache state. For COSS this means
+ * opening some log files and allocating write buffers. Return 0 if
+ * we succeed, and assign the 'func' and 'data' return pointers.
+ */
+static int
+storeCossDirWriteCleanOpen(SwapDir * sd)
+{
+ struct _clean_state *state = xcalloc(1, sizeof(*state));
+ struct stat sb;
+ sd->log.clean.write = NULL;
+ sd->log.clean.state = NULL;
+ state->cur = xstrdup(storeCossDirSwapLogFile(sd, NULL));
+ state->new = xstrdup(storeCossDirSwapLogFile(sd, ".clean"));
+ state->cln = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean"));
+ state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
+ state->outbuf_offset = 0;
+ unlink(state->new);
+ unlink(state->cln);
+ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
+ if (state->fd < 0)
+ return -1;
+ debug(20, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n",
+ state->new, state->fd);
+#if HAVE_FCHMOD
+ if (stat(state->cur, &sb) == 0)
+ fchmod(state->fd, sb.st_mode);
+#endif
+ sd->log.clean.write = storeCossDirWriteCleanEntry;
+ sd->log.clean.state = state;
+ return 0;
+}
+
+/*
+ * "write" an entry to the clean log file.
+ */
+static void
+storeCossDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+{
+ storeSwapLogData s;
+ static size_t ss = sizeof(storeSwapLogData);
+ struct _clean_state *state = sd->log.clean.state;
+ if (NULL == e) {
+ storeCossDirWriteCleanClose(sd);
+ return;
+ }
+ memset(&s, '\0', ss);
+ s.op = (char) SWAP_LOG_ADD;
+ s.swap_filen = e->swap_filen;
+ s.timestamp = e->timestamp;
+ s.lastref = e->lastref;
+ s.expires = e->expires;
+ s.lastmod = e->lastmod;
+ s.swap_file_sz = e->swap_file_sz;
+ s.refcount = e->refcount;
+ s.flags = e->flags;
+ xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
+ xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
+ state->outbuf_offset += ss;
+ /* buffered write */
+ if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeCossDirWriteCleanLogs: Current swap logfile not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+ }
+ state->outbuf_offset = 0;
+ }
+}
+
+static void
+storeCossDirWriteCleanClose(SwapDir * sd)
+{
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->fd < 0)
+ return;
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeCossDirWriteCleanLogs: Current swap logfile "
+ "not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ }
+ safe_free(state->outbuf);
+ /*
+ * You can't rename open files on Microsoft "operating systems"
+ * so we have to close before renaming.
+ */
+ storeCossDirCloseSwapLog(sd);
+ /* rename */
+ if (state->fd >= 0) {
+#ifdef _SQUID_OS2_
+ file_close(state->fd);
+ state->fd = -1;
+ if (unlink(cur) < 0)
+ debug(50, 0) ("storeCossDirWriteCleanLogs: unlinkd failed: %s, %s\n",
+ xstrerror(), cur);
+#endif
+ xrename(state->new, state->cur);
+ }
+ /* touch a timestamp file if we're not still validating */
+ if (store_dirs_rebuilding)
+ (void) 0;
+ else if (state->fd < 0)
+ (void) 0;
+ else
+ file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC));
+ /* close */
+ safe_free(state->cur);
+ safe_free(state->new);
+ safe_free(state->cln);
+ if (state->fd >= 0)
+ file_close(state->fd);
+ state->fd = -1;
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+}
+
+static void
+storeCossDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
+{
+ CossInfo *cs = (CossInfo *)sd->fsdata;
+ storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData));
+ s->op = (char) op;
+ s->swap_filen = e->swap_filen;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->swap_file_sz = e->swap_file_sz;
+ s->refcount = e->refcount;
+ s->flags = e->flags;
+ xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
+ file_write(cs->swaplog_fd,
+ -1,
+ s,
+ sizeof(storeSwapLogData),
+ NULL,
+ NULL,
+ xfree);
+}
+
+static void
+storeCossDirNewfs(SwapDir * sd)
+{
+ debug(47, 3) ("Creating swap space in %s\n", sd->path);
+}
+
+// we are shutting down, flush all membufs to disk
+static void
+storeCossDirShutdown(SwapDir *SD)
+{
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+ CossMemBuf *t;
+
+ // we need to do this synchronously!
+ for (t = cs->membufs; t; t = t->next) {
+ lseek(cs->fd, t->diskstart, SEEK_SET);
+ write(cs->fd, t->buffer, COSS_MEMBUF_SZ);
+ }
+ file_close(cs->fd);
+ cs->fd = -1;
+
+ if (cs->swaplog_fd > -1) {
+ file_close(cs->swaplog_fd);
+ cs->swaplog_fd = -1;
+ }
+
+ n_coss_dirs--;
+}
+
+/*
+ * storeCossDirCheckObj
+ *
+ * This routine is called by storeDirSelectSwapDir to see if the given
+ * object is able to be stored on this filesystem. COSS filesystems will
+ * not store everything. We don't check for maxobjsize here since its
+ * done by the upper layers.
+ */
+int
+storeCossDirCheckObj(SwapDir *SD, const StoreEntry *e)
+{
+ /* Check if the object is a special object, we can't cache these */
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+ return -1;
+
+ /* Otherwise, we're ok */
+ /* Return 900 (90%) load */
+ return 900;
+}
+
+/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
+
+void
+storeCossDirStats(SwapDir *SD, StoreEntry * sentry)
+{
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ storeAppendPrintf(sentry, "\n");
+ storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
+ storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
+ storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
+ 100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int)cs->numcollisions);
+#if 0
+ /* is this applicable? I Hope not .. */
+ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
+ SD->map->n_files_in_map, SD->map->max_n_files,
+ percent(SD->map->n_files_in_map, SD->map->max_n_files));
+#endif
+ storeAppendPrintf(sentry, "Flags:");
+ if (SD->flags.selected)
+ storeAppendPrintf(sentry, " SELECTED");
+ if (SD->flags.read_only)
+ storeAppendPrintf(sentry, " READ-ONLY");
+ storeAppendPrintf(sentry, "\n");
+}
+
+static void
+storeCossDirParse(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ unsigned int i;
+ unsigned int size;
+ unsigned int read_only = 0;
+ CossInfo *cs;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to Kbytes */
+ if (size <= 0)
+ fatal("storeCossDirParse: invalid size value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ cs = xmalloc(sizeof(CossInfo));
+ if (cs == NULL)
+ fatal("storeCossDirParse: couldn't xmalloc() CossInfo!\n");
+
+ sd->index = index;
+ sd->path = xstrdup(path);
+ sd->max_size = size;
+ sd->fsdata = cs;
+
+ cs->fd = -1;
+ cs->swaplog_fd = -1;
+ sd->flags.read_only = read_only;
+
+ sd->init = storeCossDirInit;
+ sd->newfs = storeCossDirNewfs;
+ sd->dump = storeCossDirDump;
+ sd->freefs = storeCossDirShutdown;
+ sd->dblcheck = NULL;
+ sd->statfs = storeCossDirStats;
+ sd->maintainfs = NULL;
+ sd->checkobj = storeCossDirCheckObj;
+ sd->refobj = NULL; /* LRU is done in storeCossRead */
+ sd->unrefobj = NULL;
+ sd->callback = NULL;
+ sd->sync = NULL; /* should we make it call the coss sync? */
+
+ sd->obj.create = storeCossCreate;
+ sd->obj.open = storeCossOpen;
+ sd->obj.close = storeCossClose;
+ sd->obj.read = storeCossRead;
+ sd->obj.write = storeCossWrite;
+ sd->obj.unlink = storeCossUnlink;
+
+ sd->log.open = storeCossDirOpenSwapLog;
+ sd->log.close = storeCossDirCloseSwapLog;
+ sd->log.write = storeCossDirSwapLog;
+ sd->log.clean.open = storeCossDirWriteCleanOpen;
+ sd->log.clean.write = storeCossDirWriteCleanEntry;
+
+ cs->current_offset = 0;
+ cs->fd = -1;
+ cs->swaplog_fd = -1;
+ cs->numcollisions = 0;
+ cs->membufs = memPoolAlloc(coss_membuf_pool);
+ cs->membufs->diskstart = 0;
+ cs->membufs->diskend = COSS_MEMBUF_SZ;
+ cs->membufs->lockcount = 0;
+ cs->membufs->flags.full = 0;
+ cs->membufs->flags.writing = 0;
+ cs->membufs->next = NULL;
+ cs->membufs->SD = sd;
+ cs->current_membuf = cs->membufs;
+
+ sd->repl.lru.list.head = NULL;
+ sd->repl.lru.list.tail = NULL;
+}
+
+
+static void
+storeCossDirReconfigure(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ unsigned int i;
+ unsigned int size;
+ unsigned int read_only = 0;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to Kbytes */
+ if (size <= 0)
+ fatal("storeCossDirParse: invalid size value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ if (size == sd->max_size)
+ debug (3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size);
+ else {
+ debug (3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size);
+ sd->max_size = size;
+ }
+
+ if (read_only != sd->flags.read_only) {
+ debug (3, 1) ("Cache COSS dir '%s' now %s\n", path, read_only ? "Read-Only" : "Read-Write");
+ sd->flags.read_only = read_only;
+ }
+}
+
+void
+storeCossDirDump(StoreEntry * entry, const char *name, SwapDir * s)
+{
+ storeAppendPrintf(entry, "%s %s %s %d\n",
+ name,
+ s->type,
+ s->path,
+ s->max_size >> 20);
+}
+
+#if 0
+SwapDir *
+storeCossDirPick(void)
+{
+ int i,choosenext = 0;
+ SwapDir *SD;
+
+ if (n_coss_dirs == 0)
+ return NULL;
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ SD = &Config.cacheSwap.swapDirs[i];
+ if (SD->type == SWAPDIR_COSS) {
+ if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) {
+ last_coss_pick_index = i;
+ return SD;
+ } else if (choosenext) {
+ last_coss_pick_index = i;
+ return SD;
+ } else if (last_coss_pick_index == i) {
+ choosenext = 1;
+ }
+ }
+ }
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ SD = &Config.cacheSwap.swapDirs[i];
+ if (SD->type == SWAPDIR_COSS) {
+ if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) {
+ last_coss_pick_index = i;
+ return SD;
+ } else if (choosenext) {
+ last_coss_pick_index = i;
+ return SD;
+ } else if (last_coss_pick_index == i) {
+ choosenext = 1;
+ }
+ }
+ }
+ return NULL;
+}
+#endif
+
+/*
+ * initial setup/done code
+ */
+static void
+storeCossDirDone(void)
+{
+ memPoolDestroy(coss_membuf_pool);
+ memPoolDestroy(coss_state_pool);
+ coss_initialised = 0;
+}
+
+void
+storeFsSetup_coss(storefs_entry_t *storefs)
+{
+ assert(!coss_initialised);
+
+ storefs->parsefunc = storeCossDirParse;
+ storefs->reconfigurefunc = storeCossDirReconfigure;
+ storefs->donefunc = storeCossDirDone;
+ coss_membuf_pool = memPoolCreate("COSS Membuf data", sizeof(CossMemBuf));
+ coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState));
+ coss_initialised = 1;
+}
+
--- /dev/null
+
+/*
+ * $Id: store_io_coss.cc,v 1.1 2000/05/03 17:15:47 adrian Exp $
+ *
+ * DEBUG: section 81 Storage Manager COSS Interface
+ * AUTHOR: Eric Stern
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "store_coss.h"
+
+static DWCB storeCossWriteMemBufDone;
+static DRCB storeCossReadDone;
+static void storeCossIOCallback(storeIOState * sio, int errflag);
+static char *storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb);
+static void storeCossMemBufLock(SwapDir *SD, storeIOState *e);
+static void storeCossMemBufUnlock(SwapDir *SD, storeIOState *e);
+static void storeCossWriteMemBuf(SwapDir *SD,CossMemBuf *t);
+static void storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data);
+static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start,
+ sfileno curfn, int *collision);
+static void storeCossIOFreeEntry(void *, int);
+static void storeCossMembufFree(void *, int);
+
+/* === PUBLIC =========================================================== */
+
+/*
+ * This routine sucks. I want to rewrite it when possible, and I also think
+ * that we should check after creatmembuf() to see if the object has a
+ * RELEASE_REQUEST set on it (thanks Eric!) rather than this way which seems
+ * to work..
+ * -- Adrian
+ */
+off_t
+storeCossAllocate(SwapDir *SD, const StoreEntry *e, int which)
+{
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+ CossMemBuf *newmb;
+ off_t retofs;
+ size_t allocsize;
+ int coll = 0;
+ sfileno checkf;
+
+ /* Make sure we chcek collisions if reallocating */
+ if (which == COSS_ALLOC_REALLOC)
+ checkf = e->swap_filen;
+ else
+ checkf = -1;
+
+ retofs = e->swap_filen; /* Just for defaults */
+
+ if (e->swap_file_sz > 0)
+ allocsize = e->swap_file_sz;
+ else
+ allocsize = objectLen(e) + e->mem_obj->swap_hdr_sz;
+
+ if (which != COSS_ALLOC_NOTIFY) {
+ if ((cs->current_offset + allocsize) > (SD->max_size << 10)) {
+ // tried to allocate past the end of the disk, so wrap
+ // back to the beginning
+ cs->current_membuf->flags.full = 1;
+ cs->current_membuf->diskend = cs->current_offset-1;
+ cs->current_offset = 0; // wrap back to beginning
+ newmb = storeCossCreateMemBuf(SD, 0, checkf, &coll);
+ cs->current_membuf = newmb;
+ } else if ((cs->current_offset + allocsize) > cs->current_membuf->diskend) {
+ cs->current_membuf->flags.full = 1;
+ cs->current_membuf->diskend = cs->current_offset-1;
+ newmb = storeCossCreateMemBuf(SD, cs->current_offset,
+ checkf, &coll);
+ cs->current_membuf = newmb;
+ }
+ if (coll == 0) {
+ retofs = cs->current_offset;
+ } else {
+ debug(81, 3) ("storeCossAllocate: Collision\n");
+ }
+ }
+ if (coll == 0) {
+ cs->current_offset += allocsize;
+ return retofs;
+ } else {
+ return -1;
+ }
+}
+
+void
+storeCossUnlink(SwapDir *SD, StoreEntry *e)
+{
+ debug(81, 3) ("storeCossUnlink: offset %d\n", e->swap_filen);
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+}
+
+
+storeIOState *
+storeCossCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data)
+{
+ CossState *cstate;
+ storeIOState *sio;
+
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeCossIOFreeEntry, MEM_STORE_IO);
+ cstate = memPoolAlloc(coss_state_pool);
+ sio->fsstate = cstate;
+ sio->offset = 0;
+ sio->mode = O_WRONLY;
+
+ /*
+ * this one is kinda strange - Eric called storeCossAllocate(), then
+ * storeCossOpen(O_RDONLY) .. weird. Anyway, I'm allocating this now.
+ */
+ sio->st_size = objectLen(e) + e->mem_obj->swap_hdr_sz;
+ sio->swap_dirn = SD->index;
+ sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_ALLOCATE);
+ debug(81, 3) ("storeCossCreate: offset %d, size %d, end %d\n", sio->swap_filen, sio->st_size, sio->swap_filen + sio->st_size);
+
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ cbdataLock(callback_data);
+ sio->e = (StoreEntry *)e;
+ sio->st_size = -1; // we won't know this until we read the metadata
+
+ cstate->flags.writing = 0;
+ cstate->flags.reading = 0;
+ cstate->readbuffer = NULL;
+ cstate->reqdiskoffset = -1;
+
+ /* Now add it into the LRU */
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+
+ storeCossMemBufLock(SD, sio);
+ return sio;
+}
+
+storeIOState *
+storeCossOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback,
+ STIOCB * callback, void *callback_data)
+{
+ storeIOState *sio;
+ char *p;
+ CossState *cstate;
+ sfileno f = e->swap_filen;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ debug(81, 3) ("storeCossOpen: offset %d\n", f);
+
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeCossIOFreeEntry, MEM_STORE_IO);
+ cstate = memPoolAlloc(coss_state_pool);
+
+ sio->fsstate = cstate;
+ sio->swap_filen = f;
+ sio->swap_dirn = SD->index;
+ sio->offset = 0;
+ sio->mode = O_RDONLY;
+ sio->callback = callback;
+ sio->file_callback = file_callback;
+ sio->callback_data = callback_data;
+ cbdataLock(callback_data);
+ sio->st_size = e->swap_file_sz;
+ sio->e = e;
+
+ cstate->flags.writing = 0;
+ cstate->flags.reading = 0;
+ cstate->readbuffer = NULL;
+ cstate->reqdiskoffset = -1;
+ p = storeCossMemPointerFromDiskOffset(SD, f, NULL);
+ // make local copy so we don't have to lock membuf
+ if (p) {
+ cstate->readbuffer = xmalloc(sio->st_size);
+ memcpy(cstate->readbuffer, p, sio->st_size);
+ } else {
+ /* Do the allocation */
+ /* this is the first time we've been called on a new sio
+ read the whole object into memory, then return the
+ requested amount
+ */
+ /*
+ * This bit of code actually does the LRU disk thing - we realloc
+ * a place for the object here, and the file_read() reads the object
+ * into the cossmembuf for later writing ..
+ */
+ cstate->reqdiskoffset = sio->swap_filen;
+ sio->swap_filen = -1;
+ sio->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_REALLOC);
+ if (sio->swap_filen == -1) {
+ /* We have to clean up neatly .. */
+ cbdataFree(sio);
+ cs->numcollisions++;
+ debug(81, 2) ("storeCossOpen: Reallocation of %d/%d failed\n", e->swap_dirn, e->swap_filen);
+ /* XXX XXX XXX Will squid call storeUnlink for this object? */
+ return NULL;
+ }
+ /* Notify the upper levels that we've changed file number */
+ sio->file_callback(sio->callback_data, 0, sio);
+
+ /*
+ * lock the buffer so it doesn't get swapped out on us
+ * this will get unlocked in storeCossReadDone
+ */
+ storeCossMemBufLock(SD, sio);
+
+ /*
+ * Do the LRU magic to keep the disk and memory LRUs identical
+ */
+ dlinkDelete(&sio->e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(sio->e, &sio->e->repl.lru, &SD->repl.lru.list);
+
+ /*
+ * Since we've reallocated a spot for this object, we need to
+ * write it to the cossmembuf *and* return it in the read ..
+ */
+ cstate->readbuffer = NULL;
+ }
+ return sio;
+}
+
+void
+storeCossClose(SwapDir *SD, storeIOState * sio)
+{
+ debug(81, 3) ("storeCossClose: offset %d\n",sio->swap_filen);
+ if (sio->mode == O_WRONLY)
+ storeCossMemBufUnlock(SD, sio);
+ storeCossIOCallback(sio, 0);
+}
+
+void
+storeCossRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+{
+ char *p;
+ CossState *cstate = (CossState *)sio->fsstate;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ assert(sio->read.callback == NULL);
+ assert(sio->read.callback_data == NULL);
+ sio->read.callback = callback;
+ sio->read.callback_data = callback_data;
+ cbdataLock(callback_data);
+ debug(81, 3) ("storeCossRead: offset %d\n", offset);
+ sio->offset = offset;
+ cstate->flags.reading = 1;
+ if ((offset + size) > sio->st_size)
+ size = sio->st_size - offset;
+ cstate->requestlen = size;
+ cstate->requestbuf = buf;
+ cstate->requestoffset = offset;
+ if (cstate->readbuffer == NULL) {
+ p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL);
+ file_read(cs->fd,
+ p,
+ sio->st_size,
+ cstate->reqdiskoffset,
+ storeCossReadDone,
+ sio);
+ cstate->reqdiskoffset = 0; /* XXX */
+ } else {
+ storeCossReadDone(cs->fd,
+ cstate->readbuffer,
+ sio->st_size,
+ 0,
+ sio);
+ }
+}
+
+void
+storeCossWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+{
+ char *dest;
+ CossMemBuf *membuf;
+ off_t diskoffset;
+
+ debug(81, 3) ("storeCossWrite: offset %d, len %d\n",sio->offset, size);
+ diskoffset = sio->swap_filen + sio->offset;
+ dest = storeCossMemPointerFromDiskOffset(SD, diskoffset, &membuf);
+ assert(dest != NULL);
+ memcpy(dest, buf, size);
+ sio->offset += size;
+ if (free_func)
+ (free_func)(buf);
+}
+
+
+/* === STATIC =========================================================== */
+
+static void
+storeCossReadDone(int fd, const char *buf, int len, int errflag, void *my_data)
+{
+ storeIOState *sio = my_data;
+ char *p;
+ STRCB *callback = sio->read.callback;
+ void *their_data = sio->read.callback_data;
+ SwapDir *SD = INDEXSD(sio->swap_dirn);
+ CossState *cstate = (CossState *)sio->fsstate;
+ size_t rlen;
+
+ debug(81, 3) ("storeCossReadDone: fileno %d, FD %d, len %d\n",
+ sio->swap_filen, fd, len);
+ cstate->flags.reading = 0;
+ if (errflag) {
+ debug(81, 3) ("storeCossReadDone: got failure (%d)\n", errflag);
+ rlen = -1;
+ } else {
+ if (cstate->readbuffer == NULL) {
+ cstate->readbuffer = xmalloc(sio->st_size);
+ p = storeCossMemPointerFromDiskOffset(SD, sio->swap_filen, NULL);
+ memcpy(cstate->readbuffer, p, sio->st_size);
+ storeCossMemBufUnlock(SD, sio);
+ }
+ sio->offset += len;
+ memcpy(cstate->requestbuf, &cstate->readbuffer[cstate->requestoffset],
+ cstate->requestlen);
+ rlen = (size_t) cstate->requestlen;
+ }
+ assert(callback);
+ assert(their_data);
+ sio->read.callback = NULL;
+ sio->read.callback_data = NULL;
+ if (cbdataValid(their_data))
+ callback(their_data, cstate->requestbuf, rlen);
+ cbdataUnlock(their_data);
+}
+
+static void
+storeCossIOCallback(storeIOState * sio, int errflag)
+{
+ CossState *cstate = (CossState *)sio->fsstate;
+ debug(81, 3) ("storeCossIOCallback: errflag=%d\n", errflag);
+ xfree(cstate->readbuffer);
+ if (cbdataValid(sio->callback_data))
+ sio->callback(sio->callback_data, errflag, sio);
+ sio->callback_data = NULL;
+ cbdataFree(sio);
+}
+
+static char *
+storeCossMemPointerFromDiskOffset(SwapDir *SD, size_t offset, CossMemBuf **mb)
+{
+ CossMemBuf *t;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ for (t=cs->membufs; t; t = t->next)
+ if ((offset >= t->diskstart) && (offset <= t->diskend)) {
+ if (mb)
+ *mb = t;
+ return &t->buffer[offset - t->diskstart];
+ }
+ if (mb)
+ *mb = NULL;
+ return NULL;
+}
+
+static void
+storeCossMemBufLock(SwapDir *SD, storeIOState *e)
+{
+ CossMemBuf *t;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ for (t = cs->membufs; t; t = t->next)
+ if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) {
+ debug(81, 3) ("storeCossMemBufLock: locking %08X, lockcount %d\n",t, t->lockcount);
+ t->lockcount++;
+ return;
+ }
+ debug(81, 3) ("storeCossMemBufLock: FAILED to lock %08X\n",e);
+}
+
+static void
+storeCossMemBufUnlock(SwapDir *SD, storeIOState *e)
+{
+ CossMemBuf *t;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ for (t = cs->membufs; t; t = t->next) {
+ if ((e->swap_filen >= t->diskstart) && (e->swap_filen <= t->diskend)) {
+ t->lockcount--;
+ debug(81, 3) ("storeCossMemBufUnlock: unlocking %08X, lockcount %d\n",t, t->lockcount);
+ }
+ if (t->flags.full && !t->flags.writing && !t->lockcount)
+ storeCossWriteMemBuf(SD, t);
+ }
+}
+
+
+static void
+storeCossWriteMemBuf(SwapDir *SD, CossMemBuf *t)
+{
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+ debug(81, 3) ("storeCossWriteMemBuf: offset %d, len %d\n",
+ t->diskstart, t->diskend - t->diskstart);
+ cbdataAdd(t, storeCossMembufFree, 0);
+ file_write(cs->fd, t->diskstart, &t->buffer,
+ t->diskend - t->diskstart, storeCossWriteMemBufDone, t, NULL);
+ t->flags.writing = 1;
+}
+
+
+static void
+storeCossWriteMemBufDone(int fd, int errflag, size_t len, void *my_data)
+{
+ CossMemBuf *t = my_data;
+ CossMemBuf *p,*prev;
+ CossInfo *cs = (CossInfo *)t->SD->fsdata;
+
+ debug(81, 3) ("storeCossWriteMemBufDone: len %d\n",len);
+ if (errflag) {
+ debug(81, 0) ("storeCossMemBufWriteDone: got failure (%d)\n", errflag);
+ cbdataFree(t);
+ return;
+ }
+
+ if (t == cs->membufs) {
+ cs->membufs = t->next;
+ cbdataFree(t);
+ return;
+ }
+ prev = t;
+ for (p = cs->membufs; p; p = p->next) {
+ if (t == p) {
+ prev->next = t->next;
+ cbdataFree(t);
+ return;
+ }
+ prev = p;
+ }
+ cbdataFree(t);
+}
+
+static CossMemBuf *storeCossCreateMemBuf(SwapDir *SD, size_t start,
+ sfileno curfn, int *collision)
+{
+ CossMemBuf *newmb,*t;
+ StoreEntry *e;
+ dlink_node *m,*prev;
+ int numreleased=0;
+ CossInfo *cs = (CossInfo *)SD->fsdata;
+
+ newmb = memPoolAlloc(coss_membuf_pool);
+ newmb->diskstart = start;
+ debug(81, 3) ("storeCossCreateMemBuf: creating new membuf at %d\n",newmb->diskstart);
+ newmb->diskend = newmb->diskstart + COSS_MEMBUF_SZ - 1;
+ newmb->flags.full = 0;
+ newmb->flags.writing = 0;
+ newmb->lockcount = 0;
+ newmb->SD = SD;
+ newmb->next = cs->membufs;
+ cs->membufs = newmb;
+ for (t = cs->membufs; t; t = t->next)
+ debug(81, 3) ("storeCossCreateMemBuf: membuflist %d lockcount %d\n",t->diskstart,t->lockcount);
+
+ /*
+ * XXX more evil LRU specific code. This needs to be replaced.
+ */
+ for (m = SD->repl.lru.list.tail; m; m = prev) {
+ prev = m->prev;
+ e = m->data;
+ if (curfn == e->swap_filen)
+ *collision = 1; /* Mark an object alloc collision */
+ if ((e->swap_filen >= newmb->diskstart) &&
+ (e->swap_filen <= newmb->diskend)) {
+ storeRelease(e);
+ numreleased++;
+ } else
+ break;
+ }
+ if (numreleased > 0)
+ debug(81, 3) ("storeCossCreateMemBuf: this allocation released %d storeEntries\n",numreleased);
+ return newmb;
+}
+
+/*
+ * We can't pass memFree() as a free function here, because we need to free
+ * the fsstate variable ..
+ */
+static void
+storeCossIOFreeEntry(void *sio, int foo)
+{
+ memPoolFree(coss_state_pool, ((storeIOState *)sio)->fsstate);
+ memFree(sio, MEM_STORE_IO);
+}
+
+/*
+ * We can't pass memFree() as a free function here, since we have to pass it
+ * an int to memFree(), and we aren't using static memory pool allocation here.
+ * So we have this hack here ..
+ */
+static void
+storeCossMembufFree(void *mb, int foo)
+{
+ memPoolFree(coss_membuf_pool, mb);
+}
+
--- /dev/null
+#
+# Makefile for the DISKD storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/05/03 17:15:47 adrian Exp $
+#
+
+FS = diskd
+
+prefix = @prefix@
+exec_prefix = @exec_prefix@
+exec_suffix = @exec_suffix@
+cgi_suffix = @cgi_suffix@
+top_srcdir = @top_srcdir@
+bindir = @bindir@
+libexecdir = @libexecdir@
+sysconfdir = @sysconfdir@
+localstatedir = @localstatedir@
+srcdir = @srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+LDFLAGS = @LDFLAGS@
+INSTALL = @INSTALL@
+INSTALL_BIN = @INSTALL_PROGRAM@
+MV = @MV@
+RM = @RM@
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(FS).a
+DISKD_EXE = diskd$(exec_suffix)
+
+
+OBJS = \
+ store_dir_diskd.o \
+ store_io_diskd.o
+
+UTILS = \
+ $(DISKD_EXE)
+
+all: $(OUT) $(UTILS)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) -DSQUID_PREFIX=\"$(prefix)\" $(CFLAGS) -c $<
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a diskd
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+tags:
+ ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
+
+diskd.o: $(srcdir)/diskd.c
+ $(CC) $(CFLAGS) $(srcdir)/diskd.c -c -o diskd.o
+
+$(DISKD_EXE): diskd.o
+ $(CC) $(LDFLAGS) -L../../../lib/ $(CFLAGS) diskd.o -o $(DISKD_EXE) -lmiscutil -lm
+
+install: $(UTILS)
+ @for f in $(UTILS); do \
+ if test -f $(libexecdir)/$$f; then \
+ echo $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \
+ $(MV) $(libexecdir)/$$f $(libexecdir)/-$$f; \
+ fi; \
+ echo $(INSTALL_BIN) $$f $(libexecdir); \
+ $(INSTALL_BIN) $$f $(libexecdir); \
+ if test -f $(libexecdir)/-$$f; then \
+ echo $(RM) -f $(libexecdir)/-$$f; \
+ $(RM) -f $(libexecdir)/-$$f; \
+ fi; \
+ done
+
--- /dev/null
+
+#include "config.h"
+#include "squid.h"
+
+
+#include "store_diskd.h"
+
+#include <sys/ipc.h>
+#include <sys/msg.h>
+#include <sys/shm.h>
+
+#undef assert
+#include <assert.h>
+
+
+#define STDERR_DEBUG 0
+
+typedef struct _file_state file_state;
+
+struct _file_state {
+ void *key;
+ file_state *next;
+ int id;
+ int fd;
+ off_t offset;
+};
+
+static hash_table *hash = NULL;
+static pid_t mypid;
+static char *shmbuf;
+
+static int
+do_open(diomsg * r, int len, const char *buf)
+{
+ int fd;
+ file_state *fs;
+ /*
+ * note r->offset holds open() flags
+ */
+ fd = open(buf, r->offset, 0600);
+ if (fd < 0) {
+ fprintf(stderr, "%d %s: ", (int) mypid, buf);
+ perror("open");
+ return -errno;
+ }
+ fs = xcalloc(1, sizeof(*fs));
+ fs->id = r->id;
+ fs->key = &fs->id; /* gack */
+ fs->fd = fd;
+ hash_join(hash, (hash_link *) fs);
+#if STDERR_DEBUG
+ fprintf(stderr, "%d OPEN id %d, FD %d, fs %p\n",
+ (int) mypid,
+ fs->id,
+ fs->fd,
+ fs);
+#endif
+ return fd;
+}
+
+static int
+do_close(diomsg * r, int len)
+{
+ int fd;
+ file_state *fs;
+ fs = (file_state *) hash_lookup(hash, &r->id);
+ if (NULL == fs) {
+ errno = EBADF;
+ fprintf(stderr, "%d CLOSE id %d: ", (int) mypid, r->id);
+ perror("do_close");
+ return -EBADF;
+ }
+ fd = fs->fd;
+ hash_remove_link(hash, (hash_link *) fs);
+#if STDERR_DEBUG
+ fprintf(stderr, "%d CLOSE id %d, FD %d, fs %p\n",
+ (int) mypid,
+ r->id,
+ fs->fd,
+ fs);
+#endif
+ xfree(fs);
+ return close(fd);
+}
+
+static int
+do_read(diomsg * r, int len, char *buf)
+{
+ int x;
+ int readlen = r->size;
+ file_state *fs;
+ fs = (file_state *) hash_lookup(hash, &r->id);
+ if (NULL == fs) {
+ errno = EBADF;
+ fprintf(stderr, "%d READ id %d: ", (int) mypid, r->id);
+ perror("do_read");
+ return -EBADF;
+ }
+ if (r->offset > -1 && r->offset != fs->offset) {
+#if STDERR_DEBUG
+ fprintf(stderr, "seeking to %d\n", r->offset);
+#endif
+ if (lseek(fs->fd, r->offset, SEEK_SET) < 0) {
+ fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset);
+ perror("lseek");
+ }
+ }
+ x = read(fs->fd, buf, readlen);
+#if STDERR_DEBUG
+ fprintf(stderr, "%d READ %d,%d,%d ret %d\n", (int) mypid,
+ fs->fd, readlen, r->offset, x);
+#endif
+ if (x < 0) {
+ fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd);
+ perror("read");
+ return -errno;
+ }
+ fs->offset = r->offset + x;
+ return x;
+}
+
+static int
+do_write(diomsg * r, int len, const char *buf)
+{
+ int wrtlen = r->size;
+ int x;
+ file_state *fs;
+ fs = (file_state *) hash_lookup(hash, &r->id);
+ if (NULL == fs) {
+ errno = EBADF;
+ fprintf(stderr, "%d WRITE id %d: ", (int) mypid, r->id);
+ perror("do_write");
+ return -EBADF;
+ }
+ if (r->offset > -1 && r->offset != fs->offset) {
+ if (lseek(fs->fd, r->offset, SEEK_SET) < 0) {
+ fprintf(stderr, "%d FD %d, offset %d: ", (int) mypid, fs->fd, r->offset);
+ perror("lseek");
+ }
+ }
+#if STDERR_DEBUG
+ fprintf(stderr, "%d WRITE %d,%d,%d\n", (int) mypid,
+ fs->fd, wrtlen, r->offset);
+#endif
+ x = write(fs->fd, buf, wrtlen);
+ if (x < 0) {
+ fprintf(stderr, "%d FD %d: ", (int) mypid, fs->fd);
+ perror("write");
+ return -errno;
+ }
+ fs->offset = r->offset + x;
+ return x;
+}
+
+static int
+do_unlink(diomsg * r, int len, const char *buf)
+{
+ if (truncate(buf, 0) < 0) {
+ fprintf(stderr, "%d UNLNK id %d %s: ", (int) mypid, r->id, buf);
+ perror("truncate");
+ return -errno;
+ }
+#if STDERR_DEBUG
+ fprintf(stderr, "%d UNLNK %s\n", (int) mypid, buf);
+#endif
+ return 0;
+}
+
+static void
+msg_handle(diomsg * r, int rl, diomsg * s)
+{
+ char *buf = NULL;
+ s->mtype = r->mtype;
+ s->callback_data = r->callback_data;
+ s->shm_offset = r->shm_offset;
+ s->id = r->id;
+ if (s->shm_offset > -1)
+ buf = shmbuf + s->shm_offset;
+ switch (r->mtype) {
+ case _MQD_OPEN:
+ s->status = do_open(r, rl, buf);
+ break;
+ case _MQD_CLOSE:
+ s->status = do_close(r, rl);
+ break;
+ case _MQD_READ:
+ s->status = do_read(r, rl, buf);
+ break;
+ case _MQD_WRITE:
+ s->status = do_write(r, rl, buf);
+ break;
+ case _MQD_UNLINK:
+ s->status = do_unlink(r, rl, buf);
+ break;
+ default:
+ assert(0);
+ break;
+ }
+}
+
+int
+fsCmp(const void *a, const void *b)
+{
+ const int *A = a;
+ const int *B = b;
+ return *A != *B;
+}
+
+unsigned int
+fsHash(const void *key, unsigned int n)
+{
+ /* note, n must be a power of 2! */
+ const int *k = key;
+ return (*k & (--n));
+}
+
+static void
+alarm_handler(int sig)
+{
+ (void) 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+ int key;
+ int rmsgid;
+ int smsgid;
+ int shmid;
+ diomsg rmsg;
+ diomsg smsg;
+ int rlen;
+ char rbuf[512];
+ struct sigaction sa;
+ setbuf(stdout, NULL);
+ setbuf(stderr, NULL);
+ mypid = getpid();
+ assert(4 == argc);
+ key = atoi(argv[1]);
+ rmsgid = msgget(key, 0600);
+ if (rmsgid < 0) {
+ perror("msgget");
+ return 1;
+ }
+ key = atoi(argv[2]);
+ smsgid = msgget(key, 0600);
+ if (smsgid < 0) {
+ perror("msgget");
+ return 1;
+ }
+ key = atoi(argv[3]);
+ shmid = shmget(key, 0, 0600);
+ if (shmid < 0) {
+ perror("shmget");
+ return 1;
+ }
+ shmbuf = shmat(shmid, NULL, 0);
+ if (shmbuf == (void *) -1) {
+ perror("shmat");
+ return 1;
+ }
+ hash = hash_create(fsCmp, 1 << 4, fsHash);
+ assert(hash);
+ fcntl(0, F_SETFL, SQUID_NONBLOCK);
+ memset(&sa, '\0', sizeof(sa));
+ sa.sa_handler = alarm_handler;
+ sa.sa_flags = SA_RESTART;
+ sigaction(SIGALRM, &sa, NULL);
+ for (;;) {
+ alarm(1);
+ memset(&rmsg, '\0', sizeof(rmsg));
+ rlen = msgrcv(rmsgid, &rmsg, msg_snd_rcv_sz, 0, 0);
+ if (rlen < 0) {
+ if (EINTR == errno) {
+ if (read(0, rbuf, 512) <= 0) {
+ if (EWOULDBLOCK == errno)
+ (void) 0;
+ else if (EAGAIN == errno)
+ (void) 0;
+ else
+ break;
+ }
+ }
+ if (EAGAIN == errno) {
+ continue;
+ }
+ perror("msgrcv");
+ break;
+ }
+ alarm(0);
+ msg_handle(&rmsg, rlen, &smsg);
+ if (msgsnd(smsgid, &smsg, msg_snd_rcv_sz, 0) < 0) {
+ perror("msgsnd");
+ break;
+ }
+ }
+#if STDERR_DEBUG
+ fprintf(stderr, "%d diskd exiting\n", (int) mypid);
+#endif
+ if (msgctl(rmsgid, IPC_RMID, 0) < 0)
+ perror("msgctl IPC_RMID");
+ if (msgctl(smsgid, IPC_RMID, 0) < 0)
+ perror("msgctl IPC_RMID");
+ if (shmdt(shmbuf) < 0)
+ perror("shmdt");
+ if (shmctl(shmid, IPC_RMID, 0) < 0)
+ perror("shmctl IPC_RMID");
+ return 0;
+}
+
--- /dev/null
+
+/*
+ * $Id: store_dir_diskd.cc,v 1.1 2000/05/03 17:15:47 adrian Exp $
+ *
+ * DEBUG: section 47 Store Directory Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#if HAVE_STATVFS
+#if HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+#endif
+
+#include <sys/ipc.h>
+#include <sys/msg.h>
+#include <sys/shm.h>
+
+#include "store_diskd.h"
+
+#define DefaultLevelOneDirs 16
+#define DefaultLevelTwoDirs 256
+#define STORE_META_BDISKDZ 4096
+
+#ifndef SQUID_PREFIX
+#error "SQUID_PREFIX needs defining!"
+#endif
+
+diskd_stats_t diskd_stats;
+
+typedef struct _RebuildState RebuildState;
+struct _RebuildState {
+ SwapDir *sd;
+ int n_read;
+ FILE *log;
+ int speed;
+ int curlvl1;
+ int curlvl2;
+ struct {
+ unsigned int need_to_validate:1;
+ unsigned int clean:1;
+ unsigned int init:1;
+ } flags;
+ int done;
+ int in_dir;
+ int fn;
+ struct dirent *entry;
+ DIR *td;
+ char fullpath[SQUID_MAXPATHLEN];
+ char fullfilename[SQUID_MAXPATHLEN];
+ struct _store_rebuild_data counts;
+};
+
+static int n_diskd_dirs = 0;
+static int *diskd_dir_index = NULL;
+MemPool * diskd_state_pool = NULL;
+static int diskd_initialised = 0;
+
+static char *storeDiskdDirSwapSubDir(SwapDir *, int subdirn);
+static int storeDiskdDirCreateDirectory(const char *path, int);
+static int storeDiskdDirVerifyCacheDirs(SwapDir *);
+static int storeDiskdDirVerifyDirectory(const char *path);
+static void storeDiskdDirCreateSwapSubDirs(SwapDir *);
+static char *storeDiskdDirSwapLogFile(SwapDir *, const char *);
+static EVH storeDiskdDirRebuildFromDirectory;
+static EVH storeDiskdDirRebuildFromSwapLog;
+static int storeDiskdDirGetNextFile(RebuildState *, int *sfileno, int *size);
+static StoreEntry *storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean);
+static void storeDiskdDirRebuild(SwapDir * sd);
+static void storeDiskdDirCloseTmpSwapLog(SwapDir * sd);
+static FILE *storeDiskdDirOpenTmpSwapLog(SwapDir *, int *, int *);
+static STLOGOPEN storeDiskdDirOpenSwapLog;
+static STINIT storeDiskdDirInit;
+static STFREE storeDiskdDirFree;
+static STLOGCLEANOPEN storeDiskdDirWriteCleanOpen;
+static void storeDiskdDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANWRITE storeDiskdDirWriteCleanEntry;
+static STLOGCLOSE storeDiskdDirCloseSwapLog;
+static STLOGWRITE storeDiskdDirSwapLog;
+static STNEWFS storeDiskdDirNewfs;
+static STDUMP storeDiskdDirDump;
+static STMAINTAINFS storeDiskdDirMaintain;
+static STCHECKOBJ storeDiskdDirCheckObj;
+static STREFOBJ storeDiskdDirRefObj;
+static STUNREFOBJ storeDiskdDirUnrefObj;
+static QS rev_int_sort;
+static int storeDiskdDirClean(int swap_index);
+static EVH storeDiskdDirCleanEvent;
+static int storeDiskdDirIs(SwapDir * sd);
+static int storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2);
+static int storeDiskdCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static void storeDiskdDirStats(SwapDir *, StoreEntry *);
+static void storeDiskdDirInitBitmap(SwapDir *);
+static int storeDiskdDirValidFileno(SwapDir *, sfileno);
+static int storeDiskdDirCheckExpired(SwapDir *, StoreEntry *);
+#if !HEAP_REPLACEMENT
+static time_t storeDiskdDirExpiredReferenceAge(SwapDir *);
+#endif
+static void storeDiskdStats(StoreEntry * sentry);
+static void storeDiskdDirSync(SwapDir *);
+static void storeDiskdDirCallback(SwapDir *);
+
+
+/*
+ * These functions were ripped straight out of the heart of store_dir.c.
+ * They assume that the given filenum is on a diskd partiton, which may or
+ * may not be true..
+ * XXX this evilness should be tidied up at a later date!
+ */
+
+int
+storeDiskdDirMapBitTest(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ diskdinfo_t *diskdinfo;
+ diskdinfo = (diskdinfo_t *)SD->fsdata;
+ return file_map_bit_test(diskdinfo->map, filn);
+}
+
+void
+storeDiskdDirMapBitSet(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ diskdinfo_t *diskdinfo;
+ diskdinfo = (diskdinfo_t *)SD->fsdata;
+ file_map_bit_set(diskdinfo->map, filn);
+}
+
+void
+storeDiskdDirMapBitReset(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ diskdinfo_t *diskdinfo;
+ diskdinfo = (diskdinfo_t *)SD->fsdata;
+ file_map_bit_reset(diskdinfo->map, filn);
+}
+
+int
+storeDiskdDirMapBitAllocate(SwapDir *SD)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+ int fn;
+ fn = file_map_allocate(diskdinfo->map, diskdinfo->suggest);
+ file_map_bit_set(diskdinfo->map, fn);
+ diskdinfo->suggest = fn + 1;
+ return fn;
+}
+
+/*
+ * Initialise the diskd bitmap
+ *
+ * If there already is a bitmap, and the numobjects is larger than currently
+ * configured, we allocate a new bitmap and 'grow' the old one into it.
+ */
+static void
+storeDiskdDirInitBitmap(SwapDir *sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+
+ if (diskdinfo->map == NULL) {
+ /* First time */
+ diskdinfo->map = file_map_create();
+ } else if (diskdinfo->map->max_n_files) {
+ /* it grew, need to expand */
+ /* XXX We don't need it anymore .. */
+ }
+ /* else it shrunk, and we leave the old one in place */
+}
+
+static char *
+storeDiskdDirSwapSubDir(SwapDir * sd, int subdirn)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ assert(0 <= subdirn && subdirn < diskdinfo->l1);
+ snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
+ return fullfilename;
+}
+
+static int
+storeDiskdDirCreateDirectory(const char *path, int should_exist)
+{
+ int created = 0;
+ struct stat st;
+ getCurrentTime();
+ if (0 == stat(path, &st)) {
+ if (S_ISDIR(st.st_mode)) {
+ debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
+ } else {
+ fatalf("Swap directory %s is not a directory.", path);
+ }
+ } else if (0 == mkdir(path, 0755)) {
+ debug(20, should_exist ? 1 : 3) ("%s created\n", path);
+ created = 1;
+ } else {
+ fatalf("Failed to make swap directory %s: %s",
+ path, xstrerror());
+ }
+ return created;
+}
+
+static int
+storeDiskdDirVerifyDirectory(const char *path)
+{
+ struct stat sb;
+ if (stat(path, &sb) < 0) {
+ debug(20, 0) ("%s: %s\n", path, xstrerror());
+ return -1;
+ }
+ if (S_ISDIR(sb.st_mode) == 0) {
+ debug(20, 0) ("%s is not a directory\n", path);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * This function is called by storeDiskdDirInit(). If this returns < 0,
+ * then Squid exits, complains about swap directories not
+ * existing, and instructs the admin to run 'squid -z'
+ */
+static int
+storeDiskdDirVerifyCacheDirs(SwapDir * sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ int j;
+ const char *path = sd->path;
+
+ if (storeDiskdDirVerifyDirectory(path) < 0)
+ return -1;
+ for (j = 0; j < diskdinfo->l1; j++) {
+ path = storeDiskdDirSwapSubDir(sd, j);
+ if (storeDiskdDirVerifyDirectory(path) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void
+storeDiskdDirCreateSwapSubDirs(SwapDir * sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ int i, k;
+ int should_exist;
+ LOCAL_ARRAY(char, name, MAXPATHLEN);
+ for (i = 0; i < diskdinfo->l1; i++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
+ if (storeDiskdDirCreateDirectory(name, 0))
+ should_exist = 0;
+ else
+ should_exist = 1;
+ debug(47, 1) ("Making directories in %s\n", name);
+ for (k = 0; k < diskdinfo->l2; k++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
+ storeDiskdDirCreateDirectory(name, should_exist);
+ }
+ }
+}
+
+static char *
+storeDiskdDirSwapLogFile(SwapDir * sd, const char *ext)
+{
+ LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, digit, 32);
+ char *pathtmp2;
+ if (Config.Log.swap) {
+ xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
+ while (index(pathtmp,'/'))
+ *index(pathtmp,'/')='.';
+ while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.')
+ pathtmp[strlen(pathtmp)-1]= '\0';
+ for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
+ snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2);
+ if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
+ strcat(path, ".");
+ snprintf(digit, 32, "%02d", sd->index);
+ strncat(path, digit, 3);
+ }
+ } else {
+ xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
+ strcat(path, "/swap.state");
+ }
+ if (ext)
+ strncat(path, ext, 16);
+ return path;
+}
+
+static void
+storeDiskdDirOpenSwapLog(SwapDir * sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ char *path;
+ int fd;
+ path = storeDiskdDirSwapLogFile(sd, NULL);
+ fd = file_open(path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", path, xstrerror());
+ fatal("storeDiskdDirOpenSwapLog: Failed to open swap log.");
+ }
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+ diskdinfo->swaplog_fd = fd;
+ if (0 == n_diskd_dirs)
+ assert(NULL == diskd_dir_index);
+ n_diskd_dirs++;
+ assert(n_diskd_dirs <= Config.cacheSwap.n_configured);
+}
+
+static void
+storeDiskdDirCloseSwapLog(SwapDir * sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ if (diskdinfo->swaplog_fd < 0) /* not open */
+ return;
+ file_close(diskdinfo->swaplog_fd);
+ debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
+ sd->index, diskdinfo->swaplog_fd);
+ diskdinfo->swaplog_fd = -1;
+ n_diskd_dirs--;
+ assert(n_diskd_dirs >= 0);
+ if (0 == n_diskd_dirs)
+ safe_free(diskd_dir_index);
+}
+
+static void
+storeDiskdDirInit(SwapDir * sd)
+{
+ static int started_clean_event = 0;
+ int x;
+ int i;
+ int rfd;
+ int ikey = (getpid() << 16) + (sd->index << 4);
+ char *args[5];
+ char skey1[32];
+ char skey2[32];
+ char skey3[32];
+ diskdinfo_t *diskdinfo = (diskdinfo_t *) sd->fsdata;
+ static const char *errmsg =
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
+
+ diskdinfo->smsgid = msgget((key_t) ikey, 0700 | IPC_CREAT);
+ if (diskdinfo->smsgid < 0) {
+ debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror());
+ fatal("msgget failed");
+ }
+ diskdinfo->rmsgid = msgget((key_t) (ikey + 1), 0700 | IPC_CREAT);
+ if (diskdinfo->rmsgid < 0) {
+ debug(50, 0) ("storeDiskdInit: msgget: %s\n", xstrerror());
+ fatal("msgget failed");
+ }
+ diskdinfo->shm.id = shmget((key_t) (ikey + 2),
+ SHMBUFS * SHMBUF_BLKSZ, 0600 | IPC_CREAT);
+ if (diskdinfo->shm.id < 0) {
+ debug(50, 0) ("storeDiskdInit: shmget: %s\n", xstrerror());
+ fatal("shmget failed");
+ }
+ diskdinfo->shm.buf = shmat(diskdinfo->shm.id, NULL, 0);
+ if (diskdinfo->shm.buf == (void *) -1) {
+ debug(50, 0) ("storeDiskdInit: shmat: %s\n", xstrerror());
+ fatal("shmat failed");
+ }
+ diskd_stats.shmbuf_count += SHMBUFS;
+ for (i = 0; i < SHMBUFS; i++)
+ storeDiskdShmPut(sd, i * SHMBUF_BLKSZ);
+ snprintf(skey1, 32, "%d", ikey);
+ snprintf(skey2, 32, "%d", ikey + 1);
+ snprintf(skey3, 32, "%d", ikey + 2);
+ args[0] = "diskd";
+ args[1] = skey1;
+ args[2] = skey2;
+ args[3] = skey3;
+ args[4] = NULL;
+#if HAVE_POLL && defined(_SQUID_OSF_)
+ /* pipes and poll() don't get along on DUNIX -DW */
+ x = ipcCreate(IPC_TCP_SOCKET,
+#else
+ x = ipcCreate(IPC_FIFO,
+#endif
+ SQUID_PREFIX "/bin/diskd",
+ args,
+ "diskd",
+ &rfd,
+ &diskdinfo->wfd);
+ if (x < 0)
+ fatal("execl " SQUID_PREFIX "/bin/diskd failed");
+ if (rfd != diskdinfo->wfd)
+ comm_close(rfd);
+ fd_note(diskdinfo->wfd, "squid -> diskd");
+ commSetTimeout(diskdinfo->wfd, -1, NULL, NULL);
+ commSetNonBlocking(diskdinfo->wfd);
+ storeDiskdDirInitBitmap(sd);
+ if (storeDiskdDirVerifyCacheDirs(sd) < 0)
+ fatal(errmsg);
+ storeDiskdDirOpenSwapLog(sd);
+ storeDiskdDirRebuild(sd);
+ if (!started_clean_event) {
+ eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL, 15.0, 1);
+ started_clean_event = 1;
+ }
+}
+
+
+static void
+storeDiskdStats(StoreEntry * sentry)
+{
+ storeAppendPrintf(sentry, "sent_count: %d\n", diskd_stats.sent_count);
+ storeAppendPrintf(sentry, "recv_count: %d\n", diskd_stats.recv_count);
+ storeAppendPrintf(sentry, "max_away: %d\n", diskd_stats.max_away);
+ storeAppendPrintf(sentry, "max_shmuse: %d\n", diskd_stats.max_shmuse);
+ storeAppendPrintf(sentry, "open_fail_queue_len: %d\n", diskd_stats.open_fail_queue_len);
+ storeAppendPrintf(sentry, "block_queue_len: %d\n", diskd_stats.block_queue_len);
+ diskd_stats.max_away = diskd_stats.max_shmuse = 0;
+}
+
+/*
+ * storeDiskdDirSync
+ *
+ * Sync any pending data. We just sit around and read the queue
+ * until the data has finished writing.
+ */
+static void
+storeDiskdDirSync(SwapDir *SD)
+{
+ /* XXX NOT DONE YET! */
+#warning "storeDiskdSync() needs to be written"
+}
+
+
+/*
+ * storeDiskdDirCallback
+ *
+ * Handle callbacks. If we have more than magic2 requests away, we block
+ * until the queue is below magic2. Otherwise, we simply return when we
+ * don't get a message.
+ */
+static void
+storeDiskdDirCallback(SwapDir *SD)
+{
+ diomsg M;
+ int x;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+
+ if (diskdinfo->away >= diskdinfo->magic2)
+ diskd_stats.block_queue_len++;
+
+ if (diskd_stats.sent_count - diskd_stats.recv_count >
+ diskd_stats.max_away) {
+ diskd_stats.max_away = diskd_stats.sent_count - diskd_stats.recv_count;
+ diskd_stats.max_shmuse = diskd_stats.shmbuf_count;
+ }
+
+ /* if we are above magic2, we do not break under any reason */
+ while (1) {
+ memset(&M, '\0', sizeof(M));
+ x = msgrcv(diskdinfo->rmsgid, &M, msg_snd_rcv_sz, 0, IPC_NOWAIT);
+ if (x < 0) {
+ if (diskdinfo->away >= diskdinfo->magic2)
+ continue;
+ else
+ break;
+ } else if (x != msg_snd_rcv_sz) {
+ debug(81, 1) ("storeDiskdReadIndividualQueue: msgget returns %d\n",
+ x);
+ break;
+ }
+ diskd_stats.recv_count++;
+ diskdinfo->away--;
+ storeDiskdHandle(&M);
+ if (M.shm_offset > -1)
+ storeDiskdShmPut(SD, M.shm_offset);
+ }
+}
+
+
+
+static void
+storeDiskdDirRebuildFromDirectory(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
+ StoreEntry *e = NULL;
+ StoreEntry tmpe;
+ cache_key key[MD5_DIGEST_CHARS];
+ int sfileno = 0;
+ int count;
+ int size;
+ struct stat sb;
+ int swap_hdr_len;
+ int fd = -1;
+ tlv *tlv_list;
+ tlv *t;
+ assert(rb != NULL);
+ debug(20, 3) ("storeDiskdDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
+ for (count = 0; count < rb->speed; count++) {
+ assert(fd == -1);
+ fd = storeDiskdDirGetNextFile(rb, &sfileno, &size);
+ if (fd == -2) {
+ debug(20, 1) ("Done scanning %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ store_dirs_rebuilding--;
+ storeDiskdDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ } else if (fd < 0) {
+ continue;
+ }
+ assert(fd > -1);
+ /* lets get file stats here */
+ if (fstat(fd, &sb) < 0) {
+ debug(20, 1) ("storeDiskdDirRebuildFromDirectory: fstat(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %s %7d files opened so far.\n",
+ rb->sd->path, rb->counts.scancount);
+ debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
+ Counter.syscalls.disk.reads++;
+ if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
+ debug(20, 1) ("storeDiskdDirRebuildFromDirectory: read(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ swap_hdr_len = 0;
+#if USE_TRUNCATE
+ if (sb.st_size == 0)
+ continue;
+#endif
+ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
+ if (tlv_list == NULL) {
+ debug(20, 1) ("storeDiskdDirRebuildFromDirectory: failed to get meta data\n");
+ /* XXX shouldn't this be a call to storeDiskdUnlink ? */
+ storeDiskdDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ debug(20, 3) ("storeDiskdDirRebuildFromDirectory: successful swap meta unpacking\n");
+ memset(key, '\0', MD5_DIGEST_CHARS);
+ memset(&tmpe, '\0', sizeof(StoreEntry));
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_KEY:
+ assert(t->length == MD5_DIGEST_CHARS);
+ xmemcpy(key, t->value, MD5_DIGEST_CHARS);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ default:
+ break;
+ }
+ }
+ storeSwapTLVFree(tlv_list);
+ tlv_list = NULL;
+ if (storeKeyNull(key)) {
+ debug(20, 1) ("storeDiskdDirRebuildFromDirectory: NULL key\n");
+ storeDiskdDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ tmpe.key = key;
+ /* check sizes */
+ if (tmpe.swap_file_sz == 0) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeDiskdDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
+ tmpe.swap_file_sz, (int) sb.st_size);
+ storeDiskdDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
+ storeDiskdDirUnlinkFile(SD, sfileno);
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(key);
+ if (e && e->lastref >= tmpe.lastref) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (NULL != e) {
+ /* URL already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeRelease(e); /* release old entry */
+ rb->counts.dupcount++;
+ }
+ rb->counts.objcount++;
+ storeEntryDump(&tmpe, 5);
+ e = storeDiskdDirAddDiskRestore(SD, key,
+ sfileno,
+ tmpe.swap_file_sz,
+ tmpe.expires,
+ tmpe.timestamp,
+ tmpe.lastref,
+ tmpe.lastmod,
+ tmpe.refcount, /* refcount */
+ tmpe.flags, /* flags */
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromDirectory, rb, 0.0, 1);
+}
+
+static void
+storeDiskdDirRebuildFromSwapLog(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogData s;
+ size_t ss = sizeof(storeSwapLogData);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ debug(20, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ store_dirs_rebuilding--;
+ storeDiskdDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ }
+ rb->n_read++;
+ if (s.op <= SWAP_LOG_NOP)
+ continue;
+ if (s.op >= SWAP_LOG_MAX)
+ continue;
+ debug(20, 3) ("storeDiskdDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ if ((e = storeGet(s.key)) != NULL) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeDiskdDirReplRemove(e);
+ if (e->swap_filen > -1) {
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %7d %s Entries read so far.\n",
+ rb->counts.scancount, rb->sd->path);
+ if (!storeDiskdDirValidFileno(SD, s.swap_filen)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeDiskdDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+#if HEAP_REPLACEMENT
+ storeHeapPositionUpdate(e, SD);
+ storeDiskdDirUnrefObj(SD, e);
+#endif
+ } else {
+ debug_trap("storeDiskdDirRebuildFromSwapLog: bad condition");
+ debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
+ continue;
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeDiskdDirReplRemove(e);
+ if (e->swap_filen > -1) {
+ /* Make sure we don't actually unlink the file */
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeDiskdDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeDiskdDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+static int
+storeDiskdDirGetNextFile(RebuildState * rb, int *sfileno, int *size)
+{
+ SwapDir *SD = rb->sd;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+ int fd = -1;
+ int used = 0;
+ int dirs_opened = 0;
+ debug(20, 3) ("storeDiskdDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
+ rb->flags.init,
+ rb->sd->index,
+ rb->curlvl1,
+ rb->curlvl2);
+ if (rb->done)
+ return -2;
+ while (fd < 0 && rb->done == 0) {
+ fd = -1;
+ if (0 == rb->flags.init) { /* initialize, open first file */
+ rb->done = 0;
+ rb->curlvl1 = 0;
+ rb->curlvl2 = 0;
+ rb->in_dir = 0;
+ rb->flags.init = 1;
+ assert(Config.cacheSwap.n_configured > 0);
+ }
+ if (0 == rb->in_dir) { /* we need to read in a new directory */
+ snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ rb->sd->path,
+ rb->curlvl1, rb->curlvl2);
+ if (rb->flags.init && rb->td != NULL)
+ closedir(rb->td);
+ rb->td = NULL;
+ if (dirs_opened)
+ return -1;
+ rb->td = opendir(rb->fullpath);
+ dirs_opened++;
+ if (rb->td == NULL) {
+ debug(50, 1) ("storeDiskdDirGetNextFile: opendir: %s: %s\n",
+ rb->fullpath, xstrerror());
+ } else {
+ rb->entry = readdir(rb->td); /* skip . and .. */
+ rb->entry = readdir(rb->td);
+ if (rb->entry == NULL && errno == ENOENT)
+ debug(20, 1) ("storeDiskdDirGetNextFile: directory does not exist!.\n");
+ debug(20, 3) ("storeDiskdDirGetNextFile: Directory %s\n", rb->fullpath);
+ }
+ }
+ if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
+ rb->in_dir++;
+ if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
+ debug(20, 3) ("storeDiskdDirGetNextFile: invalid %s\n",
+ rb->entry->d_name);
+ continue;
+ }
+ if (!storeDiskdFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
+ debug(20, 3) ("storeDiskdDirGetNextFile: %08X does not belong in %d/%d/%d\n",
+ rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
+ continue;
+ }
+ used = storeDiskdDirMapBitTest(SD, rb->fn);
+ if (used) {
+ debug(20, 3) ("storeDiskdDirGetNextFile: Locked, continuing with next.\n");
+ continue;
+ }
+ snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
+ rb->fullpath, rb->entry->d_name);
+ debug(20, 3) ("storeDiskdDirGetNextFile: Opening %s\n", rb->fullfilename);
+ fd = file_open(rb->fullfilename, O_RDONLY);
+ if (fd < 0)
+ debug(50, 1) ("storeDiskdDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
+ else
+ store_open_disk_fd++;
+ continue;
+ }
+ rb->in_dir = 0;
+ if (++rb->curlvl2 < diskdinfo->l2)
+ continue;
+ rb->curlvl2 = 0;
+ if (++rb->curlvl1 < diskdinfo->l1)
+ continue;
+ rb->curlvl1 = 0;
+ rb->done = 1;
+ }
+ *sfileno = rb->fn;
+ return fd;
+}
+
+/* Add a new object to the cache with empty memory copy and pointer to disk
+ * use to rebuild store from disk. */
+static StoreEntry *
+storeDiskdDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean)
+{
+ StoreEntry *e = NULL;
+ debug(20, 5) ("storeDiskdAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
+ /* if you call this you'd better be sure file_number is not
+ * already in use! */
+ e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ e->store_status = STORE_OK;
+ storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->swap_status = SWAPOUT_DONE;
+ e->swap_filen = file_number;
+ e->swap_dirn = SD->index;
+ e->swap_file_sz = swap_file_sz;
+ e->lock_count = 0;
+#if !HEAP_REPLACEMENT
+ e->refcount = 0;
+#endif
+ e->lastref = lastref;
+ e->timestamp = timestamp;
+ e->expires = expires;
+ e->lastmod = lastmod;
+ e->refcount = refcount;
+ e->flags = flags;
+ EBIT_SET(e->flags, ENTRY_CACHABLE);
+ EBIT_CLR(e->flags, RELEASE_REQUEST);
+ EBIT_CLR(e->flags, KEY_PRIVATE);
+ e->ping_status = PING_NONE;
+ EBIT_CLR(e->flags, ENTRY_VALIDATED);
+ storeDiskdDirMapBitSet(SD, e->swap_filen);
+ storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
+ storeDiskdDirReplAdd(SD, e);
+ return e;
+}
+
+static void
+storeDiskdDirRebuild(SwapDir * sd)
+{
+ RebuildState *rb = xcalloc(1, sizeof(*rb));
+ int clean = 0;
+ int zero = 0;
+ FILE *fp;
+ EVH *func = NULL;
+ rb->sd = sd;
+ rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ /*
+ * If the swap.state file exists in the cache_dir, then
+ * we'll use storeDiskdDirRebuildFromSwapLog(), otherwise we'll
+ * use storeDiskdDirRebuildFromDirectory() to open up each file
+ * and suck in the meta data.
+ */
+ fp = storeDiskdDirOpenTmpSwapLog(sd, &clean, &zero);
+ if (fp == NULL || zero) {
+ if (fp != NULL)
+ fclose(fp);
+ func = storeDiskdDirRebuildFromDirectory;
+ } else {
+ func = storeDiskdDirRebuildFromSwapLog;
+ rb->log = fp;
+ rb->flags.clean = (unsigned int) clean;
+ }
+ if (!clean)
+ rb->flags.need_to_validate = 1;
+ debug(20, 1) ("Rebuilding storage in %s (%s)\n",
+ sd->path, clean ? "CLEAN" : "DIRTY");
+ store_dirs_rebuilding++;
+ cbdataAdd(rb, cbdataXfree, 0);
+ eventAdd("storeRebuild", func, rb, 0.0, 1);
+}
+
+static void
+storeDiskdDirCloseTmpSwapLog(SwapDir * sd)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
+ char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new"));
+ int fd;
+ file_close(diskdinfo->swaplog_fd);
+#ifdef _SQUID_OS2_
+ if (unlink(swaplog_path) < 0) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeDiskdDirCloseTmpSwapLog: unlink failed");
+ }
+#endif
+ if (xrename(new_path, swaplog_path) < 0) {
+ fatal("storeDiskdDirCloseTmpSwapLog: rename failed");
+ }
+ fd = file_open(swaplog_path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeDiskdDirCloseTmpSwapLog: Failed to open swap log.");
+ }
+ safe_free(swaplog_path);
+ safe_free(new_path);
+ diskdinfo->swaplog_fd = fd;
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+}
+
+static FILE *
+storeDiskdDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
+ char *clean_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean"));
+ char *new_path = xstrdup(storeDiskdDirSwapLogFile(sd, ".new"));
+ struct stat log_sb;
+ struct stat clean_sb;
+ FILE *fp;
+ int fd;
+ if (stat(swaplog_path, &log_sb) < 0) {
+ debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return NULL;
+ }
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
+ /* close the existing write-only FD */
+ if (diskdinfo->swaplog_fd >= 0)
+ file_close(diskdinfo->swaplog_fd);
+ /* open a write-only FD for the new log */
+ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
+ }
+ diskdinfo->swaplog_fd = fd;
+ /* open a read-only stream of the old log */
+ fp = fopen(swaplog_path, "r");
+ if (fp == NULL) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("Failed to open swap log for reading");
+ }
+ memset(&clean_sb, '\0', sizeof(struct stat));
+ if (stat(clean_path, &clean_sb) < 0)
+ *clean_flag = 0;
+ else if (clean_sb.st_mtime < log_sb.st_mtime)
+ *clean_flag = 0;
+ else
+ *clean_flag = 1;
+ safeunlink(clean_path, 1);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return fp;
+}
+
+struct _clean_state {
+ char *cur;
+ char *new;
+ char *cln;
+ char *outbuf;
+ off_t outbuf_offset;
+ int fd;
+};
+
+#define CLEAN_BUF_SZ 16384
+/*
+ * Begin the process to write clean cache state. For DISKD this means
+ * opening some log files and allocating write buffers. Return 0 if
+ * we succeed, and assign the 'func' and 'data' return pointers.
+ */
+static int
+storeDiskdDirWriteCleanOpen(SwapDir * sd)
+{
+ struct _clean_state *state = xcalloc(1, sizeof(*state));
+ struct stat sb;
+ sd->log.clean.write = NULL;
+ sd->log.clean.state = NULL;
+ state->cur = xstrdup(storeDiskdDirSwapLogFile(sd, NULL));
+ state->new = xstrdup(storeDiskdDirSwapLogFile(sd, ".clean"));
+ state->cln = xstrdup(storeDiskdDirSwapLogFile(sd, ".last-clean"));
+ state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
+ state->outbuf_offset = 0;
+ unlink(state->new);
+ unlink(state->cln);
+ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
+ if (state->fd < 0)
+ return -1;
+ debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
+ state->new, state->fd);
+#if HAVE_FCHMOD
+ if (stat(state->cur, &sb) == 0)
+ fchmod(state->fd, sb.st_mode);
+#endif
+ sd->log.clean.write = storeDiskdDirWriteCleanEntry;
+ sd->log.clean.state = state;
+ return 0;
+}
+
+/*
+ * "write" an entry to the clean log file.
+ */
+static void
+storeDiskdDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+{
+ storeSwapLogData s;
+ static size_t ss = sizeof(storeSwapLogData);
+ struct _clean_state *state = sd->log.clean.state;
+ if (NULL == e) {
+ storeDiskdDirWriteCleanClose(sd);
+ return;
+ }
+ memset(&s, '\0', ss);
+ s.op = (char) SWAP_LOG_ADD;
+ s.swap_filen = e->swap_filen;
+ s.timestamp = e->timestamp;
+ s.lastref = e->lastref;
+ s.expires = e->expires;
+ s.lastmod = e->lastmod;
+ s.swap_file_sz = e->swap_file_sz;
+ s.refcount = e->refcount;
+ s.flags = e->flags;
+ xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
+ xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
+ state->outbuf_offset += ss;
+ /* buffered write */
+ if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+ }
+ state->outbuf_offset = 0;
+ }
+}
+
+static void
+storeDiskdDirWriteCleanClose(SwapDir * sd)
+{
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->fd < 0)
+ return;
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile "
+ "not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ }
+ safe_free(state->outbuf);
+ /*
+ * You can't rename open files on Microsoft "operating systems"
+ * so we have to close before renaming.
+ */
+ storeDiskdDirCloseSwapLog(sd);
+ /* rename */
+ if (state->fd >= 0) {
+#ifdef _SQUID_OS2_
+ file_close(state->fd);
+ state->fd = -1;
+ if (unlink(cur) < 0)
+ debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
+ xstrerror(), cur);
+#endif
+ xrename(state->new, state->cur);
+ }
+ /* touch a timestamp file if we're not still validating */
+ if (store_dirs_rebuilding)
+ (void) 0;
+ else if (state->fd < 0)
+ (void) 0;
+ else
+ file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC));
+ /* close */
+ safe_free(state->cur);
+ safe_free(state->new);
+ safe_free(state->cln);
+ if (state->fd >= 0)
+ file_close(state->fd);
+ state->fd = -1;
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+}
+
+static void
+storeDiskdDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData));
+ s->op = (char) op;
+ s->swap_filen = e->swap_filen;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->swap_file_sz = e->swap_file_sz;
+ s->refcount = e->refcount;
+ s->flags = e->flags;
+ xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
+ file_write(diskdinfo->swaplog_fd,
+ -1,
+ s,
+ sizeof(storeSwapLogData),
+ NULL,
+ NULL,
+ xfree);
+}
+
+static void
+storeDiskdDirNewfs(SwapDir * sd)
+{
+ debug(47, 3) ("Creating swap space in %s\n", sd->path);
+ storeDiskdDirCreateDirectory(sd->path, 0);
+ storeDiskdDirCreateSwapSubDirs(sd);
+}
+
+static int
+rev_int_sort(const void *A, const void *B)
+{
+ const int *i1 = A;
+ const int *i2 = B;
+ return *i2 - *i1;
+}
+
+static int
+storeDiskdDirClean(int swap_index)
+{
+ DIR *dp = NULL;
+ struct dirent *de = NULL;
+ LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
+ LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
+#if USE_TRUNCATE
+ struct stat sb;
+#endif
+ int files[20];
+ int swapfileno;
+ int fn; /* same as swapfileno, but with dirn bits set */
+ int n = 0;
+ int k = 0;
+ int N0, N1, N2;
+ int D0, D1, D2;
+ SwapDir *SD;
+ diskdinfo_t *diskdinfo;
+ N0 = n_diskd_dirs;
+ D0 = diskd_dir_index[swap_index % N0];
+ SD = &Config.cacheSwap.swapDirs[D0];
+ diskdinfo = (diskdinfo_t *)SD->fsdata;
+ N1 = diskdinfo->l1;
+ D1 = (swap_index / N0) % N1;
+ N2 = diskdinfo->l2;
+ D2 = ((swap_index / N0) / N1) % N2;
+ snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ Config.cacheSwap.swapDirs[D0].path, D1, D2);
+ debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
+ dp = opendir(p1);
+ if (dp == NULL) {
+ if (errno == ENOENT) {
+ debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
+ if (mkdir(p1, 0777) == 0)
+ return 0;
+ }
+ debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
+ safeunlink(p1, 1);
+ return 0;
+ }
+ while ((de = readdir(dp)) != NULL && k < 20) {
+ if (sscanf(de->d_name, "%X", &swapfileno) != 1)
+ continue;
+ fn = swapfileno; /* XXX should remove this cruft ! */
+ if (storeDiskdDirValidFileno(SD, fn))
+ if (storeDiskdDirMapBitTest(SD, fn))
+ if (storeDiskdFilenoBelongsHere(fn, D0, D1, D2))
+ continue;
+#if USE_TRUNCATE
+ if (!stat(de->d_name, &sb))
+ if (sb.st_size == 0)
+ continue;
+#endif
+ files[k++] = swapfileno;
+ }
+ closedir(dp);
+ if (k == 0)
+ return 0;
+ qsort(files, k, sizeof(int), rev_int_sort);
+ if (k > 10)
+ k = 10;
+ for (n = 0; n < k; n++) {
+ debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
+ snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
+#if USE_TRUNCATE
+ truncate(p2, 0);
+#else
+ safeunlink(p2, 0);
+#endif
+ Counter.swap_files_cleaned++;
+ }
+ debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
+ return k;
+}
+
+static void
+storeDiskdDirCleanEvent(void *unused)
+{
+ static int swap_index = 0;
+ int i;
+ int j = 0;
+ int n = 0;
+ /*
+ * Assert that there are DISKD cache_dirs configured, otherwise
+ * we should never be called.
+ */
+ assert(n_diskd_dirs);
+ if (NULL == diskd_dir_index) {
+ SwapDir *sd;
+ diskdinfo_t *diskdinfo;
+ /*
+ * Initialize the little array that translates DISKD cache_dir
+ * number into the Config.cacheSwap.swapDirs array index.
+ */
+ diskd_dir_index = xcalloc(n_diskd_dirs, sizeof(*diskd_dir_index));
+ for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
+ sd = &Config.cacheSwap.swapDirs[i];
+ if (!storeDiskdDirIs(sd))
+ continue;
+ diskd_dir_index[n++] = i;
+ diskdinfo = (diskdinfo_t *)sd->fsdata;
+ j += (diskdinfo->l1 * diskdinfo->l2);
+ }
+ assert(n == n_diskd_dirs);
+ /*
+ * Start the storeDiskdDirClean() swap_index with a random
+ * value. j equals the total number of DISKD level 2
+ * swap directories
+ */
+ swap_index = (int) (squid_random() % j);
+ }
+ if (0 == store_dirs_rebuilding) {
+ n = storeDiskdDirClean(swap_index);
+ swap_index++;
+ }
+ eventAdd("storeDirClean", storeDiskdDirCleanEvent, NULL,
+ 15.0 * exp(-0.25 * n), 1);
+}
+
+static int
+storeDiskdDirIs(SwapDir * sd)
+{
+ if (strncmp(sd->type, "diskd", 3) == 0)
+ return 1;
+ return 0;
+}
+
+/*
+ * Does swapfile number 'fn' belong in cachedir #F0,
+ * level1 dir #F1, level2 dir #F2?
+ */
+static int
+storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2)
+{
+ int D1, D2;
+ int L1, L2;
+ int filn = fn;
+ diskdinfo_t *diskdinfo;
+ assert(F0 < Config.cacheSwap.n_configured);
+ diskdinfo = (diskdinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata;
+ L1 = diskdinfo->l1;
+ L2 = diskdinfo->l2;
+ D1 = ((filn / L2) / L2) % L1;
+ if (F1 != D1)
+ return 0;
+ D2 = (filn / L2) % L2;
+ if (F2 != D2)
+ return 0;
+ return 1;
+}
+
+int
+storeDiskdDirValidFileno(SwapDir *SD, sfileno filn)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+ if (filn < 0)
+ return 0;
+ if (filn > diskdinfo->map->max_n_files)
+ return 0;
+ return 1;
+}
+
+void
+storeDiskdDirMaintain(SwapDir *SD)
+{
+ StoreEntry *e = NULL;
+ int scanned = 0;
+ int locked = 0;
+ int expired = 0;
+ int max_scan;
+ int max_remove;
+ double f;
+ static time_t last_warn_time = 0;
+#if !HEAP_REPLACEMENT
+ dlink_node *m;
+ dlink_node *prev = NULL;
+#else
+ heap_key age;
+ heap_key min_age = 0.0;
+ link_list *locked_entries = NULL;
+#if HEAP_REPLACEMENT_DEBUG
+ if (!verify_heap_property(SD->repl.heap.heap)) {
+ debug(20, 1) ("Heap property violated!\n");
+ }
+#endif
+#endif
+ /* We can't delete objects while rebuilding swap */
+ if (store_dirs_rebuilding) {
+ return;
+ } else {
+ f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
+ f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
+ max_scan = (int) (f * 400.0 + 100.0);
+ max_remove = (int) (f * 70.0 + 10.0);
+ /*
+ * This is kinda cheap, but so we need this priority hack?
+ */
+#if 0
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
+#endif
+ }
+ debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
+#if HEAP_REPLACEMENT
+ while (heap_nodes(SD->repl.heap.heap) > 0) {
+ if (store_swap_size < store_swap_low)
+ break;
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ age = heap_peepminkey(SD->repl.heap.heap);
+ e = heap_extractmin(SD->repl.heap.heap);
+ e->repl.node = NULL; /* no longer in the heap */
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * Entry is in use ... put it in a linked list to ignore it.
+ */
+ if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ /*
+ * If this was a "SPECIAL" do not add it back into the heap.
+ * It will always be "SPECIAL" and therefore never removed.
+ */
+ debug(20, 4) ("storeDiskdDirMaintain: locked url %s\n",
+ (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
+key));
+ linklistPush(&locked_entries, e);
+ }
+ locked++;
+ continue;
+ } else if (storeDiskdDirCheckExpired(SD, e)) {
+ /*
+ * Note: This will not check the reference age ifdef
+ * HEAP_REPLACEMENT, but it does some other useful
+ * checks...
+ */
+ expired++;
+ debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
+ age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
+ min_age = age;
+ storeRelease(e);
+ } else {
+ /*
+ * Did not expire the object so we need to add it back
+ * into the heap!
+ */
+ debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
+ storeKeyText(e->key));
+ linklistPush(&locked_entries, e);
+ continue;
+ }
+ if (store_swap_size < store_swap_low)
+ break;
+ else if (expired >= max_remove)
+ break;
+ else if (scanned >= max_scan)
+ break;
+ }
+ /*
+ * Bump the heap age factor.
+ */
+ if (min_age > 0.0)
+ SD->repl.heap.heap->age = min_age;
+ /*
+ * Reinsert all bumped locked entries back into heap...
+ */
+ while ((e = linklistShift(&locked_entries)))
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+#else
+ for (m = SD->repl.lru.list.tail; m; m = prev) {
+ prev = m->prev;
+ e = m->data;
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * If there is a locked entry at the tail of the LRU list,
+ * move it to the beginning to get it out of the way.
+ * Theoretically, we might have all locked objects at the
+ * tail, and then we'll never remove anything here and the
+ * LRU age will go to zero.
+ */
+ if (memInUse(MEM_STOREENTRY) > max_scan) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+ locked++;
+
+ } else if (storeDiskdDirCheckExpired(SD, e)) {
+ expired++;
+ storeRelease(e);
+ }
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ }
+#endif
+ debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l
+ocked %d f=%.03f\n",
+ scanned, max_scan, expired, max_remove, locked, f);
+ debug(20, 3) ("storeMaintainSwapSpace stats:\n");
+ debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
+ debug(20, 3) (" %6d were scanned\n", scanned);
+ debug(20, 3) (" %6d were locked\n", locked);
+ debug(20, 3) (" %6d were expired\n", expired);
+ if (store_swap_size < Config.Swap.maxSize)
+ return;
+ if (squid_curtime - last_warn_time < 10)
+ return;
+ debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
+ store_swap_size, Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
+}
+
+/*
+ * storeDiskdDirCheckObj
+ *
+ * This routine is called by storeDirSelectSwapDir to see if the given
+ * object is able to be stored on this filesystem. DISKD filesystems will
+ * happily store anything as long as the LRU time isn't too small.
+ */
+int
+storeDiskdDirCheckObj(SwapDir *SD, const StoreEntry *e)
+{
+ int loadav;
+
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+#if !HEAP_REPLACEMENT
+ if (storeDiskdDirExpiredReferenceAge(SD) < 300) {
+ debug(20, 3) ("storeDiskdDirCheckObj: NO: LRU Age = %d\n",
+ storeDiskdDirExpiredReferenceAge(SD));
+ /* store_check_cachable_hist.no.lru_age_too_low++; */
+ return -1;
+ }
+#endif
+
+ /* Check the queue length */
+ if (diskdinfo->away >= diskdinfo->magic1)
+ return -1;
+
+ /* Calculate the storedir load relative to magic2 on a scale of 0 .. 1000 */
+ if (diskdinfo->away == 0)
+ loadav = 0;
+ else
+ loadav = diskdinfo->magic2 * 1000 / diskdinfo->away;
+ return loadav;
+}
+
+/*
+ * storeDiskdDirRefObj
+ *
+ * This routine is called whenever an object is referenced, so we can
+ * maintain replacement information within the storage fs.
+ */
+void
+storeDiskdDirRefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeDiskdDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
+ e->swap_filen);
+#if HEAP_REPLACEMENT
+ /* Nothing to do here */
+#else
+ /* Reference the object */
+ if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
+ !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+#endif
+}
+
+/*
+ * storeDiskdDirUnrefObj
+ * This routine is called whenever the last reference to an object is
+ * removed, to maintain replacement information within the storage fs.
+ */
+void
+storeDiskdDirUnrefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeDiskdDirUnrefObj: referencing %p %d/%d\n", e,
+ e->swap_dirn, e->swap_filen);
+#if HEAP_REPLACEMENT
+ if (e->repl.node)
+ heap_update(SD->repl.heap.heap, e->repl.node, e);
+#endif
+}
+
+/*
+ * storeDiskdDirUnlinkFile
+ *
+ * This is a *synchronous* unlink which is currently used in the rebuild
+ * process. This is bad, but it'll have to stay until the dir rebuild
+ * uses storeDiskdUnlink() ..
+ */
+void
+storeDiskdDirUnlinkFile(SwapDir *SD, sfileno f)
+{
+ debug(79, 3) ("storeDiskdDirUnlinkFile: unlinking fileno %08X\n", f);
+ storeDiskdDirMapBitReset(SD, f);
+ unlinkdUnlink(storeDiskdDirFullPath(SD, f, NULL));
+}
+
+#if !HEAP_REPLACEMENT
+/*
+ * storeDiskdDirExpiredReferenceAge
+ *
+ * The LRU age is scaled exponentially between 1 minute and
+ * Config.referenceAge , when store_swap_low < store_swap_size <
+ * store_swap_high. This keeps store_swap_size within the low and high
+ * water marks. If the cache is very busy then store_swap_size stays
+ * closer to the low water mark, if it is not busy, then it will stay
+ * near the high water mark. The LRU age value can be examined on the
+ * cachemgr 'info' page.
+ */
+static time_t
+storeDiskdDirExpiredReferenceAge(SwapDir *SD)
+{
+ double x;
+ double z;
+ time_t age;
+ long store_high, store_low;
+
+ store_high = (long) (((float) SD->max_size *
+ (float) Config.Swap.highWaterMark) / (float) 100);
+ store_low = (long) (((float) SD->max_size *
+ (float) Config.Swap.lowWaterMark) / (float) 100);
+ debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
+
+ x = (double) (store_high - SD->cur_size) /
+ (store_high - store_low);
+ x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
+ z = pow((double) (Config.referenceAge / 60), x);
+ age = (time_t) (z * 60.0);
+ if (age < 60)
+ age = 60;
+ else if (age > Config.referenceAge)
+ age = Config.referenceAge;
+ return age;
+}
+#endif
+
+/*
+ * storeDiskdDirCheckExpired
+ *
+ * Check whether the given object is expired or not
+ * It breaks layering a little by calling the upper layers to find
+ * out whether the object is locked or not, but we can't help this
+ * right now.
+ */
+static int
+storeDiskdDirCheckExpired(SwapDir *SD, StoreEntry *e)
+{
+ if (storeEntryLocked(e))
+ return 0;
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ return 1;
+ if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
+ return 1;
+
+#if HEAP_REPLACEMENT
+ /*
+ * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
+ * controls the replacement of objects.
+ */
+ return 1;
+#else
+ if (squid_curtime - e->lastref > storeDiskdDirExpiredReferenceAge(SD))
+ return 1;
+ return 0;
+#endif
+}
+
+/*
+ * Add and remove the given StoreEntry from the replacement policy in
+ * use.
+ */
+
+void
+storeDiskdDirReplAdd(SwapDir *SD, StoreEntry *e)
+{
+ debug(20, 4) ("storeDiskdDirReplAdd: added node %p to dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ (void) 0;
+ } else {
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+ debug(20, 4) ("storeDiskdDirReplAdd: inserted node 0x%x\n", e->repl.node);
+ }
+#else
+ /* Shouldn't we not throw special objects into the lru ? */
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+void
+storeDiskdDirReplRemove(StoreEntry *e)
+{
+ SwapDir *SD = INDEXSD(e->swap_dirn);
+ debug(20, 4) ("storeDiskdDirReplRemove: remove node %p from dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ /* And now, release the object from the replacement policy */
+ if (e->repl.node) {
+ debug(20, 4) ("storeDiskdDirReplRemove: deleting node 0x%x\n",
+ e->repl.node);
+ heap_delete(SD->repl.heap.heap, e->repl.node);
+ e->repl.node = NULL;
+ }
+#else
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+
+/*
+ * SHM manipulation routines
+ */
+
+void *
+storeDiskdShmGet(SwapDir * sd, int *shm_offset)
+{
+ char *buf;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ buf = linklistShift(&diskdinfo->shm.stack);
+ assert(buf);
+ *shm_offset = buf - diskdinfo->shm.buf;
+ assert(0 <= *shm_offset && *shm_offset < SHMBUFS * SHMBUF_BLKSZ);
+ diskd_stats.shmbuf_count++;
+ return buf;
+}
+
+void
+storeDiskdShmPut(SwapDir * sd, int offset)
+{
+ char *buf;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ assert(offset >= 0);
+ assert(offset < SHMBUFS * SHMBUF_BLKSZ);
+ buf = diskdinfo->shm.buf + offset;
+ linklistPush(&diskdinfo->shm.stack, buf);
+ diskd_stats.shmbuf_count--;
+}
+
+
+
+
+/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
+
+void
+storeDiskdDirStats(SwapDir *SD, StoreEntry * sentry)
+{
+ diskdinfo_t *diskdinfo;
+#if HAVE_STATVFS
+ struct statvfs sfs;
+#endif
+ diskdinfo = (diskdinfo_t *)SD->fsdata;
+ storeAppendPrintf(sentry, "First level subdirectories: %d\n", diskdinfo->l1);
+ storeAppendPrintf(sentry, "Second level subdirectories: %d\n", diskdinfo->l2);
+ storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
+ storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
+ storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
+ 100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
+ diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files,
+ percent(diskdinfo->map->n_files_in_map, diskdinfo->map->max_n_files));
+#if HAVE_STATVFS
+#define fsbtoblk(num, fsbs, bs) \
+ (((fsbs) != 0 && (fsbs) < (bs)) ? \
+ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
+ if (!statvfs(SD->path, &sfs)) {
+ storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
+ fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024),
+ fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024),
+ percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
+ storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
+ sfs.f_files - sfs.f_ffree, sfs.f_files,
+ percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
+ }
+#endif
+ storeAppendPrintf(sentry, "Flags:");
+ if (SD->flags.selected)
+ storeAppendPrintf(sentry, " SELECTED");
+ if (SD->flags.read_only)
+ storeAppendPrintf(sentry, " READ-ONLY");
+ storeAppendPrintf(sentry, "\n");
+#if !HEAP_REPLACEMENT
+ storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
+ (double) storeDiskdDirExpiredReferenceAge(SD) / 86400.0);
+#else
+#if 0
+ storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
+ heap_peepminkey(sd.repl.heap.heap));
+#endif
+#endif
+ storeAppendPrintf(sentry, "Pending operations: %d\n", diskdinfo->away);
+}
+
+/*
+ * storeDiskdDirReconfigure
+ *
+ * This routine is called when the given swapdir needs reconfiguring
+ */
+void
+storeDiskdDirReconfigure(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ int magic1, magic2;
+ unsigned int read_only = 0;
+ diskdinfo_t *diskdinfo;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeDiskdDirReconfigure: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeDiskdDirReconfigure: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeDiskdDirReconfigure: invalid level 2 directories value");
+ i = GetInteger();
+ magic1 = i;
+ if (magic1 <= 0)
+ fatal("storeDiskdDirParse: invalid magic1 value");
+ i = GetInteger();
+ magic2 = i;
+ if (magic2 <= 0)
+ fatal("storeDiskdDirParse: invalid magic2 value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ /* just reconfigure it */
+ if (size == sd->max_size)
+ debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n",
+ path, size);
+ else
+ debug(3, 1) ("Cache dir '%s' size changed to %d KB\n",
+ path, size);
+ sd->max_size = size;
+ if (sd->flags.read_only != read_only)
+ debug(3, 1) ("Cache dir '%s' now %s\n",
+ path, read_only ? "Read-Only" : "Read-Write");
+ diskdinfo = (diskdinfo_t *)sd->fsdata;
+ diskdinfo->magic1 = magic1;
+ diskdinfo->magic2 = magic2;
+ sd->flags.read_only = read_only;
+ return;
+}
+
+void
+storeDiskdDirDump(StoreEntry * entry, const char *name, SwapDir * s)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata;
+ storeAppendPrintf(entry, "%s %s %s %d %d %d\n",
+ name,
+ "diskd",
+ s->path,
+ s->max_size >> 10,
+ diskdinfo->l1,
+ diskdinfo->l2);
+}
+
+/*
+ * Only "free" the filesystem specific stuff here
+ */
+static void
+storeDiskdDirFree(SwapDir * s)
+{
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)s->fsdata;
+ if (diskdinfo->swaplog_fd > -1) {
+ file_close(diskdinfo->swaplog_fd);
+ diskdinfo->swaplog_fd = -1;
+ }
+ filemapFreeMemory(diskdinfo->map);
+ xfree(diskdinfo);
+ s->fsdata = NULL; /* Will aid debugging... */
+
+}
+
+char *
+storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath)
+{
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+ int L1 = diskdinfo->l1;
+ int L2 = diskdinfo->l2;
+ if (!fullpath)
+ fullpath = fullfilename;
+ fullpath[0] = '\0';
+ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
+ SD->path,
+ ((filn / L2) / L2) % L1,
+ (filn / L2) % L2,
+ filn);
+ return fullpath;
+}
+
+/*
+ * storeDiskdCleanupDoubleCheck
+ *
+ * This is called by storeCleanup() if -S was given on the command line.
+ */
+static int
+storeDiskdCleanupDoubleCheck(SwapDir *sd, StoreEntry *e)
+{
+ struct stat sb;
+
+ if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n");
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
+ storeDiskdDirFullPath(sd, e->swap_filen, NULL));
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ if (e->swap_file_sz != sb.st_size) {
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n");
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
+ storeDiskdDirFullPath(sd, e->swap_filen, NULL));
+ debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
+ e->swap_file_sz, (int) sb.st_size);
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * storeDiskdDirParse
+ *
+ * Called when a *new* fs is being setup.
+ */
+void
+storeDiskdDirParse(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ int magic1, magic2;
+ unsigned int read_only = 0;
+ diskdinfo_t *diskdinfo;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeDiskdDirParse: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeDiskdDirParse: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeDiskdDirParse: invalid level 2 directories value");
+ i = GetInteger();
+ magic1 = i;
+ if (magic1 <= 0)
+ fatal("storeDiskdDirParse: invalid magic1 value");
+ i = GetInteger();
+ magic2 = i;
+ if (magic2 <= 0)
+ fatal("storeDiskdDirParse: invalid magic2 value");
+
+
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ diskdinfo = xmalloc(sizeof(diskdinfo_t));
+ if (diskdinfo == NULL)
+ fatal("storeDiskdDirParse: couldn't xmalloc() diskdinfo_t!\n");
+
+ sd->index = index;
+ sd->path = xstrdup(path);
+ sd->max_size = size;
+ sd->fsdata = diskdinfo;
+ diskdinfo->l1 = l1;
+ diskdinfo->l2 = l2;
+ diskdinfo->swaplog_fd = -1;
+ diskdinfo->map = NULL; /* Debugging purposes */
+ diskdinfo->suggest = 0;
+ diskdinfo->magic1 = magic1;
+ diskdinfo->magic2 = magic2;
+ sd->flags.read_only = read_only;
+ sd->init = storeDiskdDirInit;
+ sd->newfs = storeDiskdDirNewfs;
+ sd->dump = storeDiskdDirDump;
+ sd->freefs = storeDiskdDirFree;
+ sd->dblcheck = storeDiskdCleanupDoubleCheck;
+ sd->statfs = storeDiskdDirStats;
+ sd->maintainfs = storeDiskdDirMaintain;
+ sd->checkobj = storeDiskdDirCheckObj;
+ sd->refobj = storeDiskdDirRefObj;
+ sd->unrefobj = storeDiskdDirUnrefObj;
+ sd->callback = storeDiskdDirCallback;
+ sd->sync = storeDiskdDirSync;
+ sd->obj.create = storeDiskdCreate;
+ sd->obj.open = storeDiskdOpen;
+ sd->obj.close = storeDiskdClose;
+ sd->obj.read = storeDiskdRead;
+ sd->obj.write = storeDiskdWrite;
+ sd->obj.unlink = storeDiskdUnlink;
+ sd->log.open = storeDiskdDirOpenSwapLog;
+ sd->log.close = storeDiskdDirCloseSwapLog;
+ sd->log.write = storeDiskdDirSwapLog;
+ sd->log.clean.open = storeDiskdDirWriteCleanOpen;
+
+ /* Initialise replacement policy stuff */
+#if HEAP_REPLACEMENT
+ /*
+ * Create new heaps with cache replacement policies attached to them.
+ * The cache replacement policy is specified as either GDSF or LFUDA in
+ * the squid.conf configuration file. Note that the replacement policy
+ * applies only to the disk replacement algorithm. Memory replacement
+ * always uses GDSF since we want to maximize object hit rate.
+ */
+ if (Config.replPolicy) {
+ if (tolower(Config.replPolicy[0]) == 'g') {
+ debug(20, 1) ("Using GDSF disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ } else if (tolower(Config.replPolicy[0]) == 'l') {
+ if (tolower(Config.replPolicy[1]) == 'f') {
+ debug(20, 1) ("Using LFUDA disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
+ } else if (tolower(Config.replPolicy[1]) == 'r') {
+ debug(20, 1) ("Using LRU heap disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
+ }
+ } else {
+ debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+ } else {
+ debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+#else
+ sd->repl.lru.list.head = NULL;
+ sd->repl.lru.list.tail = NULL;
+#endif
+}
+
+/*
+ * Initial setup / end destruction
+ */
+void
+storeDiskdDirDone(void)
+{
+ memPoolDestroy(diskd_state_pool);
+ diskd_initialised = 0;
+}
+
+void
+storeFsSetup_diskd(storefs_entry_t *storefs)
+{
+ assert(!diskd_initialised);
+ storefs->parsefunc = storeDiskdDirParse;
+ storefs->reconfigurefunc = storeDiskdDirReconfigure;
+ storefs->donefunc = storeDiskdDirDone;
+ diskd_state_pool = memPoolCreate("DISKD IO State data", sizeof(diskdstate_t));
+ memset(&diskd_stats, '\0', sizeof(diskd_stats));
+ cachemgrRegister("diskd", "DISKD Stats", storeDiskdStats, 0, 1);
+ debug(81, 1) ("diskd started\n");
+ diskd_initialised = 1;
+}
+
--- /dev/null
+/*
+ * store_diskd.h
+ *
+ * Internal declarations for the diskd routines
+ */
+
+#ifndef __STORE_DISKD_H__
+#define __STORE_DISKD_H__
+
+/*
+ * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv.
+ * If a queue has MAGIC2 (or more) messages away, then we read the
+ * queue until the level falls below MAGIC2. Recommended value
+ * is 75% of SHMBUFS. MAGIC1 is the number of messages away which we
+ * stop allowing open/create for.
+ */
+
+struct _diskdinfo_t {
+ int swaplog_fd;
+ int l1;
+ int l2;
+ fileMap *map;
+ int suggest;
+ int smsgid;
+ int rmsgid;
+ int wfd;
+ int away;
+ struct {
+ char *buf;
+ link_list *stack;
+ int id;
+ } shm;
+ int magic1;
+ int magic2;
+};
+
+struct _diskdstate_t {
+ int id;
+ struct {
+ unsigned int close_request:1;
+ unsigned int reading:1;
+ unsigned int writing:1;
+ } flags;
+ char *read_buf;
+};
+
+enum {
+ _MQD_NOP,
+ _MQD_OPEN,
+ _MQD_CLOSE,
+ _MQD_READ,
+ _MQD_WRITE,
+ _MQD_UNLINK
+};
+
+typedef struct _diomsg {
+ mtyp_t mtype;
+ int id;
+ int seq_no;
+ void *callback_data;
+ int size;
+ int offset;
+ int status;
+ int shm_offset;
+} diomsg;
+
+struct _diskd_stats {
+ int open_fail_queue_len;
+ int block_queue_len;
+ int max_away;
+ int max_shmuse;
+ int shmbuf_count;
+ int sent_count;
+ int recv_count;
+ int sio_id;
+};
+
+typedef struct _diskd_stats diskd_stats_t;
+typedef struct _diskdinfo_t diskdinfo_t;
+typedef struct _diskdstate_t diskdstate_t;
+
+static const int msg_snd_rcv_sz = sizeof(diomsg) - sizeof(mtyp_t);
+
+/* The diskd_state memory pool */
+extern MemPool * diskd_state_pool;
+
+extern void storeDiskdDirMapBitReset(SwapDir *, sfileno);
+extern int storeDiskdDirMapBitAllocate(SwapDir *);
+extern char * storeDiskdDirFullPath(SwapDir *SD, sfileno filn, char *fullpath);
+extern void storeDiskdDirUnlinkFile(SwapDir *, sfileno);
+extern void storeDiskdDirReplAdd(SwapDir *, StoreEntry *);
+extern void storeDiskdDirReplRemove(StoreEntry *);
+extern void storeDiskdShmPut(SwapDir *, int);
+extern void *storeDiskdShmGet(SwapDir *, int *);
+extern void storeDiskdHandle(diomsg *M);
+
+
+/*
+ * Store IO stuff
+ */
+extern STOBJCREATE storeDiskdCreate;
+extern STOBJOPEN storeDiskdOpen;
+extern STOBJCLOSE storeDiskdClose;
+extern STOBJREAD storeDiskdRead;
+extern STOBJWRITE storeDiskdWrite;
+extern STOBJUNLINK storeDiskdUnlink;
+
+/*
+ * SHMBUFS is the number of shared memory buffers to allocate for
+ * Each SwapDir.
+ */
+#define SHMBUFS 96
+#define SHMBUF_BLKSZ SM_PAGE_SIZE
+/*
+ * MAGIC2 is the point at which we start blocking on msgsnd/msgrcv.
+ * If a queue has MAGIC2 (or more) messages away, then we read the
+ * queue until the level falls below MAGIC2. Recommended value
+ * is 75% of SHMBUFS.
+ */
+#define MAGIC1 Config.diskd.magic1
+#define MAGIC2 Config.diskd.magic2
+
+
+extern diskd_stats_t diskd_stats;
+
+#endif
--- /dev/null
+
+#include "config.h"
+#include "squid.h"
+#include "store_diskd.h"
+
+#include <sys/ipc.h>
+#include <sys/msg.h>
+#include <sys/shm.h>
+
+#undef assert
+#include <assert.h>
+
+
+/*
+ * DEBUG: section 81 Diskd Interface functions
+ */
+
+
+static int storeDiskdSend(int, SwapDir *, int, storeIOState *, int, int, int);
+static void storeDiskdIOCallback(storeIOState * sio, int errflag);
+static void storeDiskdIOFreeEntry(void *sio, int foo);
+
+/*
+ * SHMBUFS is the number of shared memory buffers to allocate for
+ * Each SwapDir.
+ */
+#define SHMBUFS 96
+#define SHMBUF_BLKSZ SM_PAGE_SIZE
+
+
+/* === PUBLIC =========================================================== */
+
+storeIOState *
+storeDiskdOpen(SwapDir *SD, StoreEntry *e, STFNCB *file_callback,
+ STIOCB *callback, void *callback_data)
+{
+ sfileno f = e->swap_filen;
+ int x;
+ storeIOState *sio;
+ char *buf;
+ int shm_offset;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata;
+ debug(81, 3) ("storeDiskdOpen: fileno %08X\n", f);
+ /*
+ * XXX Eventually there should be an option here to fail on open()
+ * If there are too many requests queued.
+ */
+ if (diskdinfo->away > diskdinfo->magic1) {
+ debug(81, 3) ("storeDiskdOpen: FAILING, too many requests away\n");
+ diskd_stats.open_fail_queue_len++;
+ return NULL;
+ }
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(diskd_state_pool);
+
+ sio->swap_filen = f;
+ sio->swap_dirn = SD->index;
+ sio->mode = O_RDONLY;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ sio->e = e;
+ cbdataLock(callback_data);
+
+ ((diskdstate_t *)(sio->fsstate))->flags.writing = 0;
+ ((diskdstate_t *)(sio->fsstate))->flags.reading = 0;
+ ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0;
+ ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++;
+
+ buf = storeDiskdShmGet(SD, &shm_offset);
+ /* XXX WRONG!!! :) */
+ strcpy(buf, storeDiskdDirFullPath(SD, f, NULL));
+ x = storeDiskdSend(_MQD_OPEN,
+ SD,
+ ((diskdstate_t *)(sio->fsstate))->id,
+ sio,
+ strlen(buf) + 1,
+ O_RDONLY,
+ shm_offset);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror());
+ storeDiskdShmPut(SD, shm_offset);
+ cbdataUnlock(sio->callback_data);
+ cbdataFree(sio);
+ return NULL;
+ }
+ return sio;
+}
+
+storeIOState *
+storeDiskdCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback,
+ STIOCB *callback, void *callback_data)
+{
+ sfileno f;
+ int x;
+ storeIOState *sio;
+ char *buf;
+ int shm_offset;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *) SD->fsdata;
+ /*
+ * XXX Eventually there should be an option here to fail on open()
+ * If there are too many requests queued.
+ */
+ if (diskdinfo->away > diskdinfo->magic1) {
+ diskd_stats.open_fail_queue_len++;
+ return NULL;
+ }
+
+ /* Allocate a number */
+ f = storeDiskdDirMapBitAllocate(SD);
+ debug(81, 3) ("storeDiskdCreate: fileno %08X\n", f);
+
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeDiskdIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(diskd_state_pool);
+
+ sio->swap_filen = f;
+ sio->swap_dirn = SD->index;
+ sio->mode = O_WRONLY | O_CREAT | O_TRUNC;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ sio->e = e;
+ cbdataLock(callback_data);
+
+ ((diskdstate_t *)(sio->fsstate))->flags.writing = 0;
+ ((diskdstate_t *)(sio->fsstate))->flags.reading = 0;
+ ((diskdstate_t *)(sio->fsstate))->flags.close_request = 0;
+ ((diskdstate_t *)(sio->fsstate))->id = diskd_stats.sio_id++;
+
+ buf = storeDiskdShmGet(SD, &shm_offset);
+ /* XXX WRONG!!! :) */
+ strcpy(buf, storeDiskdDirFullPath(SD, f, NULL));
+ x = storeDiskdSend(_MQD_OPEN,
+ SD,
+ ((diskdstate_t *)(sio->fsstate))->id,
+ sio,
+ strlen(buf) + 1,
+ sio->mode,
+ shm_offset);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend OPEN: %s\n", xstrerror());
+ storeDiskdShmPut(SD, shm_offset);
+ cbdataUnlock(sio->callback_data);
+ cbdataFree(sio);
+ return NULL;
+ }
+ storeDiskdDirReplAdd(SD, e);
+ return sio;
+}
+
+
+void
+storeDiskdClose(SwapDir *SD, storeIOState * sio)
+{
+ int x;
+ diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate;
+ debug(81, 3) ("storeDiskdClose: dirno %d, fileno %08X\n", SD->index,
+ sio->swap_filen);
+ x = storeDiskdSend(_MQD_CLOSE,
+ SD,
+ diskdstate->id,
+ sio,
+ 0,
+ 0,
+ -1);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend CLOSE: %s\n", xstrerror());
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ }
+}
+
+void
+storeDiskdRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+{
+ int x;
+ int shm_offset;
+ char *rbuf;
+ diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate;
+ if (!cbdataValid(sio))
+ return;
+ if (diskdstate->flags.reading) {
+ debug(81, 1) ("storeDiskdRead: already reading!\n");
+ return;
+ }
+ assert(sio->read.callback == NULL);
+ assert(sio->read.callback_data == NULL);
+ sio->read.callback = callback;
+ sio->read.callback_data = callback_data;
+ diskdstate->read_buf = buf; /* the one passed from above */
+ cbdataLock(sio->read.callback_data);
+ debug(81, 3) ("storeDiskdRead: dirno %d, fileno %08X\n", sio->swap_dirn, sio->swap_filen);
+ sio->offset = offset;
+ diskdstate->flags.reading = 1;
+ rbuf = storeDiskdShmGet(SD, &shm_offset);
+ assert(rbuf);
+ x = storeDiskdSend(_MQD_READ,
+ SD,
+ diskdstate->id,
+ sio,
+ (int) size,
+ (int) offset,
+ shm_offset);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend READ: %s\n", xstrerror());
+ storeDiskdShmPut(SD, shm_offset);
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ }
+}
+
+void
+storeDiskdWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+{
+ int x;
+ char *sbuf;
+ int shm_offset;
+ diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate;
+ debug(81, 3) ("storeDiskdWrite: dirno %d, fileno %08X\n", SD->index, sio->swap_filen);
+ if (!cbdataValid(sio)) {
+ free_func(buf);
+ return;
+ }
+ diskdstate->flags.writing = 1;
+ sbuf = storeDiskdShmGet(SD, &shm_offset);
+ xmemcpy(sbuf, buf, size);
+ if (free_func)
+ free_func(buf);
+ x = storeDiskdSend(_MQD_WRITE,
+ SD,
+ diskdstate->id,
+ sio,
+ (int) size,
+ (int) offset,
+ shm_offset);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend WRITE: %s\n", xstrerror());
+ storeDiskdShmPut(SD, shm_offset);
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ }
+}
+
+void
+storeDiskdUnlink(SwapDir *SD, StoreEntry *e)
+{
+ int x;
+ int shm_offset;
+ char *buf;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)SD->fsdata;
+
+ debug(81, 3) ("storeDiskdUnlink: dirno %d, fileno %08X\n", SD->index,
+ e->swap_filen);
+ storeDiskdDirReplRemove(e);
+ storeDiskdDirMapBitReset(SD, e->swap_filen);
+ if (diskdinfo->away >= diskdinfo->magic1) {
+ /* Damn, we need to issue a sync unlink here :( */
+ debug(50, 2) ("storeDiskUnlink: Out of queue space, sync unlink\n");
+ storeDiskdDirUnlinkFile(SD, e->swap_filen);
+ return;
+ }
+
+ /* We can attempt a diskd unlink */
+ buf = storeDiskdShmGet(SD, &shm_offset);
+ strcpy(buf, storeDiskdDirFullPath(SD, e->swap_filen, NULL));
+ x = storeDiskdSend(_MQD_UNLINK,
+ SD,
+ e->swap_filen,
+ NULL,
+ 0,
+ 0,
+ shm_offset);
+ if (x < 0) {
+ debug(50, 1) ("storeDiskdSend UNLINK: %s\n", xstrerror());
+ unlink(buf); /* XXX EWW! */
+ storeDiskdShmPut(SD, shm_offset);
+ }
+}
+
+
+/* === STATIC =========================================================== */
+
+static void
+storeDiskdOpenDone(diomsg * M)
+{
+ storeIOState *sio = M->callback_data;
+ Counter.syscalls.disk.opens++;
+ debug(81, 3) ("storeDiskdOpenDone: dirno %d, fileno %08x status %d\n",
+ sio->swap_dirn, sio->swap_filen, M->status);
+ if (M->status < 0) {
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ }
+}
+
+static void
+storeDiskdCloseDone(diomsg * M)
+{
+ storeIOState *sio = M->callback_data;
+ Counter.syscalls.disk.closes++;
+ debug(81, 3) ("storeDiskdCloseDone: dirno %d, fileno %08x status %d\n",
+ sio->swap_dirn, sio->swap_filen, M->status);
+ if (M->status < 0) {
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ return;
+ }
+ storeDiskdIOCallback(sio, DISK_OK);
+}
+
+static void
+storeDiskdReadDone(diomsg * M)
+{
+ storeIOState *sio = M->callback_data;
+ STRCB *callback = sio->read.callback;
+ SwapDir *sd = INDEXSD(sio->swap_dirn);
+ diskdstate_t *diskdstate = sio->fsstate;
+ diskdinfo_t *diskdinfo = sd->fsdata;
+ void *their_data = sio->read.callback_data;
+ char *their_buf = diskdstate->read_buf;
+ char *sbuf;
+ size_t len;
+ int valid;
+ Counter.syscalls.disk.reads++;
+ diskdstate->flags.reading = 0;
+ valid = cbdataValid(sio->read.callback_data);
+ cbdataUnlock(sio->read.callback_data);
+ debug(81, 3) ("storeDiskdReadDone: dirno %d, fileno %08x status %d\n",
+ sio->swap_dirn, sio->swap_filen, M->status);
+ if (M->status < 0) {
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ return;
+ }
+ sbuf = diskdinfo->shm.buf + M->shm_offset;
+ len = M->status;
+ xmemcpy(their_buf, sbuf, len); /* yucky copy */
+ sio->offset += len;
+ assert(callback);
+ assert(their_data);
+ sio->read.callback = NULL;
+ sio->read.callback_data = NULL;
+ if (valid)
+ callback(their_data, their_buf, len);
+}
+
+static void
+storeDiskdWriteDone(diomsg * M)
+{
+ storeIOState *sio = M->callback_data;
+ diskdstate_t *diskdstate = (diskdstate_t *)sio->fsstate;
+ Counter.syscalls.disk.writes++;
+ diskdstate->flags.writing = 0;
+ debug(81, 3) ("storeDiskdWriteDone: dirno %d, fileno %08x status %d\n",
+ sio->swap_dirn, sio->swap_filen, M->status);
+ if (M->status < 0) {
+ storeDiskdIOCallback(sio, DISK_ERROR);
+ return;
+ }
+ sio->offset += M->status;
+}
+
+static void
+storeDiskdUnlinkDone(diomsg * M)
+{
+ debug(81, 3) ("storeDiskdUnlinkDone: fileno %08x status %d\n",
+ M->id, M->status);
+ Counter.syscalls.disk.unlinks++;
+}
+
+void
+storeDiskdHandle(diomsg * M)
+{
+ int valid = M->callback_data ? cbdataValid(M->callback_data) : 1;
+ if (M->callback_data)
+ cbdataUnlock(M->callback_data);
+ if (!valid) {
+ debug(81, 3) ("storeDiskdHandle: Invalid callback_data %p\n",
+ M->callback_data);
+ /*
+ * The read operation has its own callback. If we don't
+ * call storeDiskdReadDone(), then we must make sure the
+ * callback_data gets unlocked!
+ */
+ if (_MQD_READ == M->mtype) {
+ storeIOState *sio = M->callback_data;
+ cbdataUnlock(sio->read.callback_data);
+ }
+ return;
+ }
+ switch (M->mtype) {
+ case _MQD_OPEN:
+ storeDiskdOpenDone(M);
+ break;
+ case _MQD_CLOSE:
+ storeDiskdCloseDone(M);
+ break;
+ case _MQD_READ:
+ storeDiskdReadDone(M);
+ break;
+ case _MQD_WRITE:
+ storeDiskdWriteDone(M);
+ break;
+ case _MQD_UNLINK:
+ storeDiskdUnlinkDone(M);
+ break;
+ default:
+ assert(0);
+ break;
+ }
+}
+
+static void
+storeDiskdIOCallback(storeIOState * sio, int errflag)
+{
+ int valid = cbdataValid(sio->callback_data);
+ debug(81, 3) ("storeUfsIOCallback: errflag=%d\n", errflag);
+ cbdataUnlock(sio->callback_data);
+ if (valid)
+ sio->callback(sio->callback_data, errflag, sio);
+ cbdataFree(sio);
+}
+
+static int
+storeDiskdSend(int mtype, SwapDir * sd, int id, storeIOState * sio, int size, int offset, int shm_offset)
+{
+ int x;
+ diomsg M;
+ static int send_errors = 0;
+ static int last_seq_no = 0;
+ static int seq_no = 0;
+ diskdinfo_t *diskdinfo = (diskdinfo_t *)sd->fsdata;
+ M.mtype = mtype;
+ M.callback_data = sio;
+ M.size = size;
+ M.offset = offset;
+ M.status = -1;
+ M.shm_offset = shm_offset;
+ M.id = id;
+ M.seq_no = ++seq_no;
+ if (M.callback_data)
+ cbdataLock(M.callback_data);
+ if (M.seq_no < last_seq_no)
+ debug(81, 1) ("WARNING: sequencing out of order\n");
+ x = msgsnd(diskdinfo->smsgid, &M, msg_snd_rcv_sz, IPC_NOWAIT);
+ last_seq_no = M.seq_no;
+ if (0 == x) {
+ diskd_stats.sent_count++;
+ diskdinfo->away++;
+ } else {
+ debug(50, 1) ("storeDiskdSend: msgsnd: %s\n", xstrerror());
+ if (M.callback_data)
+ cbdataUnlock(M.callback_data);
+ assert(++send_errors < 100);
+ }
+ return x;
+}
+
+
+/*
+ * We can't pass memFree() as a free function here, because we need to free
+ * the fsstate variable ..
+ */
+static void
+storeDiskdIOFreeEntry(void *sio, int foo)
+{
+ memPoolFree(diskd_state_pool, ((storeIOState *)sio)->fsstate);
+ memFree(sio, MEM_STORE_IO);
+}
+
--- /dev/null
+#
+# Makefile for the UFS storage driver for the Squid Object Cache server
+#
+# $Id: Makefile.in,v 1.1 2000/05/03 17:15:47 adrian Exp $
+#
+
+FS = ufs
+
+top_srcdir = @top_srcdir@
+VPATH = @srcdir@
+
+CC = @CC@
+MAKEDEPEND = @MAKEDEPEND@
+AR_R = @AR_R@
+RANLIB = @RANLIB@
+AC_CFLAGS = @CFLAGS@
+SHELL = /bin/sh
+
+INCLUDE = -I../../../include -I$(top_srcdir)/include -I$(top_srcdir)/src/
+CFLAGS = $(AC_CFLAGS) $(INCLUDE) $(DEFINES)
+
+OUT = ../$(FS).a
+
+OBJS = \
+ store_dir_ufs.o \
+ store_io_ufs.o
+
+
+all: $(OUT)
+
+$(OUT): $(OBJS)
+ @rm -f ../stamp
+ $(AR_R) $(OUT) $(OBJS)
+ $(RANLIB) $(OUT)
+
+$(OBJS): $(top_srcdir)/include/version.h ../../../include/autoconf.h
+
+.c.o:
+ @rm -f ../stamp
+ $(CC) $(CFLAGS) -c $<
+
+clean:
+ -rm -rf *.o *pure_* core ../$(FS).a
+
+distclean: clean
+ -rm -f Makefile
+ -rm -f Makefile.bak
+ -rm -f tags
+
+install:
+
+tags:
+ ctags *.[ch] $(top_srcdir)/src/*.[ch] $(top_srcdir)/include/*.h $(top_srcdir)/lib/*.[ch]
+
+depend:
+ $(MAKEDEPEND) $(INCLUDE) -fMakefile *.c
--- /dev/null
+
+/*
+ * $Id: store_dir_ufs.cc,v 1.1 2000/05/03 17:15:47 adrian Exp $
+ *
+ * DEBUG: section 47 Store Directory Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#if HAVE_STATVFS
+#if HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+#endif
+
+#include "store_ufs.h"
+
+#define DefaultLevelOneDirs 16
+#define DefaultLevelTwoDirs 256
+#define STORE_META_BUFSZ 4096
+
+typedef struct _RebuildState RebuildState;
+struct _RebuildState {
+ SwapDir *sd;
+ int n_read;
+ FILE *log;
+ int speed;
+ int curlvl1;
+ int curlvl2;
+ struct {
+ unsigned int need_to_validate:1;
+ unsigned int clean:1;
+ unsigned int init:1;
+ } flags;
+ int done;
+ int in_dir;
+ int fn;
+ struct dirent *entry;
+ DIR *td;
+ char fullpath[SQUID_MAXPATHLEN];
+ char fullfilename[SQUID_MAXPATHLEN];
+ struct _store_rebuild_data counts;
+};
+
+static int n_ufs_dirs = 0;
+static int *ufs_dir_index = NULL;
+MemPool * ufs_state_pool = NULL;
+static int ufs_initialised = 0;
+
+static char *storeUfsDirSwapSubDir(SwapDir *, int subdirn);
+static int storeUfsDirCreateDirectory(const char *path, int);
+static int storeUfsDirVerifyCacheDirs(SwapDir *);
+static int storeUfsDirVerifyDirectory(const char *path);
+static void storeUfsDirCreateSwapSubDirs(SwapDir *);
+static char *storeUfsDirSwapLogFile(SwapDir *, const char *);
+static EVH storeUfsDirRebuildFromDirectory;
+static EVH storeUfsDirRebuildFromSwapLog;
+static int storeUfsDirGetNextFile(RebuildState *, int *sfileno, int *size);
+static StoreEntry *storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean);
+static void storeUfsDirRebuild(SwapDir * sd);
+static void storeUfsDirCloseTmpSwapLog(SwapDir * sd);
+static FILE *storeUfsDirOpenTmpSwapLog(SwapDir *, int *, int *);
+static STLOGOPEN storeUfsDirOpenSwapLog;
+static STINIT storeUfsDirInit;
+static STFREE storeUfsDirFree;
+static STLOGCLEANOPEN storeUfsDirWriteCleanOpen;
+static void storeUfsDirWriteCleanClose(SwapDir * sd);
+static STLOGCLEANWRITE storeUfsDirWriteCleanEntry;
+static STLOGCLOSE storeUfsDirCloseSwapLog;
+static STLOGWRITE storeUfsDirSwapLog;
+static STNEWFS storeUfsDirNewfs;
+static STDUMP storeUfsDirDump;
+static STMAINTAINFS storeUfsDirMaintain;
+static STCHECKOBJ storeUfsDirCheckObj;
+static STREFOBJ storeUfsDirRefObj;
+static STUNREFOBJ storeUfsDirUnrefObj;
+static QS rev_int_sort;
+static int storeUfsDirClean(int swap_index);
+static EVH storeUfsDirCleanEvent;
+static int storeUfsDirIs(SwapDir * sd);
+static int storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2);
+static int storeUfsCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static void storeUfsDirStats(SwapDir *, StoreEntry *);
+static void storeUfsDirInitBitmap(SwapDir *);
+static int storeUfsDirValidFileno(SwapDir *, sfileno);
+static int storeUfsDirCheckExpired(SwapDir *, StoreEntry *);
+#if !HEAP_REPLACEMENT
+static time_t storeUfsDirExpiredReferenceAge(SwapDir *);
+#endif
+
+/*
+ * These functions were ripped straight out of the heart of store_dir.c.
+ * They assume that the given filenum is on a ufs partiton, which may or
+ * may not be true..
+ * XXX this evilness should be tidied up at a later date!
+ */
+
+int
+storeUfsDirMapBitTest(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ ufsinfo_t *ufsinfo;
+ ufsinfo = (ufsinfo_t *)SD->fsdata;
+ return file_map_bit_test(ufsinfo->map, filn);
+}
+
+void
+storeUfsDirMapBitSet(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ ufsinfo_t *ufsinfo;
+ ufsinfo = (ufsinfo_t *)SD->fsdata;
+ file_map_bit_set(ufsinfo->map, filn);
+}
+
+void
+storeUfsDirMapBitReset(SwapDir *SD, int fn)
+{
+ sfileno filn = fn;
+ ufsinfo_t *ufsinfo;
+ ufsinfo = (ufsinfo_t *)SD->fsdata;
+ file_map_bit_reset(ufsinfo->map, filn);
+}
+
+int
+storeUfsDirMapBitAllocate(SwapDir *SD)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata;
+ int fn;
+ fn = file_map_allocate(ufsinfo->map, ufsinfo->suggest);
+ file_map_bit_set(ufsinfo->map, fn);
+ ufsinfo->suggest = fn + 1;
+ return fn;
+}
+
+/*
+ * Initialise the ufs bitmap
+ *
+ * If there already is a bitmap, and the numobjects is larger than currently
+ * configured, we allocate a new bitmap and 'grow' the old one into it.
+ */
+static void
+storeUfsDirInitBitmap(SwapDir *sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+
+ if (ufsinfo->map == NULL) {
+ /* First time */
+ ufsinfo->map = file_map_create();
+ } else if (ufsinfo->map->max_n_files) {
+ /* it grew, need to expand */
+ /* XXX We don't need it anymore .. */
+ }
+ /* else it shrunk, and we leave the old one in place */
+}
+
+static char *
+storeUfsDirSwapSubDir(SwapDir * sd, int subdirn)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ assert(0 <= subdirn && subdirn < ufsinfo->l1);
+ snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", sd->path, subdirn);
+ return fullfilename;
+}
+
+static int
+storeUfsDirCreateDirectory(const char *path, int should_exist)
+{
+ int created = 0;
+ struct stat st;
+ getCurrentTime();
+ if (0 == stat(path, &st)) {
+ if (S_ISDIR(st.st_mode)) {
+ debug(20, should_exist ? 3 : 1) ("%s exists\n", path);
+ } else {
+ fatalf("Swap directory %s is not a directory.", path);
+ }
+ } else if (0 == mkdir(path, 0755)) {
+ debug(20, should_exist ? 1 : 3) ("%s created\n", path);
+ created = 1;
+ } else {
+ fatalf("Failed to make swap directory %s: %s",
+ path, xstrerror());
+ }
+ return created;
+}
+
+static int
+storeUfsDirVerifyDirectory(const char *path)
+{
+ struct stat sb;
+ if (stat(path, &sb) < 0) {
+ debug(20, 0) ("%s: %s\n", path, xstrerror());
+ return -1;
+ }
+ if (S_ISDIR(sb.st_mode) == 0) {
+ debug(20, 0) ("%s is not a directory\n", path);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * This function is called by storeUfsDirInit(). If this returns < 0,
+ * then Squid exits, complains about swap directories not
+ * existing, and instructs the admin to run 'squid -z'
+ */
+static int
+storeUfsDirVerifyCacheDirs(SwapDir * sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ int j;
+ const char *path = sd->path;
+
+ if (storeUfsDirVerifyDirectory(path) < 0)
+ return -1;
+ for (j = 0; j < ufsinfo->l1; j++) {
+ path = storeUfsDirSwapSubDir(sd, j);
+ if (storeUfsDirVerifyDirectory(path) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void
+storeUfsDirCreateSwapSubDirs(SwapDir * sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ int i, k;
+ int should_exist;
+ LOCAL_ARRAY(char, name, MAXPATHLEN);
+ for (i = 0; i < ufsinfo->l1; i++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X", sd->path, i);
+ if (storeUfsDirCreateDirectory(name, 0))
+ should_exist = 0;
+ else
+ should_exist = 1;
+ debug(47, 1) ("Making directories in %s\n", name);
+ for (k = 0; k < ufsinfo->l2; k++) {
+ snprintf(name, MAXPATHLEN, "%s/%02X/%02X", sd->path, i, k);
+ storeUfsDirCreateDirectory(name, should_exist);
+ }
+ }
+}
+
+static char *
+storeUfsDirSwapLogFile(SwapDir * sd, const char *ext)
+{
+ LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN);
+ LOCAL_ARRAY(char, digit, 32);
+ char *pathtmp2;
+ if (Config.Log.swap) {
+ xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64);
+ while (index(pathtmp,'/'))
+ *index(pathtmp,'/')='.';
+ while (strlen(pathtmp) && pathtmp[strlen(pathtmp)-1]=='.')
+ pathtmp[strlen(pathtmp)-1]= '\0';
+ for(pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++);
+ snprintf(path, SQUID_MAXPATHLEN-64, Config.Log.swap, pathtmp2);
+ if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) {
+ strcat(path, ".");
+ snprintf(digit, 32, "%02d", sd->index);
+ strncat(path, digit, 3);
+ }
+ } else {
+ xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64);
+ strcat(path, "/swap.state");
+ }
+ if (ext)
+ strncat(path, ext, 16);
+ return path;
+}
+
+static void
+storeUfsDirOpenSwapLog(SwapDir * sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ char *path;
+ int fd;
+ path = storeUfsDirSwapLogFile(sd, NULL);
+ fd = file_open(path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", path, xstrerror());
+ fatal("storeUfsDirOpenSwapLog: Failed to open swap log.");
+ }
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+ ufsinfo->swaplog_fd = fd;
+ if (0 == n_ufs_dirs)
+ assert(NULL == ufs_dir_index);
+ n_ufs_dirs++;
+ assert(n_ufs_dirs <= Config.cacheSwap.n_configured);
+}
+
+static void
+storeUfsDirCloseSwapLog(SwapDir * sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ if (ufsinfo->swaplog_fd < 0) /* not open */
+ return;
+ file_close(ufsinfo->swaplog_fd);
+ debug(47, 3) ("Cache Dir #%d log closed on FD %d\n",
+ sd->index, ufsinfo->swaplog_fd);
+ ufsinfo->swaplog_fd = -1;
+ n_ufs_dirs--;
+ assert(n_ufs_dirs >= 0);
+ if (0 == n_ufs_dirs)
+ safe_free(ufs_dir_index);
+}
+
+static void
+storeUfsDirInit(SwapDir * sd)
+{
+ static int started_clean_event = 0;
+ static const char *errmsg =
+ "\tFailed to verify one of the swap directories, Check cache.log\n"
+ "\tfor details. Run 'squid -z' to create swap directories\n"
+ "\tif needed, or if running Squid for the first time.";
+ storeUfsDirInitBitmap(sd);
+ if (storeUfsDirVerifyCacheDirs(sd) < 0)
+ fatal(errmsg);
+ storeUfsDirOpenSwapLog(sd);
+ storeUfsDirRebuild(sd);
+ if (!started_clean_event) {
+ eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL, 15.0, 1);
+ started_clean_event = 1;
+ }
+}
+
+static void
+storeUfsDirRebuildFromDirectory(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
+ StoreEntry *e = NULL;
+ StoreEntry tmpe;
+ cache_key key[MD5_DIGEST_CHARS];
+ int sfileno = 0;
+ int count;
+ int size;
+ struct stat sb;
+ int swap_hdr_len;
+ int fd = -1;
+ tlv *tlv_list;
+ tlv *t;
+ assert(rb != NULL);
+ debug(20, 3) ("storeUfsDirRebuildFromDirectory: DIR #%d\n", rb->sd->index);
+ for (count = 0; count < rb->speed; count++) {
+ assert(fd == -1);
+ fd = storeUfsDirGetNextFile(rb, &sfileno, &size);
+ if (fd == -2) {
+ debug(20, 1) ("Done scanning %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ store_dirs_rebuilding--;
+ storeUfsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ } else if (fd < 0) {
+ continue;
+ }
+ assert(fd > -1);
+ /* lets get file stats here */
+ if (fstat(fd, &sb) < 0) {
+ debug(20, 1) ("storeUfsDirRebuildFromDirectory: fstat(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %s %7d files opened so far.\n",
+ rb->sd->path, rb->counts.scancount);
+ debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
+ Counter.syscalls.disk.reads++;
+ if (read(fd, hdr_buf, SM_PAGE_SIZE) < 0) {
+ debug(20, 1) ("storeUfsDirRebuildFromDirectory: read(FD %d): %s\n",
+ fd, xstrerror());
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ continue;
+ }
+ file_close(fd);
+ store_open_disk_fd--;
+ fd = -1;
+ swap_hdr_len = 0;
+#if USE_TRUNCATE
+ if (sb.st_size == 0)
+ continue;
+#endif
+ tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
+ if (tlv_list == NULL) {
+ debug(20, 1) ("storeUfsDirRebuildFromDirectory: failed to get meta data\n");
+ /* XXX shouldn't this be a call to storeUfsUnlink ? */
+ storeUfsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ debug(20, 3) ("storeUfsDirRebuildFromDirectory: successful swap meta unpacking\n");
+ memset(key, '\0', MD5_DIGEST_CHARS);
+ memset(&tmpe, '\0', sizeof(StoreEntry));
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_KEY:
+ assert(t->length == MD5_DIGEST_CHARS);
+ xmemcpy(key, t->value, MD5_DIGEST_CHARS);
+ break;
+ case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE);
+ xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
+ break;
+ default:
+ break;
+ }
+ }
+ storeSwapTLVFree(tlv_list);
+ tlv_list = NULL;
+ if (storeKeyNull(key)) {
+ debug(20, 1) ("storeUfsDirRebuildFromDirectory: NULL key\n");
+ storeUfsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ tmpe.key = key;
+ /* check sizes */
+ if (tmpe.swap_file_sz == 0) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
+ tmpe.swap_file_sz = sb.st_size;
+ } else if (tmpe.swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeUfsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
+ tmpe.swap_file_sz, (int) sb.st_size);
+ storeUfsDirUnlinkFile(SD, sfileno);
+ continue;
+ }
+ if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
+ storeUfsDirUnlinkFile(SD, sfileno);
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(key);
+ if (e && e->lastref >= tmpe.lastref) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (NULL != e) {
+ /* URL already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeRelease(e); /* release old entry */
+ rb->counts.dupcount++;
+ }
+ rb->counts.objcount++;
+ storeEntryDump(&tmpe, 5);
+ e = storeUfsDirAddDiskRestore(SD, key,
+ sfileno,
+ tmpe.swap_file_sz,
+ tmpe.expires,
+ tmpe.timestamp,
+ tmpe.lastref,
+ tmpe.lastmod,
+ tmpe.refcount, /* refcount */
+ tmpe.flags, /* flags */
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeUfsDirRebuildFromDirectory, rb, 0.0, 1);
+}
+
+static void
+storeUfsDirRebuildFromSwapLog(void *data)
+{
+ RebuildState *rb = data;
+ SwapDir *SD = rb->sd;
+ StoreEntry *e = NULL;
+ storeSwapLogData s;
+ size_t ss = sizeof(storeSwapLogData);
+ int count;
+ int used; /* is swapfile already in use? */
+ int disk_entry_newer; /* is the log entry newer than current entry? */
+ double x;
+ assert(rb != NULL);
+ /* load a number of objects per invocation */
+ for (count = 0; count < rb->speed; count++) {
+ if (fread(&s, ss, 1, rb->log) != 1) {
+ debug(20, 1) ("Done reading %s swaplog (%d entries)\n",
+ rb->sd->path, rb->n_read);
+ fclose(rb->log);
+ rb->log = NULL;
+ store_dirs_rebuilding--;
+ storeUfsDirCloseTmpSwapLog(rb->sd);
+ storeRebuildComplete(&rb->counts);
+ cbdataFree(rb);
+ return;
+ }
+ rb->n_read++;
+ if (s.op <= SWAP_LOG_NOP)
+ continue;
+ if (s.op >= SWAP_LOG_MAX)
+ continue;
+ debug(20, 3) ("storeUfsDirRebuildFromSwapLog: %s %s %08X\n",
+ swap_log_op_str[(int) s.op],
+ storeKeyText(s.key),
+ s.swap_filen);
+ if (s.op == SWAP_LOG_ADD) {
+ (void) 0;
+ } else if (s.op == SWAP_LOG_DEL) {
+ if ((e = storeGet(s.key)) != NULL) {
+ /*
+ * Make sure we don't unlink the file, it might be
+ * in use by a subsequent entry. Also note that
+ * we don't have to subtract from store_swap_size
+ * because adding to store_swap_size happens in
+ * the cleanup procedure.
+ */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeUfsDirReplRemove(e);
+ if (e->swap_filen > -1) {
+ storeUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.objcount--;
+ rb->counts.cancelcount++;
+ }
+ continue;
+ } else {
+ x = log(++rb->counts.bad_log_op) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
+ rb->counts.bad_log_op);
+ rb->counts.invalid++;
+ continue;
+ }
+ if ((++rb->counts.scancount & 0xFFFF) == 0)
+ debug(20, 3) (" %7d %s Entries read so far.\n",
+ rb->counts.scancount, rb->sd->path);
+ if (!storeUfsDirValidFileno(SD, s.swap_filen)) {
+ rb->counts.invalid++;
+ continue;
+ }
+ if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
+ rb->counts.badflags++;
+ continue;
+ }
+ e = storeGet(s.key);
+ used = storeUfsDirMapBitTest(SD, s.swap_filen);
+ /* If this URL already exists in the cache, does the swap log
+ * appear to have a newer entry? Compare 'lastref' from the
+ * swap log to e->lastref. */
+ disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
+ if (used && !disk_entry_newer) {
+ /* log entry is old, ignore it */
+ rb->counts.clashcount++;
+ continue;
+ } else if (used && e && e->swap_filen == s.swap_filen && e->swap_dirn == SD->index) {
+ /* swapfile taken, same URL, newer, update meta */
+ if (e->store_status == STORE_OK) {
+ e->lastref = s.timestamp;
+ e->timestamp = s.timestamp;
+ e->expires = s.expires;
+ e->lastmod = s.lastmod;
+ e->flags = s.flags;
+ e->refcount += s.refcount;
+#if HEAP_REPLACEMENT
+ storeHeapPositionUpdate(e, SD);
+ storeUfsDirUnrefObj(SD, e);
+#endif
+ } else {
+ debug_trap("storeUfsDirRebuildFromSwapLog: bad condition");
+ debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
+ }
+ continue;
+ } else if (used) {
+ /* swapfile in use, not by this URL, log entry is newer */
+ /* This is sorta bad: the log entry should NOT be newer at this
+ * point. If the log is dirty, the filesize check should have
+ * caught this. If the log is clean, there should never be a
+ * newer entry. */
+ debug(20, 1) ("WARNING: newer swaplog entry for dirno %d, fileno %08X\n",
+ SD->index, s.swap_filen);
+ /* I'm tempted to remove the swapfile here just to be safe,
+ * but there is a bad race condition in the NOVM version if
+ * the swapfile has recently been opened for writing, but
+ * not yet opened for reading. Because we can't map
+ * swapfiles back to StoreEntrys, we don't know the state
+ * of the entry using that file. */
+ /* We'll assume the existing entry is valid, probably because
+ * were in a slow rebuild and the the swap file number got taken
+ * and the validation procedure hasn't run. */
+ assert(rb->flags.need_to_validate);
+ rb->counts.clashcount++;
+ continue;
+ } else if (e && !disk_entry_newer) {
+ /* key already exists, current entry is newer */
+ /* keep old, ignore new */
+ rb->counts.dupcount++;
+ continue;
+ } else if (e) {
+ /* key already exists, this swapfile not being used */
+ /* junk old, load new */
+ storeExpireNow(e);
+ storeReleaseRequest(e);
+ storeUfsDirReplRemove(e);
+ if (e->swap_filen > -1) {
+ /* Make sure we don't actually unlink the file */
+ storeUfsDirMapBitReset(SD, e->swap_filen);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
+ }
+ storeRelease(e);
+ rb->counts.dupcount++;
+ } else {
+ /* URL doesnt exist, swapfile not in use */
+ /* load new */
+ (void) 0;
+ }
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeUfsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
+ }
+ eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
+}
+
+static int
+storeUfsDirGetNextFile(RebuildState * rb, int *sfileno, int *size)
+{
+ SwapDir *SD = rb->sd;
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata;
+ int fd = -1;
+ int used = 0;
+ int dirs_opened = 0;
+ debug(20, 3) ("storeUfsDirGetNextFile: flag=%d, %d: /%02X/%02X\n",
+ rb->flags.init,
+ rb->sd->index,
+ rb->curlvl1,
+ rb->curlvl2);
+ if (rb->done)
+ return -2;
+ while (fd < 0 && rb->done == 0) {
+ fd = -1;
+ if (0 == rb->flags.init) { /* initialize, open first file */
+ rb->done = 0;
+ rb->curlvl1 = 0;
+ rb->curlvl2 = 0;
+ rb->in_dir = 0;
+ rb->flags.init = 1;
+ assert(Config.cacheSwap.n_configured > 0);
+ }
+ if (0 == rb->in_dir) { /* we need to read in a new directory */
+ snprintf(rb->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ rb->sd->path,
+ rb->curlvl1, rb->curlvl2);
+ if (rb->flags.init && rb->td != NULL)
+ closedir(rb->td);
+ rb->td = NULL;
+ if (dirs_opened)
+ return -1;
+ rb->td = opendir(rb->fullpath);
+ dirs_opened++;
+ if (rb->td == NULL) {
+ debug(50, 1) ("storeUfsDirGetNextFile: opendir: %s: %s\n",
+ rb->fullpath, xstrerror());
+ } else {
+ rb->entry = readdir(rb->td); /* skip . and .. */
+ rb->entry = readdir(rb->td);
+ if (rb->entry == NULL && errno == ENOENT)
+ debug(20, 1) ("storeUfsDirGetNextFile: directory does not exist!.\n");
+ debug(20, 3) ("storeUfsDirGetNextFile: Directory %s\n", rb->fullpath);
+ }
+ }
+ if (rb->td != NULL && (rb->entry = readdir(rb->td)) != NULL) {
+ rb->in_dir++;
+ if (sscanf(rb->entry->d_name, "%x", &rb->fn) != 1) {
+ debug(20, 3) ("storeUfsDirGetNextFile: invalid %s\n",
+ rb->entry->d_name);
+ continue;
+ }
+ if (!storeUfsFilenoBelongsHere(rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2)) {
+ debug(20, 3) ("storeUfsDirGetNextFile: %08X does not belong in %d/%d/%d\n",
+ rb->fn, rb->sd->index, rb->curlvl1, rb->curlvl2);
+ continue;
+ }
+ used = storeUfsDirMapBitTest(SD, rb->fn);
+ if (used) {
+ debug(20, 3) ("storeUfsDirGetNextFile: Locked, continuing with next.\n");
+ continue;
+ }
+ snprintf(rb->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
+ rb->fullpath, rb->entry->d_name);
+ debug(20, 3) ("storeUfsDirGetNextFile: Opening %s\n", rb->fullfilename);
+ fd = file_open(rb->fullfilename, O_RDONLY);
+ if (fd < 0)
+ debug(50, 1) ("storeUfsDirGetNextFile: %s: %s\n", rb->fullfilename, xstrerror());
+ else
+ store_open_disk_fd++;
+ continue;
+ }
+ rb->in_dir = 0;
+ if (++rb->curlvl2 < ufsinfo->l2)
+ continue;
+ rb->curlvl2 = 0;
+ if (++rb->curlvl1 < ufsinfo->l1)
+ continue;
+ rb->curlvl1 = 0;
+ rb->done = 1;
+ }
+ *sfileno = rb->fn;
+ return fd;
+}
+
+/* Add a new object to the cache with empty memory copy and pointer to disk
+ * use to rebuild store from disk. */
+static StoreEntry *
+storeUfsDirAddDiskRestore(SwapDir *SD, const cache_key * key,
+ int file_number,
+ size_t swap_file_sz,
+ time_t expires,
+ time_t timestamp,
+ time_t lastref,
+ time_t lastmod,
+ u_num32 refcount,
+ u_short flags,
+ int clean)
+{
+ StoreEntry *e = NULL;
+ debug(20, 5) ("storeUfsAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
+ /* if you call this you'd better be sure file_number is not
+ * already in use! */
+ e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
+ e->store_status = STORE_OK;
+ storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->swap_status = SWAPOUT_DONE;
+ e->swap_filen = file_number;
+ e->swap_dirn = SD->index;
+ e->swap_file_sz = swap_file_sz;
+ e->lock_count = 0;
+#if !HEAP_REPLACEMENT
+ e->refcount = 0;
+#endif
+ e->lastref = lastref;
+ e->timestamp = timestamp;
+ e->expires = expires;
+ e->lastmod = lastmod;
+ e->refcount = refcount;
+ e->flags = flags;
+ EBIT_SET(e->flags, ENTRY_CACHABLE);
+ EBIT_CLR(e->flags, RELEASE_REQUEST);
+ EBIT_CLR(e->flags, KEY_PRIVATE);
+ e->ping_status = PING_NONE;
+ EBIT_CLR(e->flags, ENTRY_VALIDATED);
+ storeUfsDirMapBitSet(SD, e->swap_filen);
+ storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
+ storeUfsDirReplAdd(SD, e);
+ return e;
+}
+
+static void
+storeUfsDirRebuild(SwapDir * sd)
+{
+ RebuildState *rb = xcalloc(1, sizeof(*rb));
+ int clean = 0;
+ int zero = 0;
+ FILE *fp;
+ EVH *func = NULL;
+ rb->sd = sd;
+ rb->speed = opt_foreground_rebuild ? 1 << 30 : 50;
+ /*
+ * If the swap.state file exists in the cache_dir, then
+ * we'll use storeUfsDirRebuildFromSwapLog(), otherwise we'll
+ * use storeUfsDirRebuildFromDirectory() to open up each file
+ * and suck in the meta data.
+ */
+ fp = storeUfsDirOpenTmpSwapLog(sd, &clean, &zero);
+ if (fp == NULL || zero) {
+ if (fp != NULL)
+ fclose(fp);
+ func = storeUfsDirRebuildFromDirectory;
+ } else {
+ func = storeUfsDirRebuildFromSwapLog;
+ rb->log = fp;
+ rb->flags.clean = (unsigned int) clean;
+ }
+ if (!clean)
+ rb->flags.need_to_validate = 1;
+ debug(20, 1) ("Rebuilding storage in %s (%s)\n",
+ sd->path, clean ? "CLEAN" : "DIRTY");
+ store_dirs_rebuilding++;
+ cbdataAdd(rb, cbdataXfree, 0);
+ eventAdd("storeRebuild", func, rb, 0.0, 1);
+}
+
+static void
+storeUfsDirCloseTmpSwapLog(SwapDir * sd)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
+ char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new"));
+ int fd;
+ file_close(ufsinfo->swaplog_fd);
+#ifdef _SQUID_OS2_
+ if (unlink(swaplog_path) < 0) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeUfsDirCloseTmpSwapLog: unlink failed");
+ }
+#endif
+ if (xrename(new_path, swaplog_path) < 0) {
+ fatal("storeUfsDirCloseTmpSwapLog: rename failed");
+ }
+ fd = file_open(swaplog_path, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("storeUfsDirCloseTmpSwapLog: Failed to open swap log.");
+ }
+ safe_free(swaplog_path);
+ safe_free(new_path);
+ ufsinfo->swaplog_fd = fd;
+ debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", sd->index, fd);
+}
+
+static FILE *
+storeUfsDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ char *swaplog_path = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
+ char *clean_path = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
+ char *new_path = xstrdup(storeUfsDirSwapLogFile(sd, ".new"));
+ struct stat log_sb;
+ struct stat clean_sb;
+ FILE *fp;
+ int fd;
+ if (stat(swaplog_path, &log_sb) < 0) {
+ debug(47, 1) ("Cache Dir #%d: No log file\n", sd->index);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return NULL;
+ }
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
+ /* close the existing write-only FD */
+ if (ufsinfo->swaplog_fd >= 0)
+ file_close(ufsinfo->swaplog_fd);
+ /* open a write-only FD for the new log */
+ fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC);
+ if (fd < 0) {
+ debug(50, 1) ("%s: %s\n", new_path, xstrerror());
+ fatal("storeDirOpenTmpSwapLog: Failed to open swap log.");
+ }
+ ufsinfo->swaplog_fd = fd;
+ /* open a read-only stream of the old log */
+ fp = fopen(swaplog_path, "r");
+ if (fp == NULL) {
+ debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror());
+ fatal("Failed to open swap log for reading");
+ }
+ memset(&clean_sb, '\0', sizeof(struct stat));
+ if (stat(clean_path, &clean_sb) < 0)
+ *clean_flag = 0;
+ else if (clean_sb.st_mtime < log_sb.st_mtime)
+ *clean_flag = 0;
+ else
+ *clean_flag = 1;
+ safeunlink(clean_path, 1);
+ safe_free(swaplog_path);
+ safe_free(clean_path);
+ safe_free(new_path);
+ return fp;
+}
+
+struct _clean_state {
+ char *cur;
+ char *new;
+ char *cln;
+ char *outbuf;
+ off_t outbuf_offset;
+ int fd;
+};
+
+#define CLEAN_BUF_SZ 16384
+/*
+ * Begin the process to write clean cache state. For UFS this means
+ * opening some log files and allocating write buffers. Return 0 if
+ * we succeed, and assign the 'func' and 'data' return pointers.
+ */
+static int
+storeUfsDirWriteCleanOpen(SwapDir * sd)
+{
+ struct _clean_state *state = xcalloc(1, sizeof(*state));
+ struct stat sb;
+ sd->log.clean.write = NULL;
+ sd->log.clean.state = NULL;
+ state->cur = xstrdup(storeUfsDirSwapLogFile(sd, NULL));
+ state->new = xstrdup(storeUfsDirSwapLogFile(sd, ".clean"));
+ state->cln = xstrdup(storeUfsDirSwapLogFile(sd, ".last-clean"));
+ state->outbuf = xcalloc(CLEAN_BUF_SZ, 1);
+ state->outbuf_offset = 0;
+ unlink(state->new);
+ unlink(state->cln);
+ state->fd = file_open(state->new, O_WRONLY | O_CREAT | O_TRUNC);
+ if (state->fd < 0)
+ return -1;
+ debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n",
+ state->new, state->fd);
+#if HAVE_FCHMOD
+ if (stat(state->cur, &sb) == 0)
+ fchmod(state->fd, sb.st_mode);
+#endif
+ sd->log.clean.write = storeUfsDirWriteCleanEntry;
+ sd->log.clean.state = state;
+ return 0;
+}
+
+/*
+ * "write" an entry to the clean log file.
+ */
+static void
+storeUfsDirWriteCleanEntry(const StoreEntry * e, SwapDir * sd)
+{
+ storeSwapLogData s;
+ static size_t ss = sizeof(storeSwapLogData);
+ struct _clean_state *state = sd->log.clean.state;
+ if (NULL == e) {
+ storeUfsDirWriteCleanClose(sd);
+ return;
+ }
+ memset(&s, '\0', ss);
+ s.op = (char) SWAP_LOG_ADD;
+ s.swap_filen = e->swap_filen;
+ s.timestamp = e->timestamp;
+ s.lastref = e->lastref;
+ s.expires = e->expires;
+ s.lastmod = e->lastmod;
+ s.swap_file_sz = e->swap_file_sz;
+ s.refcount = e->refcount;
+ s.flags = e->flags;
+ xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS);
+ xmemcpy(state->outbuf + state->outbuf_offset, &s, ss);
+ state->outbuf_offset += ss;
+ /* buffered write */
+ if (state->outbuf_offset + ss > CLEAN_BUF_SZ) {
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+ }
+ state->outbuf_offset = 0;
+ }
+}
+
+static void
+storeUfsDirWriteCleanClose(SwapDir * sd)
+{
+ struct _clean_state *state = sd->log.clean.state;
+ if (state->fd < 0)
+ return;
+ if (write(state->fd, state->outbuf, state->outbuf_offset) < 0) {
+ debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n",
+ state->new, xstrerror());
+ debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile "
+ "not replaced.\n");
+ file_close(state->fd);
+ state->fd = -1;
+ unlink(state->new);
+ }
+ safe_free(state->outbuf);
+ /*
+ * You can't rename open files on Microsoft "operating systems"
+ * so we have to close before renaming.
+ */
+ storeUfsDirCloseSwapLog(sd);
+ /* rename */
+ if (state->fd >= 0) {
+#ifdef _SQUID_OS2_
+ file_close(state->fd);
+ state->fd = -1;
+ if (unlink(cur) < 0)
+ debug(50, 0) ("storeDirWriteCleanLogs: unlinkd failed: %s, %s\n",
+ xstrerror(), cur);
+#endif
+ xrename(state->new, state->cur);
+ }
+ /* touch a timestamp file if we're not still validating */
+ if (store_dirs_rebuilding)
+ (void) 0;
+ else if (state->fd < 0)
+ (void) 0;
+ else
+ file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC));
+ /* close */
+ safe_free(state->cur);
+ safe_free(state->new);
+ safe_free(state->cln);
+ if (state->fd >= 0)
+ file_close(state->fd);
+ state->fd = -1;
+ safe_free(state);
+ sd->log.clean.state = NULL;
+ sd->log.clean.write = NULL;
+}
+
+static void
+storeUfsDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)sd->fsdata;
+ storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData));
+ s->op = (char) op;
+ s->swap_filen = e->swap_filen;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->swap_file_sz = e->swap_file_sz;
+ s->refcount = e->refcount;
+ s->flags = e->flags;
+ xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
+ file_write(ufsinfo->swaplog_fd,
+ -1,
+ s,
+ sizeof(storeSwapLogData),
+ NULL,
+ NULL,
+ xfree);
+}
+
+static void
+storeUfsDirNewfs(SwapDir * sd)
+{
+ debug(47, 3) ("Creating swap space in %s\n", sd->path);
+ storeUfsDirCreateDirectory(sd->path, 0);
+ storeUfsDirCreateSwapSubDirs(sd);
+}
+
+static int
+rev_int_sort(const void *A, const void *B)
+{
+ const int *i1 = A;
+ const int *i2 = B;
+ return *i2 - *i1;
+}
+
+static int
+storeUfsDirClean(int swap_index)
+{
+ DIR *dp = NULL;
+ struct dirent *de = NULL;
+ LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
+ LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
+#if USE_TRUNCATE
+ struct stat sb;
+#endif
+ int files[20];
+ int swapfileno;
+ int fn; /* same as swapfileno, but with dirn bits set */
+ int n = 0;
+ int k = 0;
+ int N0, N1, N2;
+ int D0, D1, D2;
+ SwapDir *SD;
+ ufsinfo_t *ufsinfo;
+ N0 = n_ufs_dirs;
+ D0 = ufs_dir_index[swap_index % N0];
+ SD = &Config.cacheSwap.swapDirs[D0];
+ ufsinfo = (ufsinfo_t *)SD->fsdata;
+ N1 = ufsinfo->l1;
+ D1 = (swap_index / N0) % N1;
+ N2 = ufsinfo->l2;
+ D2 = ((swap_index / N0) / N1) % N2;
+ snprintf(p1, SQUID_MAXPATHLEN, "%s/%02X/%02X",
+ Config.cacheSwap.swapDirs[D0].path, D1, D2);
+ debug(36, 3) ("storeDirClean: Cleaning directory %s\n", p1);
+ dp = opendir(p1);
+ if (dp == NULL) {
+ if (errno == ENOENT) {
+ debug(36, 0) ("storeDirClean: WARNING: Creating %s\n", p1);
+ if (mkdir(p1, 0777) == 0)
+ return 0;
+ }
+ debug(50, 0) ("storeDirClean: %s: %s\n", p1, xstrerror());
+ safeunlink(p1, 1);
+ return 0;
+ }
+ while ((de = readdir(dp)) != NULL && k < 20) {
+ if (sscanf(de->d_name, "%X", &swapfileno) != 1)
+ continue;
+ fn = swapfileno; /* XXX should remove this cruft ! */
+ if (storeUfsDirValidFileno(SD, fn))
+ if (storeUfsDirMapBitTest(SD, fn))
+ if (storeUfsFilenoBelongsHere(fn, D0, D1, D2))
+ continue;
+#if USE_TRUNCATE
+ if (!stat(de->d_name, &sb))
+ if (sb.st_size == 0)
+ continue;
+#endif
+ files[k++] = swapfileno;
+ }
+ closedir(dp);
+ if (k == 0)
+ return 0;
+ qsort(files, k, sizeof(int), rev_int_sort);
+ if (k > 10)
+ k = 10;
+ for (n = 0; n < k; n++) {
+ debug(36, 3) ("storeDirClean: Cleaning file %08X\n", files[n]);
+ snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
+#if USE_TRUNCATE
+ truncate(p2, 0);
+#else
+ safeunlink(p2, 0);
+#endif
+ Counter.swap_files_cleaned++;
+ }
+ debug(36, 3) ("Cleaned %d unused files from %s\n", k, p1);
+ return k;
+}
+
+static void
+storeUfsDirCleanEvent(void *unused)
+{
+ static int swap_index = 0;
+ int i;
+ int j = 0;
+ int n = 0;
+ /*
+ * Assert that there are UFS cache_dirs configured, otherwise
+ * we should never be called.
+ */
+ assert(n_ufs_dirs);
+ if (NULL == ufs_dir_index) {
+ SwapDir *sd;
+ ufsinfo_t *ufsinfo;
+ /*
+ * Initialize the little array that translates UFS cache_dir
+ * number into the Config.cacheSwap.swapDirs array index.
+ */
+ ufs_dir_index = xcalloc(n_ufs_dirs, sizeof(*ufs_dir_index));
+ for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) {
+ sd = &Config.cacheSwap.swapDirs[i];
+ if (!storeUfsDirIs(sd))
+ continue;
+ ufs_dir_index[n++] = i;
+ ufsinfo = (ufsinfo_t *)sd->fsdata;
+ j += (ufsinfo->l1 * ufsinfo->l2);
+ }
+ assert(n == n_ufs_dirs);
+ /*
+ * Start the storeUfsDirClean() swap_index with a random
+ * value. j equals the total number of UFS level 2
+ * swap directories
+ */
+ swap_index = (int) (squid_random() % j);
+ }
+ if (0 == store_dirs_rebuilding) {
+ n = storeUfsDirClean(swap_index);
+ swap_index++;
+ }
+ eventAdd("storeDirClean", storeUfsDirCleanEvent, NULL,
+ 15.0 * exp(-0.25 * n), 1);
+}
+
+static int
+storeUfsDirIs(SwapDir * sd)
+{
+ if (strncmp(sd->type, "ufs", 3) == 0)
+ return 1;
+ return 0;
+}
+
+/*
+ * Does swapfile number 'fn' belong in cachedir #F0,
+ * level1 dir #F1, level2 dir #F2?
+ */
+static int
+storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2)
+{
+ int D1, D2;
+ int L1, L2;
+ int filn = fn;
+ ufsinfo_t *ufsinfo;
+ assert(F0 < Config.cacheSwap.n_configured);
+ ufsinfo = (ufsinfo_t *)Config.cacheSwap.swapDirs[F0].fsdata;
+ L1 = ufsinfo->l1;
+ L2 = ufsinfo->l2;
+ D1 = ((filn / L2) / L2) % L1;
+ if (F1 != D1)
+ return 0;
+ D2 = (filn / L2) % L2;
+ if (F2 != D2)
+ return 0;
+ return 1;
+}
+
+int
+storeUfsDirValidFileno(SwapDir *SD, sfileno filn)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata;
+ if (filn < 0)
+ return 0;
+ if (filn > ufsinfo->map->max_n_files)
+ return 0;
+ return 1;
+}
+
+void
+storeUfsDirMaintain(SwapDir *SD)
+{
+ StoreEntry *e = NULL;
+ int scanned = 0;
+ int locked = 0;
+ int expired = 0;
+ int max_scan;
+ int max_remove;
+ double f;
+ static time_t last_warn_time = 0;
+#if !HEAP_REPLACEMENT
+ dlink_node *m;
+ dlink_node *prev = NULL;
+#else
+ heap_key age;
+ heap_key min_age = 0.0;
+ link_list *locked_entries = NULL;
+#if HEAP_REPLACEMENT_DEBUG
+ if (!verify_heap_property(SD->repl.heap.heap)) {
+ debug(20, 1) ("Heap property violated!\n");
+ }
+#endif
+#endif
+ /* We can't delete objects while rebuilding swap */
+ if (store_dirs_rebuilding) {
+ return;
+ } else {
+ f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
+ f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
+ max_scan = (int) (f * 400.0 + 100.0);
+ max_remove = (int) (f * 70.0 + 10.0);
+ /*
+ * This is kinda cheap, but so we need this priority hack?
+ */
+#if 0
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
+#endif
+ }
+ debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n", f, max_scan, max_remove);
+#if HEAP_REPLACEMENT
+ while (heap_nodes(SD->repl.heap.heap) > 0) {
+ if (store_swap_size < store_swap_low)
+ break;
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ age = heap_peepminkey(SD->repl.heap.heap);
+ e = heap_extractmin(SD->repl.heap.heap);
+ e->repl.node = NULL; /* no longer in the heap */
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * Entry is in use ... put it in a linked list to ignore it.
+ */
+ if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ /*
+ * If this was a "SPECIAL" do not add it back into the heap.
+ * It will always be "SPECIAL" and therefore never removed.
+ */
+ debug(20, 4) ("storeUfsDirMaintain: locked url %s\n",
+ (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->
+key));
+ linklistPush(&locked_entries, e);
+ }
+ locked++;
+ continue;
+ } else if (storeUfsDirCheckExpired(SD, e)) {
+ /*
+ * Note: This will not check the reference age ifdef
+ * HEAP_REPLACEMENT, but it does some other useful
+ * checks...
+ */
+ expired++;
+ debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
+ age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
+ min_age = age;
+ storeRelease(e);
+ } else {
+ /*
+ * Did not expire the object so we need to add it back
+ * into the heap!
+ */
+ debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
+ storeKeyText(e->key));
+ linklistPush(&locked_entries, e);
+ continue;
+ }
+ if (store_swap_size < store_swap_low)
+ break;
+ else if (expired >= max_remove)
+ break;
+ else if (scanned >= max_scan)
+ break;
+ }
+ /*
+ * Bump the heap age factor.
+ */
+ if (min_age > 0.0)
+ SD->repl.heap.heap->age = min_age;
+ /*
+ * Reinsert all bumped locked entries back into heap...
+ */
+ while ((e = linklistShift(&locked_entries)))
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+#else
+ for (m = SD->repl.lru.list.tail; m; m = prev) {
+ prev = m->prev;
+ e = m->data;
+ scanned++;
+ if (storeEntryLocked(e)) {
+ /*
+ * If there is a locked entry at the tail of the LRU list,
+ * move it to the beginning to get it out of the way.
+ * Theoretically, we might have all locked objects at the
+ * tail, and then we'll never remove anything here and the
+ * LRU age will go to zero.
+ */
+ if (memInUse(MEM_STOREENTRY) > max_scan) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+ locked++;
+
+ } else if (storeUfsDirCheckExpired(SD, e)) {
+ expired++;
+ storeRelease(e);
+ }
+ if (expired >= max_remove)
+ break;
+ if (scanned >= max_scan)
+ break;
+ }
+#endif
+ debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d l
+ocked %d f=%.03f\n",
+ scanned, max_scan, expired, max_remove, locked, f);
+ debug(20, 3) ("storeMaintainSwapSpace stats:\n");
+ debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
+ debug(20, 3) (" %6d were scanned\n", scanned);
+ debug(20, 3) (" %6d were locked\n", locked);
+ debug(20, 3) (" %6d were expired\n", expired);
+ if (store_swap_size < Config.Swap.maxSize)
+ return;
+ if (squid_curtime - last_warn_time < 10)
+ return;
+ debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
+ store_swap_size, Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
+}
+
+/*
+ * storeUfsDirCheckObj
+ *
+ * This routine is called by storeDirSelectSwapDir to see if the given
+ * object is able to be stored on this filesystem. UFS filesystems will
+ * happily store anything as long as the LRU time isn't too small.
+ */
+int
+storeUfsDirCheckObj(SwapDir *SD, const StoreEntry *e)
+{
+#if !HEAP_REPLACEMENT
+ if (storeUfsDirExpiredReferenceAge(SD) < 300) {
+ debug(20, 3) ("storeUfsDirCheckObj: NO: LRU Age = %d\n",
+ storeUfsDirExpiredReferenceAge(SD));
+ /* store_check_cachable_hist.no.lru_age_too_low++; */
+ return -1;
+ }
+#endif
+ /* Return 999 (99.9%) constant load */
+ return 999;
+}
+
+/*
+ * storeUfsDirRefObj
+ *
+ * This routine is called whenever an object is referenced, so we can
+ * maintain replacement information within the storage fs.
+ */
+void
+storeUfsDirRefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeUfsDirRefObj: referencing %p %d/%d\n", e, e->swap_dirn,
+ e->swap_filen);
+#if HEAP_REPLACEMENT
+ /* Nothing to do here */
+#else
+ /* Reference the object */
+ if (!EBIT_TEST(e->flags, RELEASE_REQUEST) &&
+ !EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+ }
+#endif
+}
+
+/*
+ * storeUfsDirUnrefObj
+ * This routine is called whenever the last reference to an object is
+ * removed, to maintain replacement information within the storage fs.
+ */
+void
+storeUfsDirUnrefObj(SwapDir *SD, StoreEntry *e)
+{
+ debug(1, 3) ("storeUfsDirUnrefObj: referencing %p %d/%d\n", e, e->swap_dirn,
+ e->swap_filen);
+#if HEAP_REPLACEMENT
+ if (e->repl.node)
+ heap_update(SD->repl.heap.heap, e->repl.node, e);
+#endif
+}
+
+/*
+ * storeUfsDirUnlinkFile
+ *
+ * This routine unlinks a file and pulls it out of the bitmap.
+ * It used to be in storeUfsUnlink(), however an interface change
+ * forced this bit of code here. Eeek.
+ */
+void
+storeUfsDirUnlinkFile(SwapDir *SD, sfileno f)
+{
+ debug(79, 3) ("storeUfsDirUnlinkFile: unlinking fileno %08X\n", f);
+ storeUfsDirMapBitReset(SD, f);
+ unlinkdUnlink(storeUfsDirFullPath(SD, f, NULL));
+}
+
+#if !HEAP_REPLACEMENT
+/*
+ * storeUfsDirExpiredReferenceAge
+ *
+ * The LRU age is scaled exponentially between 1 minute and
+ * Config.referenceAge , when store_swap_low < store_swap_size <
+ * store_swap_high. This keeps store_swap_size within the low and high
+ * water marks. If the cache is very busy then store_swap_size stays
+ * closer to the low water mark, if it is not busy, then it will stay
+ * near the high water mark. The LRU age value can be examined on the
+ * cachemgr 'info' page.
+ */
+static time_t
+storeUfsDirExpiredReferenceAge(SwapDir *SD)
+{
+ double x;
+ double z;
+ time_t age;
+ long store_high, store_low;
+
+ store_high = (long) (((float) SD->max_size *
+ (float) Config.Swap.highWaterMark) / (float) 100);
+ store_low = (long) (((float) SD->max_size *
+ (float) Config.Swap.lowWaterMark) / (float) 100);
+ debug(20, 20) ("RA: Dir %s, hi=%d, lo=%d, cur=%d\n", SD->path, store_high, store_low, SD->cur_size);
+
+ x = (double) (store_high - SD->cur_size) /
+ (store_high - store_low);
+ x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
+ z = pow((double) (Config.referenceAge / 60), x);
+ age = (time_t) (z * 60.0);
+ if (age < 60)
+ age = 60;
+ else if (age > Config.referenceAge)
+ age = Config.referenceAge;
+ return age;
+}
+#endif
+
+/*
+ * storeUfsDirCheckExpired
+ *
+ * Check whether the given object is expired or not
+ * It breaks layering a little by calling the upper layers to find
+ * out whether the object is locked or not, but we can't help this
+ * right now.
+ */
+static int
+storeUfsDirCheckExpired(SwapDir *SD, StoreEntry *e)
+{
+ if (storeEntryLocked(e))
+ return 0;
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ return 1;
+ if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
+ return 1;
+
+#if HEAP_REPLACEMENT
+ /*
+ * with HEAP_REPLACEMENT we are not using the LRU reference age, the heap
+ * controls the replacement of objects.
+ */
+ return 1;
+#else
+ if (squid_curtime - e->lastref > storeUfsDirExpiredReferenceAge(SD))
+ return 1;
+ return 0;
+#endif
+}
+
+/*
+ * Add and remove the given StoreEntry from the replacement policy in
+ * use.
+ */
+
+void
+storeUfsDirReplAdd(SwapDir *SD, StoreEntry *e)
+{
+ debug(20, 4) ("storeUfsDirReplAdd: added node %p to dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ (void) 0;
+ } else {
+ e->repl.node = heap_insert(SD->repl.heap.heap, e);
+ debug(20, 4) ("storeUfsDirReplAdd: inserted node 0x%x\n", e->repl.node);
+ }
+#else
+ /* Shouldn't we not throw special objects into the lru ? */
+ dlinkAdd(e, &e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+void
+storeUfsDirReplRemove(StoreEntry *e)
+{
+ SwapDir *SD = INDEXSD(e->swap_dirn);
+ debug(20, 4) ("storeUfsDirReplRemove: remove node %p from dir %d\n", e,
+ SD->index);
+#if HEAP_REPLACEMENT
+ /* And now, release the object from the replacement policy */
+ if (e->repl.node) {
+ debug(20, 4) ("storeUfsDirReplRemove: deleting node 0x%x\n",
+ e->repl.node);
+ heap_delete(SD->repl.heap.heap, e->repl.node);
+ e->repl.node = NULL;
+ }
+#else
+ dlinkDelete(&e->repl.lru, &SD->repl.lru.list);
+#endif
+}
+
+
+
+/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */
+
+void
+storeUfsDirStats(SwapDir *SD, StoreEntry * sentry)
+{
+ ufsinfo_t *ufsinfo;
+#if HAVE_STATVFS
+ struct statvfs sfs;
+#endif
+ ufsinfo = (ufsinfo_t *)SD->fsdata;
+ storeAppendPrintf(sentry, "First level subdirectories: %d\n", ufsinfo->l1);
+ storeAppendPrintf(sentry, "Second level subdirectories: %d\n", ufsinfo->l2);
+ storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size);
+ storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size);
+ storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n",
+ 100.0 * SD->cur_size / SD->max_size);
+ storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n",
+ ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files,
+ percent(ufsinfo->map->n_files_in_map, ufsinfo->map->max_n_files));
+#if HAVE_STATVFS
+#define fsbtoblk(num, fsbs, bs) \
+ (((fsbs) != 0 && (fsbs) < (bs)) ? \
+ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
+ if (!statvfs(SD->path, &sfs)) {
+ storeAppendPrintf(sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
+ fsbtoblk((sfs.f_blocks - sfs.f_bfree), sfs.f_frsize, 1024),
+ fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024),
+ percent(sfs.f_blocks - sfs.f_bfree, sfs.f_blocks));
+ storeAppendPrintf(sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
+ sfs.f_files - sfs.f_ffree, sfs.f_files,
+ percent(sfs.f_files - sfs.f_ffree, sfs.f_files));
+ }
+#endif
+ storeAppendPrintf(sentry, "Flags:");
+ if (SD->flags.selected)
+ storeAppendPrintf(sentry, " SELECTED");
+ if (SD->flags.read_only)
+ storeAppendPrintf(sentry, " READ-ONLY");
+ storeAppendPrintf(sentry, "\n");
+#if !HEAP_REPLACEMENT
+ storeAppendPrintf(sentry, "LRU Expiration Age: %6.2f days\n",
+ (double) storeUfsDirExpiredReferenceAge(SD) / 86400.0);
+#else
+#if 0
+ storeAppendPrintf(sentry, "Storage Replacement Threshold:\t%f\n",
+ heap_peepminkey(sd.repl.heap.heap));
+#endif
+#endif
+}
+
+/*
+ * storeUfsDirReconfigure
+ *
+ * This routine is called when the given swapdir needs reconfiguring
+ */
+void
+storeUfsDirReconfigure(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ unsigned int read_only = 0;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeUfsDirReconfigure: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeUfsDirReconfigure: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeUfsDirReconfigure: invalid level 2 directories value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ /* just reconfigure it */
+ if (size == sd->max_size)
+ debug(3, 1) ("Cache dir '%s' size remains unchanged at %d KB\n",
+ path, size);
+ else
+ debug(3, 1) ("Cache dir '%s' size changed to %d KB\n",
+ path, size);
+ sd->max_size = size;
+ if (sd->flags.read_only != read_only)
+ debug(3, 1) ("Cache dir '%s' now %s\n",
+ path, read_only ? "Read-Only" : "Read-Write");
+ sd->flags.read_only = read_only;
+ return;
+}
+
+void
+storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata;
+ storeAppendPrintf(entry, "%s %s %s %d %d %d\n",
+ name,
+ "ufs",
+ s->path,
+ s->max_size >> 10,
+ ufsinfo->l1,
+ ufsinfo->l2);
+}
+
+/*
+ * Only "free" the filesystem specific stuff here
+ */
+static void
+storeUfsDirFree(SwapDir * s)
+{
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)s->fsdata;
+ if (ufsinfo->swaplog_fd > -1) {
+ file_close(ufsinfo->swaplog_fd);
+ ufsinfo->swaplog_fd = -1;
+ }
+ filemapFreeMemory(ufsinfo->map);
+ xfree(ufsinfo);
+ s->fsdata = NULL; /* Will aid debugging... */
+
+}
+
+char *
+storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath)
+{
+ LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN);
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata;
+ int L1 = ufsinfo->l1;
+ int L2 = ufsinfo->l2;
+ if (!fullpath)
+ fullpath = fullfilename;
+ fullpath[0] = '\0';
+ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
+ SD->path,
+ ((filn / L2) / L2) % L1,
+ (filn / L2) % L2,
+ filn);
+ return fullpath;
+}
+
+/*
+ * storeUfsCleanupDoubleCheck
+ *
+ * This is called by storeCleanup() if -S was given on the command line.
+ */
+static int
+storeUfsCleanupDoubleCheck(SwapDir *sd, StoreEntry *e)
+{
+ struct stat sb;
+
+ if (stat(storeUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n");
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
+ storeUfsDirFullPath(sd, e->swap_filen, NULL));
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ if (e->swap_file_sz != sb.st_size) {
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n");
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
+ storeUfsDirFullPath(sd, e->swap_filen, NULL));
+ debug(20, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
+ e->swap_file_sz, (int) sb.st_size);
+ storeEntryDump(e, 0);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * storeUfsDirParse
+ *
+ * Called when a *new* fs is being setup.
+ */
+void
+storeUfsDirParse(SwapDir *sd, int index, char *path)
+{
+ char *token;
+ int i;
+ int size;
+ int l1;
+ int l2;
+ unsigned int read_only = 0;
+ ufsinfo_t *ufsinfo;
+
+ i = GetInteger();
+ size = i << 10; /* Mbytes to kbytes */
+ if (size <= 0)
+ fatal("storeUfsDirParse: invalid size value");
+ i = GetInteger();
+ l1 = i;
+ if (l1 <= 0)
+ fatal("storeUfsDirParse: invalid level 1 directories value");
+ i = GetInteger();
+ l2 = i;
+ if (l2 <= 0)
+ fatal("storeUfsDirParse: invalid level 2 directories value");
+ if ((token = strtok(NULL, w_space)))
+ if (!strcasecmp(token, "read-only"))
+ read_only = 1;
+
+ ufsinfo = xmalloc(sizeof(ufsinfo_t));
+ if (ufsinfo == NULL)
+ fatal("storeUfsDirParse: couldn't xmalloc() ufsinfo_t!\n");
+
+ sd->index = index;
+ sd->path = xstrdup(path);
+ sd->max_size = size;
+ sd->fsdata = ufsinfo;
+ ufsinfo->l1 = l1;
+ ufsinfo->l2 = l2;
+ ufsinfo->swaplog_fd = -1;
+ ufsinfo->map = NULL; /* Debugging purposes */
+ ufsinfo->suggest = 0;
+ sd->flags.read_only = read_only;
+ sd->init = storeUfsDirInit;
+ sd->newfs = storeUfsDirNewfs;
+ sd->dump = storeUfsDirDump;
+ sd->freefs = storeUfsDirFree;
+ sd->dblcheck = storeUfsCleanupDoubleCheck;
+ sd->statfs = storeUfsDirStats;
+ sd->maintainfs = storeUfsDirMaintain;
+ sd->checkobj = storeUfsDirCheckObj;
+ sd->refobj = storeUfsDirRefObj;
+ sd->unrefobj = storeUfsDirUnrefObj;
+ sd->callback = NULL;
+ sd->sync = NULL;
+ sd->obj.create = storeUfsCreate;
+ sd->obj.open = storeUfsOpen;
+ sd->obj.close = storeUfsClose;
+ sd->obj.read = storeUfsRead;
+ sd->obj.write = storeUfsWrite;
+ sd->obj.unlink = storeUfsUnlink;
+ sd->log.open = storeUfsDirOpenSwapLog;
+ sd->log.close = storeUfsDirCloseSwapLog;
+ sd->log.write = storeUfsDirSwapLog;
+ sd->log.clean.open = storeUfsDirWriteCleanOpen;
+
+ /* Initialise replacement policy stuff */
+#if HEAP_REPLACEMENT
+ /*
+ * Create new heaps with cache replacement policies attached to them.
+ * The cache replacement policy is specified as either GDSF or LFUDA in
+ * the squid.conf configuration file. Note that the replacement policy
+ * applies only to the disk replacement algorithm. Memory replacement
+ * always uses GDSF since we want to maximize object hit rate.
+ */
+ if (Config.replPolicy) {
+ if (tolower(Config.replPolicy[0]) == 'g') {
+ debug(20, 1) ("Using GDSF disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ } else if (tolower(Config.replPolicy[0]) == 'l') {
+ if (tolower(Config.replPolicy[1]) == 'f') {
+ debug(20, 1) ("Using LFUDA disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
+ } else if (tolower(Config.replPolicy[1]) == 'r') {
+ debug(20, 1) ("Using LRU heap disk replacement policy\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
+ }
+ } else {
+ debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+ } else {
+ debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
+ sd->repl.heap.heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
+ }
+#else
+ sd->repl.lru.list.head = NULL;
+ sd->repl.lru.list.tail = NULL;
+#endif
+}
+
+/*
+ * Initial setup / end destruction
+ */
+void
+storeUfsDirDone(void)
+{
+ memPoolDestroy(ufs_state_pool);
+ ufs_initialised = 0;
+}
+
+void
+storeFsSetup_ufs(storefs_entry_t *storefs)
+{
+ assert(!ufs_initialised);
+ storefs->parsefunc = storeUfsDirParse;
+ storefs->reconfigurefunc = storeUfsDirReconfigure;
+ storefs->donefunc = storeUfsDirDone;
+ ufs_state_pool = memPoolCreate("UFS IO State data", sizeof(ufsstate_t));
+ ufs_initialised = 1;
+}
+
--- /dev/null
+
+/*
+ * $Id: store_io_ufs.cc,v 1.1 2000/05/03 17:15:48 adrian Exp $
+ *
+ * DEBUG: section 79 Storage Manager UFS Interface
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * ----------------------------------------------------------
+ *
+ * Squid is the result of efforts by numerous individuals from the
+ * Internet community. Development is led by Duane Wessels of the
+ * National Laboratory for Applied Network Research and funded by the
+ * National Science Foundation. Squid is Copyrighted (C) 1998 by
+ * Duane Wessels and the University of California San Diego. Please
+ * see the COPYRIGHT file for full details. Squid incorporates
+ * software developed and/or copyrighted by other sources. Please see
+ * the CREDITS file for full details.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ *
+ */
+
+#include "squid.h"
+#include "store_ufs.h"
+
+
+static DRCB storeUfsReadDone;
+static DWCB storeUfsWriteDone;
+static void storeUfsIOCallback(storeIOState * sio, int errflag);
+static void storeUfsIOFreeEntry(void *, int);
+
+/* === PUBLIC =========================================================== */
+
+storeIOState *
+storeUfsOpen(SwapDir *SD, StoreEntry *e, STFNCB * file_callback,
+ STIOCB * callback, void *callback_data)
+{
+ sfileno f = e->swap_filen;
+ char *path = storeUfsDirFullPath(SD, f, NULL);
+ storeIOState *sio;
+ struct stat sb;
+ int fd;
+ debug(79, 3) ("storeUfsOpen: fileno %08X\n", f);
+ fd = file_open(path, O_RDONLY);
+ if (fd < 0) {
+ debug(79, 3) ("storeUfsOpen: got failure (%d)\n", errno);
+ return NULL;
+ }
+ debug(79, 3) ("storeUfsOpen: opened FD %d\n", fd);
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeUfsIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(ufs_state_pool);
+
+ sio->swap_filen = f;
+ sio->swap_dirn = SD->index;
+ sio->mode = O_RDONLY;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ cbdataLock(callback_data);
+ sio->e = e;
+ ((ufsstate_t *)(sio->fsstate))->fd = fd;
+ ((ufsstate_t *)(sio->fsstate))->flags.writing = 0;
+ ((ufsstate_t *)(sio->fsstate))->flags.reading = 0;
+ ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0;
+ if (fstat(fd, &sb) == 0)
+ sio->st_size = sb.st_size;
+ store_open_disk_fd++;
+
+ /* We should update the heap/dlink position here ! */
+ return sio;
+}
+
+storeIOState *
+storeUfsCreate(SwapDir *SD, StoreEntry *e, STFNCB *file_callback, STIOCB *callback, void *callback_data)
+{
+ storeIOState *sio;
+ int fd;
+ int mode = (O_WRONLY | O_CREAT | O_TRUNC);
+ char *path;
+ ufsinfo_t *ufsinfo = (ufsinfo_t *)SD->fsdata;
+ sfileno filn;
+ sdirno dirn;
+
+ /* Allocate a number */
+ dirn = SD->index;
+ filn = storeUfsDirMapBitAllocate(SD);
+ ufsinfo->suggest = filn + 1;
+ /* Shouldn't we handle a 'bitmap full' error here? */
+ path = storeUfsDirFullPath(SD, filn, NULL);
+
+ debug(79, 3) ("storeUfsCreate: fileno %08X\n", filn);
+ fd = file_open(path, mode);
+ if (fd < 0) {
+ debug(79, 3) ("storeUfsCreate: got failure (%d)\n", errno);
+ return NULL;
+ }
+ debug(79, 3) ("storeUfsCreate: opened FD %d\n", fd);
+ sio = memAllocate(MEM_STORE_IO);
+ cbdataAdd(sio, storeUfsIOFreeEntry, MEM_STORE_IO);
+ sio->fsstate = memPoolAlloc(ufs_state_pool);
+
+ sio->swap_filen = filn;
+ sio->swap_dirn = dirn;
+ sio->mode = mode;
+ sio->callback = callback;
+ sio->callback_data = callback_data;
+ cbdataLock(callback_data);
+ sio->e = (StoreEntry *) e;
+ ((ufsstate_t *)(sio->fsstate))->fd = fd;
+ ((ufsstate_t *)(sio->fsstate))->flags.writing = 0;
+ ((ufsstate_t *)(sio->fsstate))->flags.reading = 0;
+ ((ufsstate_t *)(sio->fsstate))->flags.close_request = 0;
+ store_open_disk_fd++;
+
+ /* now insert into the replacement policy */
+ storeUfsDirReplAdd(SD, e);
+ return sio;
+}
+
+void
+storeUfsClose(SwapDir *SD, storeIOState * sio)
+{
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+
+ debug(79, 3) ("storeUfsClose: dirno %d, fileno %08X, FD %d\n",
+ sio->swap_dirn, sio->swap_filen, ufsstate->fd);
+ if (ufsstate->flags.reading || ufsstate->flags.writing) {
+ ufsstate->flags.close_request = 1;
+ return;
+ }
+ storeUfsIOCallback(sio, 0);
+}
+
+void
+storeUfsRead(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
+{
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+
+ assert(sio->read.callback == NULL);
+ assert(sio->read.callback_data == NULL);
+ sio->read.callback = callback;
+ sio->read.callback_data = callback_data;
+ cbdataLock(callback_data);
+ debug(79, 3) ("storeUfsRead: dirno %d, fileno %08X, FD %d\n",
+ sio->swap_dirn, sio->swap_filen, ufsstate->fd);
+ sio->offset = offset;
+ ufsstate->flags.reading = 1;
+ file_read(ufsstate->fd,
+ buf,
+ size,
+ offset,
+ storeUfsReadDone,
+ sio);
+}
+
+void
+storeUfsWrite(SwapDir *SD, storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
+{
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+ debug(79, 3) ("storeUfsWrite: dirn %d, fileno %08X, FD %d\n", sio->swap_dirn, sio->swap_filen, ufsstate->fd);
+ ufsstate->flags.writing = 1;
+ file_write(ufsstate->fd,
+ offset,
+ buf,
+ size,
+ storeUfsWriteDone,
+ sio,
+ free_func);
+}
+
+void
+storeUfsUnlink(SwapDir *SD, StoreEntry *e)
+{
+ debug(79, 3) ("storeUfsUnlink: fileno %08X\n", e->swap_filen);
+ storeUfsDirReplRemove(e);
+ storeUfsDirUnlinkFile(SD, e->swap_filen);
+}
+
+/* === STATIC =========================================================== */
+
+static void
+storeUfsReadDone(int fd, const char *buf, int len, int errflag, void *my_data)
+{
+ storeIOState *sio = my_data;
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+ STRCB *callback = sio->read.callback;
+ void *their_data = sio->read.callback_data;
+ ssize_t rlen;
+
+ debug(79, 3) ("storeUfsReadDone: dirno %d, fileno %08X, FD %d, len %d\n",
+ sio->swap_dirn, sio->swap_filen, fd, len);
+ ufsstate->flags.reading = 0;
+ if (errflag) {
+ debug(79, 3) ("storeUfsReadDone: got failure (%d)\n", errflag);
+ rlen = -1;
+ } else {
+ rlen = (ssize_t) len;
+ sio->offset += len;
+ }
+ assert(callback);
+ assert(their_data);
+ sio->read.callback = NULL;
+ sio->read.callback_data = NULL;
+ if (cbdataValid(their_data))
+ callback(their_data, buf, (size_t) rlen);
+ cbdataUnlock(their_data);
+}
+
+static void
+storeUfsWriteDone(int fd, int errflag, size_t len, void *my_data)
+{
+ storeIOState *sio = my_data;
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+ debug(79, 3) ("storeUfsWriteDone: dirno %d, fileno %08X, FD %d, len %d\n",
+ sio->swap_dirn, sio->swap_filen, fd, len);
+ ufsstate->flags.writing = 0;
+ if (errflag) {
+ debug(79, 0) ("storeUfsWriteDone: got failure (%d)\n", errflag);
+ storeUfsIOCallback(sio, errflag);
+ return;
+ }
+ sio->offset += len;
+ if (ufsstate->flags.close_request)
+ storeUfsIOCallback(sio, errflag);
+}
+
+static void
+storeUfsIOCallback(storeIOState * sio, int errflag)
+{
+ ufsstate_t *ufsstate = (ufsstate_t *)sio->fsstate;
+ debug(79, 3) ("storeUfsIOCallback: errflag=%d\n", errflag);
+ if (ufsstate->fd > -1) {
+ file_close(ufsstate->fd);
+ store_open_disk_fd--;
+ }
+ if (cbdataValid(sio->callback_data))
+ sio->callback(sio->callback_data, errflag, sio);
+ cbdataUnlock(sio->callback_data);
+ sio->callback_data = NULL;
+ sio->callback = NULL;
+ cbdataFree(sio);
+}
+
+
+/*
+ * We can't pass memFree() as a free function here, because we need to free
+ * the fsstate variable ..
+ */
+static void
+storeUfsIOFreeEntry(void *sio, int foo)
+{
+ memPoolFree(ufs_state_pool, ((storeIOState *)sio)->fsstate);
+ memFree(sio, MEM_STORE_IO);
+}
--- /dev/null
+/*
+ * store_ufs.h
+ *
+ * Internal declarations for the ufs routines
+ */
+
+#ifndef __STORE_UFS_H__
+#define __STORE_UFS_H__
+
+struct _ufsinfo_t {
+ int swaplog_fd;
+ int l1;
+ int l2;
+ fileMap *map;
+ int suggest;
+};
+
+struct _ufsstate_t {
+ int fd;
+ struct {
+ unsigned int close_request:1;
+ unsigned int reading:1;
+ unsigned int writing:1;
+ } flags;
+};
+
+typedef struct _ufsinfo_t ufsinfo_t;
+typedef struct _ufsstate_t ufsstate_t;
+
+/* The ufs_state memory pool */
+extern MemPool * ufs_state_pool;
+
+extern void storeUfsDirMapBitReset(SwapDir *, sfileno);
+extern int storeUfsDirMapBitAllocate(SwapDir *);
+extern char * storeUfsDirFullPath(SwapDir *SD, sfileno filn, char *fullpath);
+extern void storeUfsDirUnlinkFile(SwapDir *, sfileno);
+extern void storeUfsDirReplAdd(SwapDir *SD, StoreEntry *);
+extern void storeUfsDirReplRemove(StoreEntry *);
+
+/*
+ * Store IO stuff
+ */
+extern STOBJCREATE storeUfsCreate;
+extern STOBJOPEN storeUfsOpen;
+extern STOBJCLOSE storeUfsClose;
+extern STOBJREAD storeUfsRead;
+extern STOBJWRITE storeUfsWrite;
+extern STOBJUNLINK storeUfsUnlink;
+
+#endif
/*
- * $Id: globals.h,v 1.87 2000/03/06 16:23:31 wessels Exp $
+ * $Id: globals.h,v 1.88 2000/05/03 17:15:42 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
extern int store_hash_buckets; /* 0 */
extern hash_table *store_table; /* NULL */
#if HEAP_REPLACEMENT
-extern heap *store_heap;
extern heap *inmem_heap;
#else
-extern dlink_list store_list;
#endif
extern dlink_list ClientActiveRequests;
extern const String StringNull; /* { 0, 0, NULL } */
extern request_flags null_request_flags;
extern int store_open_disk_fd; /* 0 */
extern const char *SwapDirType[];
+extern storefs_entry_t *storefs_list; /* NULL */
+extern int store_swap_low;
+extern int store_swap_high;
+extern int store_pages_max;
+extern size_t store_maxobjsize;
/*
- * $Id: main.cc,v 1.309 2000/05/02 20:13:58 hno Exp $
+ * $Id: main.cc,v 1.310 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 1 Startup and Main Loop
* AUTHOR: Harvest Derived
" -F Don't serve any requests until store is rebuilt.\n"
" -N No daemon mode.\n"
" -R Do not set REUSEADDR on port.\n"
+ " -S Double-check swap during rebuild.\n"
" -V Virtual host httpd-accelerator.\n"
" -X Force full debugging.\n"
" -Y Only return UDP_HIT or UDP_MISS_NOFETCH during fast reload.\n",
#endif
if (!configured_once) {
-#if USE_ASYNC_IO
- aioInit();
-#endif
unlinkdInit();
urlInitialize();
cachemgrInit();
#endif
memInit(); /* memInit is required for config parsing */
eventInit(); /* eventInit() is required for config parsing */
+ storeFsInit(); /* required for config parsing */
parse_err = parseConfigFile(ConfigFile);
if (opt_parse_cfg_only)
releaseServerSockets();
commCloseAllSockets();
unlinkdClose();
-#if USE_ASYNC_IO
- aioSync(); /* flush pending object writes / unlinks */
-#endif
+ storeDirSync(); /* Flush pending object writes/unlinks */
storeDirWriteCleanLogs(0);
PrintRusage();
dumpMallocStats();
-#if USE_ASYNC_IO
- aioSync(); /* flush log writes */
-#endif
+ storeDirSync(); /* Flush log writes */
storeLogClose();
accessLogClose();
-#if USE_ASYNC_IO
- aioSync(); /* flush log close */
-#endif
+ storeDirSync(); /* Flush log close */
#if PURIFY || XMALLOC_TRACE
+ storeFsDone();
configFreeMemory();
storeFreeMemory();
/*stmemFreeMemory(); */
/*
- * $Id: mem.cc,v 1.44 2000/05/02 20:58:30 hno Exp $
+ * $Id: mem.cc,v 1.45 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 13 High Level Memory Pool Management
* AUTHOR: Harvest Derived
memDataInit(MEM_ACL_TIME_DATA, "acl_time_data", sizeof(acl_time_data), 0);
memDataInit(MEM_ACL_PROXY_AUTH_USER, "acl_proxy_auth_user",
sizeof(acl_proxy_auth_user), 0);
- memDataInit(MEM_AIO_RESULT_T, "aio_result_t", sizeof(aio_result_t), 0);
memDataInit(MEM_CACHEMGR_PASSWD, "cachemgr_passwd",
sizeof(cachemgr_passwd), 0);
#if USE_CACHE_DIGESTS
#if USE_CACHE_DIGESTS
memDataInit(MEM_DIGEST_FETCH_STATE, "DigestFetchState", sizeof(DigestFetchState), 0);
#endif
- memDataInit(MEM_DISK_BUF, "Disk I/O Buffer", DISK_PAGE_SIZE, 200);
memDataInit(MEM_DLINK_LIST, "dlink_list", sizeof(dlink_list), 10);
memDataInit(MEM_DLINK_NODE, "dlink_node", sizeof(dlink_node), 10);
memDataInit(MEM_DNSSERVER_T, "dnsserver_t", sizeof(dnsserver_t), 0);
memFree(p, MEM_8K_BUF);
}
-void
-memFreeDISK(void *p)
-{
- memFree(p, MEM_DISK_BUF);
-}
/*
- * $Id: protos.h,v 1.362 2000/05/02 20:41:23 hno Exp $
+ * $Id: protos.h,v 1.363 2000/05/03 17:15:42 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
extern wordlist *aclDumpGeneric(const acl *);
extern int aclPurgeMethodInUse(acl_access *);
-#if USE_ASYNC_IO
-extern int aio_cancel(aio_result_t *);
-extern int aio_open(const char *, int, mode_t, aio_result_t *);
-extern int aio_read(int, char *, int, off_t, int, aio_result_t *);
-extern int aio_write(int, char *, int, off_t, int, aio_result_t *);
-extern int aio_close(int, aio_result_t *);
-extern int aio_stat(const char *, struct stat *, aio_result_t *);
-extern int aio_unlink(const char *, aio_result_t *);
-extern int aio_opendir(const char *, aio_result_t *);
-extern aio_result_t *aio_poll_done(void);
-extern int aio_operations_pending(void);
-extern int aio_overloaded(void);
-extern int aio_sync(void);
-extern int aio_get_queue_len(void);
-
-extern void aioInit(void);
-extern void aioCancel(int);
-extern void aioOpen(const char *, int, mode_t, AIOCB *, void *);
-extern void aioClose(int);
-extern void aioWrite(int, int offset, char *, int size, AIOCB *, void *, FREE *);
-extern void aioRead(int, int offset, char *, int size, AIOCB *, void *);
-extern void aioStat(char *, struct stat *, AIOCB *, void *);
-extern void aioUnlink(const char *, AIOCB *, void *);
-extern void aioCheckCallbacks(void);
-extern void aioSync(void);
-extern int aioQueueSize(void);
-#endif
-
/*
* cache_cf.c
*/
#endif
extern void commUpdateReadBits(int, PF *);
extern void commUpdateWriteBits(int, PF *);
+extern void comm_quick_poll_required();
extern void packerToStoreInit(Packer * p, StoreEntry * e);
extern void packerToMemInit(Packer * p, MemBuf * mb);
/* packs, then prints an object using debug() */
extern void debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm);
-
+/* disk.c */
extern int file_open(const char *path, int mode);
extern void file_close(int fd);
extern void file_write(int, off_t, void *, int len, DWCB *, void *, FREE *);
extern void file_read(int, char *, int, off_t, DRCB *, void *);
extern void disk_init(void);
+/* diskd.c */
+extern diskd_queue * afile_create_queue(void);
+extern void afile_destroy_queue(diskd_queue *);
+extern void afile_sync_queue(diskd_queue *);
+extern void afile_sync(void);
+extern void afile_open(const char *path, int mode, DOCB *, void *);
+extern void afile_close(int fd, DCCB *callback, void *data);
+extern void afile_write(int, off_t, void *, int len, DWCB *, void *, FREE *);
+extern void afile_write_mbuf(int fd, off_t, MemBuf, DWCB *, void *);
+extern void afile_read(int, char *, int, off_t, DRCB *, void *);
+extern void afile_unlink(const char *path, DUCB *, void *);
+extern void afile_truncate(const char *path, DTCB *, void *);
+
extern void dnsShutdown(void);
extern void dnsInit(void);
extern void dnsSubmit(const char *lookup, HLPCB * callback, void *data);
extern double statRequestHitMemoryRatio(int minutes);
extern double statRequestHitDiskRatio(int minutes);
extern double statByteHitRatio(int minutes);
-
+extern int storeEntryLocked(const StoreEntry *);
/* StatHist */
extern void memFree2K(void *);
extern void memFree4K(void *);
extern void memFree8K(void *);
-extern void memFreeDISK(void *);
extern int memInUse(mem_type);
extern size_t memTotalAllocated(void);
extern void memDataInit(mem_type, const char *, size_t, int);
extern void InvokeHandlers(StoreEntry *);
extern int storeEntryValidToSend(StoreEntry *);
extern void storeTimestampsSet(StoreEntry *);
-#if !HEAP_REPLACEMENT
-extern time_t storeExpiredReferenceAge(void);
-#endif
extern void storeRegisterAbort(StoreEntry * e, STABH * cb, void *);
extern void storeUnregisterAbort(StoreEntry * e);
extern void storeMemObjectDump(MemObject * mem);
extern HttpReply *storeEntryReply(StoreEntry *);
extern int storeTooManyDiskFilesOpen(void);
extern void storeEntryReset(StoreEntry *);
-extern void storeHeapPositionUpdate(StoreEntry *);
+extern void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
extern void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
+extern void storeFsInit(void);
+extern void storeFsDone(void);
+extern void storeFsAdd(char *, STSETUP *);
+
+/* store_heap_replacement.c */
+#ifdef HEAP_REPLACEMENT
+extern heap_key HeapKeyGen_StoreEntry_LFUDA(void *, double);
+extern heap_key HeapKeyGen_StoreEntry_GDSF(void *, double);
+extern heap_key HeapKeyGen_StoreEntry_LRU(void *, double);
+#endif
+
+/* store_modules.c */
+extern void storeFsSetup(void);
/* store_io.c */
-extern STOBJOPEN storeOpen;
-extern STOBJCLOSE storeClose;
-extern STOBJREAD storeRead;
-extern STOBJWRITE storeWrite;
-extern STOBJUNLINK storeUnlink;
+extern storeIOState * storeCreate(StoreEntry *, STFNCB *, STIOCB *, void *);
+extern storeIOState * storeOpen(StoreEntry *, STFNCB *, STIOCB *, void *);
+extern void storeClose(storeIOState *);
+extern void storeRead(storeIOState *, char *, size_t, off_t, STRCB *, void *);
+extern void storeWrite(storeIOState *, char *, size_t, off_t, FREE *);
+extern void storeUnlink(StoreEntry *);
extern off_t storeOffset(storeIOState *);
-/*
- * store_io_ufs.c
- */
-extern storeIOState *storeUfsOpen(sfileno, mode_t, STIOCB *, void *);
-extern void storeUfsClose(storeIOState * sio);
-extern void storeUfsRead(storeIOState *, char *, size_t, off_t, STRCB *, void *);
-extern void storeUfsWrite(storeIOState *, char *, size_t, off_t, FREE *);
-extern void storeUfsUnlink(int fileno);
-
-#if USE_ASYNC_IO
-/*
- * store_io_ufs.c
- */
-extern storeIOState *storeAufsOpen(sfileno, mode_t, STIOCB *, void *);
-extern void storeAufsClose(storeIOState * sio);
-extern void storeAufsRead(storeIOState *, char *, size_t, off_t, STRCB *, void *);
-extern void storeAufsWrite(storeIOState *, char *, size_t, off_t, FREE *);
-extern void storeAufsUnlink(int fileno);
-#endif
-
-#if USE_DISKD
-/*
- * diskd.c
- */
-extern storeIOState *storeDiskdOpen(sfileno, mode_t, STIOCB *, void *);
-extern void storeDiskdClose(storeIOState * sio);
-extern void storeDiskdRead(storeIOState *, char *, size_t, off_t, STRCB *, void *);
-extern void storeDiskdWrite(storeIOState *, char *, size_t, off_t, FREE *);
-extern void storeDiskdUnlink(int fileno);
-extern STINIT storeDiskdInit;
-extern void storeDiskdReadQueue(void);
-#endif
-
/*
* store_log.c
*/
extern char *storeSwapFullPath(int, char *);
extern char *storeSwapSubSubDir(int, char *);
extern const char *storeSwapPath(int);
-extern int storeDirMapAllocate(void);
-extern int storeDirMapBitTest(int fn);
-extern int storeDirMapBitsInUse(void);
-extern int storeDirNumber(int fileno);
-extern int storeDirProperFileno(int dirn, int fn);
-extern int storeDirValidFileno(int fn, int);
extern int storeDirWriteCleanLogs(int reopen);
+extern int storeDirSelectSwapDir(const StoreEntry *);
extern int storeVerifySwapDirs(void);
extern void storeCreateSwapDirectories(void);
extern void storeDirCloseSwapLogs(void);
extern void storeDirCloseTmpSwapLog(int dirn);
extern void storeDirConfigure(void);
-extern void storeDirDiskFull(int fn);
+extern void storeDirDiskFull(sdirno);
extern void storeDirInit(void);
-extern void storeDirMapBitReset(int fn);
-extern void storeDirMapBitSet(int fn);
extern void storeDirOpenSwapLogs(void);
extern void storeDirSwapLog(const StoreEntry *, int op);
-extern void storeDirUpdateSwapSize(int fn, size_t size, int sign);
+extern void storeDirUpdateSwapSize(SwapDir *, size_t size, int sign);
+extern void storeDirSync(void);
+extern void storeDirCallback(void);
extern void storeDirLRUDelete(StoreEntry *);
extern void storeDirLRUAdd(StoreEntry *);
-/*
- * store_dir_ufs.c
- */
-extern OBJH storeUfsDirStats;
-extern void storeUfsDirParse(cacheSwap * swap);
-extern void storeUfsDirDump(StoreEntry * entry, const char *name, SwapDir * s);
-extern void storeUfsDirFree(SwapDir *);
-extern char *storeUfsFullPath(sfileno fn, char *fullpath);
-extern STINIT storeUfsDirInit;
-#if USE_ASYNC_IO
-extern void storeAufsDirParse(cacheSwap * swap);
-#endif
-#if USE_DISKD
-extern void storeDiskdDirParse(cacheSwap *);
-#endif
-
-
/*
* store_swapmeta.c
*/
/*
- * $Id: snmp_agent.cc,v 1.73 2000/03/06 16:23:34 wessels Exp $
+ * $Id: snmp_agent.cc,v 1.74 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 49 SNMP Interface
* AUTHOR: Kostas Anagnostakis
ASN_INTEGER);
break;
case PERF_SYS_CURLRUEXP:
+ /* No global LRU info anymore */
Answer = snmp_var_new_integer(Var->name, Var->name_length,
-#if !HEAP_REPLACEMENT
- (snint) (storeExpiredReferenceAge() * 100),
-#else
- 0,
-#endif
+ 0,
SMI_TIMETICKS);
break;
case PERF_SYS_CURUNLREQ:
/*
- * $Id: squid.h,v 1.200 2000/05/02 20:58:30 hno Exp $
+ * $Id: squid.h,v 1.201 2000/05/03 17:15:42 adrian Exp $
*
* AUTHOR: Duane Wessels
*
#define SQUID_NONBLOCK O_NDELAY
#endif
-#define SWAP_DIR_SHIFT 24
-#define SWAP_FILE_MASK 0x00FFFFFF
+#include <sys/param.h>
+#include <sys/mount.h>
+
+/*
+ * I'm sick of having to keep doing this ..
+ */
+
+#define INDEXSD(i) (&Config.cacheSwap.swapDirs[(i)])
#endif /* SQUID_H */
/*
- * $Id: stat.cc,v 1.324 2000/03/06 16:23:34 wessels Exp $
+ * $Id: stat.cc,v 1.325 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 18 Cache Manager Statistics
* AUTHOR: Harvest Derived
(int) e->lock_count,
storePendingNClients(e),
(int) e->refcount);
- storeAppendPrintf(s, "\tSwap File %#08X\n",
- e->swap_file_number);
+ storeAppendPrintf(s, "\tSwap Dir %d, File %#08X\n",
+ e->swap_dirn, e->swap_filen);
if (mem != NULL) {
storeAppendPrintf(s, "\tinmem_lo: %d\n", (int) mem->inmem_lo);
storeAppendPrintf(s, "\tinmem_hi: %d\n", (int) mem->inmem_hi);
store_swap_size);
storeAppendPrintf(sentry, "\tStorage Mem size:\t%d KB\n",
(int) (store_mem_size >> 10));
-#if HEAP_REPLACEMENT
- storeAppendPrintf(sentry, "\tStorage Replacement Threshold:\t%f\n",
- heap_peepminkey(store_heap));
-#else
- storeAppendPrintf(sentry, "\tStorage LRU Expiration Age:\t%6.2f days\n",
- (double) storeExpiredReferenceAge() / 86400.0);
-#endif
storeAppendPrintf(sentry, "\tMean Object Size:\t%0.2f KB\n",
n_disk_objects ? (double) store_swap_size / n_disk_objects : 0.0);
storeAppendPrintf(sentry, "\tRequests given to unlinkd:\t%d\n",
memInUse(MEM_MEMOBJECT));
storeAppendPrintf(sentry, "\t%6d Hot Object Cache Items\n",
hot_obj_count);
- storeAppendPrintf(sentry, "\t%6d Filemap bits set\n",
- storeDirMapBitsInUse());
storeAppendPrintf(sentry, "\t%6d on-disk objects\n",
n_disk_objects);
/*
- * $Id: store.cc,v 1.519 2000/05/02 21:07:36 hno Exp $
+ * $Id: store.cc,v 1.520 2000/05/03 17:15:42 adrian Exp $
*
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
/*
* local function prototypes
*/
-static int storeCheckExpired(const StoreEntry *);
-static int storeEntryLocked(const StoreEntry *);
static int storeEntryValidLength(const StoreEntry *);
static void storeGetMemSpace(int);
static void storeHashDelete(StoreEntry *);
static int storeKeepInMemory(const StoreEntry *);
static OBJH storeCheckCachableStats;
static EVH storeLateRelease;
-#if HEAP_REPLACEMENT
-static heap_key_func HeapKeyGen_StoreEntry_LFUDA;
-static heap_key_func HeapKeyGen_StoreEntry_GDSF;
-static heap_key_func HeapKeyGen_StoreEntry_LRU;
-#endif
/*
* local variables
#else
static dlink_list inmem_list;
#endif
-static int store_pages_max = 0;
-static int store_swap_high = 0;
-static int store_swap_low = 0;
+int store_pages_max = 0;
+int store_swap_high = 0;
+int store_swap_low = 0;
static Stack LateReleaseStack;
#if URL_CHECKSUM_DEBUG
e->mem_obj = new_MemObject(url, log_url);
debug(20, 3) ("new_StoreEntry: returning %p\n", e);
e->expires = e->lastmod = e->lastref = e->timestamp = -1;
- e->swap_file_number = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
return e;
}
e, storeKeyText(key));
e->key = storeKeyDup(key);
hash_join(store_table, (hash_link *) e);
-#if HEAP_REPLACEMENT
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- (void) 0;
- } else {
- e->node = heap_insert(store_heap, e);
- debug(20, 4) ("storeHashInsert: inserted node %p\n", e->node);
- }
-#endif
}
static void
storeHashDelete(StoreEntry * e)
{
hash_remove_link(store_table, (hash_link *) e);
-#if HEAP_REPLACEMENT
- if (e->node) {
- debug(20, 4) ("storeHashDelete: deleting node %p\n", e->node);
- heap_delete(store_heap, e->node);
- e->node = NULL;
- }
-#endif
storeKeyFree(e->key);
e->key = NULL;
}
void
storeLockObject(StoreEntry * e)
{
- if (e->lock_count++ == 0) {
-#if HEAP_REPLACEMENT
- /*
- * There is no reason to take any action here. Squid by
- * default is moving locked objects to the end of the LRU
- * list to keep them from getting bumped into by the
- * replacement algorithm. We can't do that so we will just
- * have to handle them.
- */
- debug(20, 4) ("storeLockObject: just locked node %p\n", e->node);
-#else
- storeDirLRUDelete(e);
- storeDirLRUAdd(e);
-#endif
- }
+ SwapDir *SD;
+
+ if (e->swap_dirn > -1)
+ SD = INDEXSD(e->swap_dirn);
+ else
+ SD = NULL;
+
+ e->lock_count++;
debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
storeKeyText(e->key), (int) e->lock_count);
e->lastref = squid_curtime;
+ /* Notify the fs that we're referencing this object again */
+ if (SD != NULL && SD->refobj != NULL)
+ SD->refobj(SD, e);
}
void
int
storeUnlockObject(StoreEntry * e)
{
+ SwapDir *SD;
e->lock_count--;
debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
storeKeyText(e->key), e->lock_count);
if (e->store_status == STORE_PENDING)
EBIT_SET(e->flags, RELEASE_REQUEST);
assert(storePendingNClients(e) == 0);
+ /* Notify the fs that we're not referencing this object any more */
+ if (e->swap_filen > -1)
+ SD = INDEXSD(e->swap_dirn);
+ else
+ SD = NULL;
+ if (SD != NULL && SD->unrefobj != NULL)
+ SD->unrefobj(SD, e);
#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e);
+ storeHeapPositionUpdate(e, SD);
#else
+#if 0
+ /* Note: From 2.4. Not sure how this relates to the unrefobj() call above */
storeDirLRUDelete(e);
storeDirLRUAdd(e);
+#endif
#endif
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
storeRelease(e);
if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE))
return; /* is already private */
if (e->key) {
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_DEL);
storeHashDelete(e);
}
storeHashDelete(e);
EBIT_CLR(e->flags, KEY_PRIVATE);
storeHashInsert(e, newkey);
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_ADD);
}
e->store_status = STORE_PENDING;
storeSetMemStatus(e, NOT_IN_MEMORY);
e->swap_status = SWAPOUT_NONE;
- e->swap_file_number = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
e->refcount = 0;
e->lastref = squid_curtime;
e->timestamp = 0; /* set in storeTimestampsSet() */
}
/* Notify the client side */
InvokeHandlers(e);
- /* Do we need to close the swapout file? */
- /* Not if we never started swapping out */
- if (e->swap_file_number > -1) {
- storeSwapOutFileClose(e);
- }
+ /* Close any swapout file */
+ storeSwapOutFileClose(e);
storeUnlockObject(e); /* unlock */
}
void
storeMaintainSwapSpace(void *datanotused)
{
- StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
- int max_scan;
- int max_remove;
int i;
- int j;
- static int ndir = 0;
- double f;
- static time_t last_warn_time = 0;
-#if !HEAP_REPLACEMENT
- SwapDir *sd;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#if HEAP_REPLACEMENT_DEBUG
- if (!verify_heap_property(store_heap)) {
- debug(20, 1) ("Heap property violated!\n");
- }
-#endif
-#endif
- /* We can't delete objects while rebuilding swap */
- if (store_dirs_rebuilding) {
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
- return;
- } else {
- f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
- max_scan = (int) (f * 400.0 + 100.0);
- if ((max_remove = stat5minClientRequests()) < 10)
- max_remove = 10;
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
- }
- debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n",
- f, max_scan, max_remove);
-#if HEAP_REPLACEMENT
- while (heap_nodes(store_heap) > 0) {
- if (store_swap_size < store_swap_low)
- break;
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- age = heap_peepminkey(store_heap);
- e = heap_extractmin(store_heap);
- e->node = NULL; /* no longer in the heap */
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * Entry is in use ... put it in a linked list to ignore it.
- */
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- /*
- * If this was a "SPECIAL" do not add it back into the heap.
- * It will always be "SPECIAL" and therefore never removed.
- */
- debug(20, 4) ("storeMaintainSwapSpace: locked url %s\n",
- (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- }
- locked++;
- continue;
- } else if (storeCheckExpired(e)) {
- /*
- * Note: This will not check the reference age ifdef
- * HEAP_REPLACEMENT, but it does some other useful
- * checks...
- */
- expired++;
- debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
- age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
- min_age = age;
- storeRelease(e);
- } else {
- /*
- * Did not expire the object so we need to add it back
- * into the heap!
- */
- debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
- if (store_swap_size < store_swap_low)
- break;
- else if (expired >= max_remove)
- break;
- else if (scanned >= max_scan)
- break;
- }
- /*
- * Bump the heap age factor.
- */
- if (min_age > 0.0)
- store_heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->node = heap_insert(store_heap, e);
-#else
+ SwapDir *SD;
+
+ /* walk each fs */
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- sd = &Config.cacheSwap.swapDirs[i];
- sd->lru_walker = sd->lru_list.tail;
+ /* call the maintain function .. */
+ SD = INDEXSD(i);
+ if (SD->maintainfs != NULL)
+ SD->maintainfs(SD);
}
- do {
- j = 0;
- for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- if (ndir >= Config.cacheSwap.n_configured)
- ndir = ndir % Config.cacheSwap.n_configured;
- sd = &Config.cacheSwap.swapDirs[ndir++];
- if (sd->cur_size < sd->low_size)
- continue;
- if (NULL == sd->lru_walker)
- continue;
- e = sd->lru_walker->data;
- sd->lru_walker = sd->lru_walker->prev;
- j++;
- scanned++;
- sd->scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- storeDirLRUDelete(e);
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL))
- storeDirLRUAdd(e);
- }
- locked++;
- } else if (storeCheckExpired(e)) {
- expired++;
- sd->removals++;
- storeRelease(e);
- }
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- }
- } while (j > 0 && expired < max_remove && scanned < max_scan);
-#endif
- debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n",
- scanned, max_scan, expired, max_remove, locked, f);
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+
+ /* Reregister a maintain event .. */
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
}
/* release an object from a cache */
-/* return number of objects released. */
void
storeRelease(StoreEntry * e)
{
storeReleaseRequest(e);
return;
}
- if (store_dirs_rebuilding && e->swap_file_number > -1) {
+ if (store_dirs_rebuilding && e->swap_filen > -1) {
storeSetPrivateKey(e);
if (e->mem_obj) {
storeSetMemStatus(e, NOT_IN_MEMORY);
destroy_MemObject(e);
}
- if (e->swap_file_number > -1) {
+ if (e->swap_filen > -1) {
/*
* Fake a call to storeLockObject(). When rebuilding is done,
* we'll just call storeUnlockObject() on these.
}
}
storeLog(STORE_LOG_RELEASE, e);
- if (e->swap_file_number > -1) {
- storeUnlink(e->swap_file_number);
+ if (e->swap_filen > -1) {
+ storeUnlink(e);
if (e->swap_status == SWAPOUT_DONE)
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1);
+ storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1);
if (!EBIT_TEST(e->flags, KEY_PRIVATE))
storeDirSwapLog(e, SWAP_LOG_DEL);
+#if 0
+ /* From 2.4. I think we do this in storeUnlink? */
storeSwapFileNumberSet(e, -1);
+#endif
}
storeSetMemStatus(e, NOT_IN_MEMORY);
destroy_StoreEntry(e);
}
/* return 1 if a store entry is locked */
-static int
+int
storeEntryLocked(const StoreEntry * e)
{
if (e->lock_count)
debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize);
}
-#if HEAP_REPLACEMENT
-#include "store_heap_replacement.c"
-#endif
-
void
storeInit(void)
{
storeDigestInit();
storeLogOpen();
#if HEAP_REPLACEMENT
- /*
- * Create new heaps with cache replacement policies attached to them.
- * The cache replacement policy is specified as either GDSF or LFUDA in
- * the squid.conf configuration file. Note that the replacement policy
- * applies only to the disk replacement algorithm. Memory replacement
- * always uses GDSF since we want to maximize object hit rate.
- */
inmem_heap = new_heap(1000, HeapKeyGen_StoreEntry_GDSF);
- if (Config.replPolicy) {
- if (tolower(Config.replPolicy[0]) == 'g') {
- debug(20, 1) ("Using GDSF disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- } else if (tolower(Config.replPolicy[0]) == 'l') {
- if (tolower(Config.replPolicy[1]) == 'f') {
- debug(20, 1) ("Using LFUDA disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
- } else if (tolower(Config.replPolicy[1]) == 'r') {
- debug(20, 1) ("Using LRU heap disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
- }
- } else {
- debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
- } else {
- debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
-#else
- inmem_list.head = inmem_list.tail = NULL;
#endif
stackInit(&LateReleaseStack);
eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
return mem->inmem_lo == 0;
}
-static int
-storeCheckExpired(const StoreEntry * e)
-{
- if (storeEntryLocked(e))
- return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
- return 1;
-}
-
-#if !HEAP_REPLACEMENT
-/*
- * storeExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-time_t
-storeExpiredReferenceAge(void)
-{
- double x;
- double z;
- time_t age;
- x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > 31536000)
- age = 31536000;
- return age;
-}
-#endif
-
void
storeNegativeCache(StoreEntry * e)
{
debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
- debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number);
+ debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn);
+ debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen);
debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count);
debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status);
debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status);
}
#if HEAP_REPLACEMENT
+/*
+ * This routine only handles memory updates these days
+ */
void
-storeHeapPositionUpdate(StoreEntry * e)
+storeHeapPositionUpdate(StoreEntry * e, SwapDir * SD)
{
- if (e->node)
- heap_update(store_heap, e->node, e);
if (e->mem_obj && e->mem_obj->node)
heap_update(inmem_heap, e->mem_obj->node, e);
}
#endif
+/*
+ * storeFsInit
+ *
+ * This routine calls the SETUP routine for each fs type.
+ * I don't know where the best place for this is, and I'm not going to shuffle
+ * around large chunks of code right now (that can be done once its working.)
+ */
+void
+storeFsInit(void)
+{
+ storeFsSetup();
+}
+
+
+/*
+ * similar to above, but is called when a graceful shutdown is to occur
+ * of each fs module.
+ */
+void
+storeFsDone(void)
+{
+ int i = 0;
+
+ while (storefs_list[i].typestr != NULL) {
+ storefs_list[i].donefunc();
+ i++;
+ }
+}
+
+/*
+ * called to add another store fs module
+ */
+void storeFsAdd(char *type, STSETUP *setup)
+{
+ int i;
+ /* find the number of currently known storefs types */
+ for (i = 0; storefs_list && storefs_list[i].typestr; i++) {
+ assert(strcmp(storefs_list[i].typestr, type)!=0);
+ }
+ /* add the new type */
+ storefs_list = xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t));
+ memset(&storefs_list[i+1],0,sizeof(storefs_entry_t));
+ storefs_list[i].typestr = type;
+ /* Call the FS to set up capabilities and initialize the FS driver */
+ setup(&storefs_list[i]);
+}
+
+#if 0
void
storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
{
storeDirLRUAdd(e);
}
}
+#endif
/*
- * $Id: store_client.cc,v 1.86 2000/04/18 06:06:17 wessels Exp $
+ * $Id: store_client.cc,v 1.87 2000/05/03 17:15:43 adrian Exp $
*
* DEBUG: section 20 Storage Manager Client-Side Interface
* AUTHOR: Duane Wessels
if (sc->type == STORE_DISK_CLIENT)
/* assert we'll be able to get the data we want */
/* maybe we should open swapin_fd here */
- assert(e->swap_file_number > -1 || storeSwapOutAble(e));
+ assert(e->swap_filen > -1 || storeSwapOutAble(e));
for (T = &mem->clients; *T; T = &(*T)->next);
*T = sc;
#if DELAY_POOLS
STCB *callback = sc->callback;
char *buf = sc->copy_buf;
assert(sc->callback);
- assert(sc->copy_buf);
sc->callback = NULL;
sc->copy_buf = NULL;
if (cbdataValid(sc->callback_data))
{
MemObject *mem = e->mem_obj;
size_t sz;
+
if (storeClientNoMoreToSend(e, sc)) {
/* There is no more to send! */
storeClientCallback(sc, 0);
/* What the client wants is in memory */
debug(20, 3) ("storeClientCopy3: Copying from memory\n");
sz = stmemCopy(&mem->data_hdr,
- sc->copy_offset, sc->copy_buf, sc->copy_size);
- storeClientCallback(sc, sz);
- return;
+ sc->copy_offset, sc->copy_buf, sc->copy_size);
+ storeClientCallback(sc, sz);
+ return;
}
+ /* What the client wants is not in memory. Schedule a disk read */
assert(STORE_DISK_CLIENT == sc->type);
assert(!sc->flags.disk_io_pending);
debug(20, 3) ("storeClientCopy3: reading from STORE\n");
return;
}
if (tlv_list == NULL) {
- debug(20, 1) ("WRNING: failed to unpack meta data\n");
+ debug(20, 1) ("WARNING: failed to unpack meta data\n");
storeClientCallback(sc, -1);
return;
}
nx = sc->next;
if (sc->callback_data == NULL) /* open slot */
continue;
+ if (sc->type != STORE_MEM_CLIENT)
+ continue;
if (sc->type == STORE_DISK_CLIENT)
if (NULL != sc->swapin_sio)
continue;
/*
- * $Id: store_dir.cc,v 1.106 2000/03/09 04:50:13 wessels Exp $
+ * $Id: store_dir.cc,v 1.107 2000/05/03 17:15:43 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
#include "squid.h"
+static int storeDirValidSwapDirSize(int, size_t);
static void storeDirLRUWalkInitHead(SwapDir * sd);
static void *storeDirLRUWalkNext(SwapDir * sd);
-const char *SwapDirType[] =
-{
- "ufs",
- "!ERROR!"
-};
-
void
storeDirInit(void)
{
} while (pid > 0 || (pid < 0 && errno == EINTR));
}
+/*
+ * Determine whether the given directory can handle this object
+ * size
+ *
+ * Note: if the object size is -1, then the only swapdirs that
+ * will return true here are ones that have max_obj_size = -1,
+ * ie any-sized-object swapdirs. This is a good thing.
+ */
+static int
+storeDirValidSwapDirSize(int swapdir, size_t objsize)
+{
+ /*
+ * If the swapdir's max_obj_size is -1, then it definitely can
+ */
+ if (Config.cacheSwap.swapDirs[swapdir].max_objsize == -1)
+ return 1;
+ /*
+ * Else, make sure that the max object size is larger than objsize
+ */
+ if (Config.cacheSwap.swapDirs[swapdir].max_objsize > objsize)
+ return 1;
+ else
+ return 0;
+}
+
+
+#if UNUSED /* Squid-2..4.DEVEL3 code */
/*
* This new selection scheme simply does round-robin on all SwapDirs.
* A SwapDir is skipped if it is over the max_size (100%) limit. If
}
#endif
-#if OLD
+#endif /* Squid-2.4.DEVEL3 code */
+
/*
- * This is Stew Forster's selection algorithm.
- * Spread load across least 3/4 of the store directories
+ * Spread load across all of the store directories
+ *
+ * Note: We should modify this later on to prefer sticking objects
+ * in the *tightest fit* swapdir to conserve space, along with the
+ * actual swapdir usage. But for now, this hack will do while
+ * testing, so you should order your swapdirs in the config file
+ * from smallest maxobjsize to unlimited (-1) maxobjsize.
+ *
+ * We also have to choose nleast == nconf since we need to consider
+ * ALL swapdirs, regardless of state. Again, this is a hack while
+ * we sort out the real usefulness of this algorithm.
*/
-static int
-storeDirSelectSwapDir(void)
+int
+storeDirSelectSwapDir(const StoreEntry *e)
{
- double least_used = 1.0;
- double high = (double) Config.Swap.highWaterMark / 100.0;
- double u;
- int dirn;
- int i, j;
+ size_t objsize;
+ size_t least_size;
+ size_t least_objsize;
+ int least_load = 1000;
+ int load;
+ int dirn = -1;
+ int i;
SwapDir *SD;
- static int nleast = 0;
- static int nconf = 0;
- static int *dirq = NULL;
- static double *diru = NULL;
- /*
- * Handle simplest case of a single swap directory immediately
- */
- if (Config.cacheSwap.n_configured == 1)
- return 0;
- /*
- * Initialise dirq on the first call or on change of number of dirs
- */
- if (nconf != Config.cacheSwap.n_configured) {
- nconf = Config.cacheSwap.n_configured;
- nleast = (nconf * 3) / 4;
- safe_free(dirq);
- dirq = (int *) xmalloc(sizeof(int) * nleast);
- safe_free(diru);
- diru = (double *) xmalloc(sizeof(double) * nconf);
- for (j = 0; j < nleast; j++)
- dirq[j] = -1;
- }
- /*
- * Scan for a non-negative dirn in the dirq array and return that one
- */
- dirn = -1;
- for (j = 0; j < nleast; j++) {
- dirn = dirq[j];
- if (dirn < 0)
- continue;
- dirq[j] = -1;
- break;
- }
- /*
- * If we found a valid dirn return it
- */
- if (dirn >= 0)
- return dirn;
- /*
- * Now for the real guts of the algorithm - building the dirq array
- */
- for (i = 0; i < nconf; i++) {
- diru[i] = 1.1;
- SD = &Config.cacheSwap.swapDirs[i];
+
+ /* Calculate the object size */
+ objsize = objectLen(e);
+ if (objsize != -1)
+ objsize += e->mem_obj->swap_hdr_sz;
+ /* Initial defaults */
+ least_size = Config.cacheSwap.swapDirs[0].cur_size;
+ least_objsize = Config.cacheSwap.swapDirs[0].max_objsize;
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ SD = &Config.cacheSwap.swapDirs[i];
SD->flags.selected = 0;
- if (SD->flags.read_only)
- continue;
- u = (double) SD->cur_size / SD->max_size;
- if (u > high)
+ if (SD->flags.read_only)
+ continue;
+ /* Valid for object size check */
+ if (!storeDirValidSwapDirSize(i, objsize))
+ continue;
+ load = SD->checkobj(SD, e);
+ if (load < 0)
continue;
- diru[i] = u;
+ if (SD->cur_size > SD->max_size)
+ continue;
+ if (load > least_load)
+ continue;
+ if ((least_objsize > 0) && (objsize > least_objsize))
+ continue;
+ /* Only use leastsize if the load is equal */
+ if ((load == least_load) && (SD->cur_size > least_size))
+ continue;
+ least_load = load;
+ least_size = SD->cur_size;
+ dirn = i;
}
- for (j = 0; j < nleast; j++) {
- dirq[j] = -1;
- least_used = 1.0;
- dirn = -1;
- for (i = 0; i < nconf; i++) {
- if (diru[i] < least_used) {
- least_used = diru[i];
- dirn = i;
- }
- }
- if (dirn < 0)
- break;
- dirq[j] = dirn;
- diru[dirn] = 1.1;
- /* set selected flag for debugging/cachemgr only */
- Config.cacheSwap.swapDirs[dirn].flags.selected = 1;
- }
- /*
- * Setup default return of 0 if no least found
- */
- if (dirq[0] < 0)
- dirq[0] = 0;
- dirn = dirq[0];
- dirq[0] = -1;
- return dirn;
-}
-#endif
-int
-storeDirValidFileno(int fn, int flag)
-{
- int dirn = fn >> SWAP_DIR_SHIFT;
- int filn = fn & SWAP_FILE_MASK;
- if (dirn > Config.cacheSwap.n_configured)
- return 0;
- if (dirn < 0)
- return 0;
- if (filn < 0)
- return 0;
- /*
- * If flag is set it means out-of-range file number should
- * be considered invalid.
- */
- if (flag)
- if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
- return 0;
- return 1;
-}
-
-int
-storeDirMapBitTest(int fn)
-{
- int dirn = fn >> SWAP_DIR_SHIFT;
- int filn = fn & SWAP_FILE_MASK;
- return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn);
-}
+ if (dirn >= 0)
+ Config.cacheSwap.swapDirs[dirn].flags.selected = 1;
-void
-storeDirMapBitSet(int fn)
-{
- int dirn = fn >> SWAP_DIR_SHIFT;
- int filn = fn & SWAP_FILE_MASK;
- file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn);
+ return dirn;
}
-void
-storeDirMapBitReset(int fn)
-{
- int dirn = fn >> SWAP_DIR_SHIFT;
- int filn = fn & SWAP_FILE_MASK;
- file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn);
-}
-int
-storeDirMapAllocate(void)
-{
- int dirn = storeDirSelectSwapDir();
- SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
- int filn = file_map_allocate(SD->map, SD->suggest);
- SD->suggest = filn + 1;
- return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK);
-}
char *
storeSwapDir(int dirn)
return Config.cacheSwap.swapDirs[dirn].path;
}
-int
-storeDirNumber(int swap_file_number)
-{
- return swap_file_number >> SWAP_DIR_SHIFT;
-}
-
-int
-storeDirProperFileno(int dirn, int fn)
-{
- return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK);
-}
-
/*
* An entry written to the swap log MUST have the following
* properties.
* a public ADD, change the key, then log a private
* DEL. So we need to log a DEL before we change a
* key from public to private.
- * 2. It MUST have a valid (> -1) swap_file_number.
+ * 2. It MUST have a valid (> -1) swap_filen.
*/
void
storeDirSwapLog(const StoreEntry * e, int op)
{
- int dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
SwapDir *sd;
- assert(dirn < Config.cacheSwap.n_configured);
assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
- assert(e->swap_file_number >= 0);
+ assert(e->swap_filen >= 0);
/*
* icons and such; don't write them to the swap log
*/
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
return;
assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
- debug(20, 3) ("storeDirSwapLog: %s %s %08X\n",
+ debug(20, 3) ("storeDirSwapLog: %s %s %d %08X\n",
swap_log_op_str[op],
storeKeyText(e->key),
- e->swap_file_number);
- sd = &Config.cacheSwap.swapDirs[dirn];
+ e->swap_dirn,
+ e->swap_filen);
+ sd = &Config.cacheSwap.swapDirs[e->swap_dirn];
sd->log.write(sd, e, op);
}
void
-storeDirUpdateSwapSize(int fn, size_t size, int sign)
+storeDirUpdateSwapSize(SwapDir *SD, size_t size, int sign)
{
- int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured;
int k = ((size + 1023) >> 10) * sign;
- Config.cacheSwap.swapDirs[dirn].cur_size += k;
+ SD->cur_size += k;
store_swap_size += k;
if (sign > 0)
n_disk_objects++;
void
storeDirStats(StoreEntry * sentry)
{
+ int i;
+ SwapDir *SD;
+
storeAppendPrintf(sentry, "Store Directory Statistics:\n");
storeAppendPrintf(sentry, "Store Entries : %d\n",
memInUse(MEM_STOREENTRY));
storeAppendPrintf(sentry, "Current Capacity : %d%% used, %d%% free\n",
percent((int) store_swap_size, (int) Config.Swap.maxSize),
percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize));
- storeUfsDirStats(sentry); /* XXX */
-}
-int
-storeDirMapBitsInUse(void)
-{
- int i;
- int n = 0;
- for (i = 0; i < Config.cacheSwap.n_configured; i++)
- n += Config.cacheSwap.swapDirs[i].map->n_files_in_map;
- return n;
+ /* Now go through each swapdir, calling its statfs routine */
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ storeAppendPrintf(sentry, "\n");
+ SD = &(Config.cacheSwap.swapDirs[i]);
+ storeAppendPrintf(sentry, "Store Directory #%d (%s): %s\n", i, SD->type,
+ storeSwapDir(i));
+ SD->statfs(SD, sentry);
+ }
}
void
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
SD = &Config.cacheSwap.swapDirs[i];;
Config.Swap.maxSize += SD->max_size;
- if (NULL == SD->map)
- SD->map = file_map_create();
SD->low_size = (int) (((float) SD->max_size *
(float) Config.Swap.lowWaterMark) / 100.0);
}
}
void
-storeDirDiskFull(int fn)
+storeDirDiskFull(sdirno dirn)
{
- int dirn = fn >> SWAP_DIR_SHIFT;
SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
assert(0 <= dirn && dirn < Config.cacheSwap.n_configured);
SD->max_size = SD->cur_size;
* storeDirWriteCleanLogs
*
* Writes a "clean" swap log file from in-memory metadata.
+ * This is a rewrite of the original function to troll each
+ * StoreDir and write the logs, and flush at the end of
+ * the run. Thanks goes to Eric Stern, since this solution
+ * came out of his COSS code.
*/
#define CLEAN_BUF_SZ 16384
int
int dirn;
#if HEAP_REPLACEMENT
int node;
-#else
- int j;
#endif
if (store_dirs_rebuilding) {
debug(20, 1) ("Not currently OK to rewrite swap log.\n");
debug(20, 1) ("log.clean.open() failed for dir #%d\n", sd->index);
continue;
}
-#if !HEAP_REPLACEMENT
- storeDirLRUWalkInitHead(sd);
-#endif
- }
-#if HEAP_REPLACEMENT
- if (NULL == store_heap)
- return 0;
- for (node = 0; node < heap_nodes(store_heap); node++) {
- e = (StoreEntry *) heap_peep(store_heap, node);
- if (e->swap_file_number < 0)
- continue;
- if (e->swap_status != SWAPOUT_DONE)
- continue;
- if (e->swap_file_sz <= 0)
- continue;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- continue;
- if (EBIT_TEST(e->flags, KEY_PRIVATE))
- continue;
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
- continue;
- dirn = storeDirNumber(e->swap_file_number);
- sd = &Config.cacheSwap.swapDirs[dirn];
- if (NULL == sd->log.clean.write)
- continue;
- sd->log.clean.write(e, sd);
- if ((++n & 0xFFFF) == 0) {
- getCurrentTime();
- debug(20, 1) (" %7d entries written so far.\n", n);
- }
- }
- /* flush */
- for (dirn = 0; dirn < N; dirn++) {
- sd = &Config.cacheSwap.swapDirs[dirn];
if (NULL == sd->log.clean.write)
continue;
- sd->log.clean.write(NULL, sd);
- }
+#if HEAP_REPLACEMENT
+ if (NULL == sd->repl.heap.heap)
+ continue;
+#endif
+#if HEAP_REPLACEMENT
+ for (node = 0; node < heap_nodes(sd->repl.heap.heap); node++)
#else
- do {
- j = 0;
- for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
- sd = &Config.cacheSwap.swapDirs[dirn];
- if (NULL == sd->log.clean.write)
- continue;
- e = storeDirLRUWalkNext(sd);
- if (NULL == e) {
- sd->log.clean.write(NULL, sd);
- continue;
- }
- j++;
- if (e->swap_file_number < 0)
+ storeDirLRUWalkInitHead(sd);
+ while ((e = storeDirLRUWalkNext(sd)) != NULL)
+#endif
+ {
+#if HEAP_REPLACEMENT
+ e = (StoreEntry *) heap_peep(sd->repl.heap.heap, node);
+#endif
+ if (e->swap_filen < 0)
continue;
if (e->swap_status != SWAPOUT_DONE)
continue;
debug(20, 1) (" %7d entries written so far.\n", n);
}
}
- } while (j > 0);
-#endif
+ /* Flush */
+ sd->log.clean.write(NULL, sd);
+ }
if (reopen)
storeDirOpenSwapLogs();
getCurrentTime();
}
#undef CLEAN_BUF_SZ
+/*
+ * sync all avaliable fs'es ..
+ */
+void
+storeDirSync(void)
+{
+ int i;
+ SwapDir *SD;
+
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ SD = &Config.cacheSwap.swapDirs[i];
+ if (SD->sync != NULL)
+ SD->sync(SD);
+ }
+}
+
+/*
+ * handle callbacks all avaliable fs'es ..
+ */
+void
+storeDirCallback(void)
+{
+ int i;
+ SwapDir *SD;
+
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ SD = &Config.cacheSwap.swapDirs[i];
+ if (SD->callback != NULL)
+ SD->callback(SD);
+ }
+}
+
+#if 0 /* from Squid-2.4.DEVEL3 */
void
storeDirLRUDelete(StoreEntry * e)
{
SwapDir *sd;
- if (e->swap_file_number < 0)
+ if (e->swap_filen < 0)
return;
- sd = &Config.cacheSwap.swapDirs[e->swap_file_number >> SWAP_DIR_SHIFT];
- dlinkDelete(&e->lru, &sd->lru_list);
+ sd = &Config.cacheSwap.swapDirs[e->swap_dirn];
+ dlinkDelete(&e->lru, &sd->repl.lru.list);
}
void
SwapDir *sd;
if (e->swap_file_number < 0)
return;
- sd = &Config.cacheSwap.swapDirs[e->swap_file_number >> SWAP_DIR_SHIFT];
- dlinkAdd(e, &e->lru, &sd->lru_list);
+ sd = &Config.cacheSwap.swapDirs[e->swap_dirn];
+ dlinkAdd(e, &e->lru, &sd->repl.lru.list);
}
+#endif /* from Squid-2.4.DEVEL3 */
static void
storeDirLRUWalkInitHead(SwapDir * sd)
{
- sd->lru_walker = sd->lru_list.head;
+ sd->repl.lru.walker = sd->repl.lru.list.head;
}
static void *
storeDirLRUWalkNext(SwapDir * sd)
{
void *p;
- if (NULL == sd->lru_walker)
+ if (NULL == sd->repl.lru.walker)
return NULL;
- p = sd->lru_walker->data;
- sd->lru_walker = sd->lru_walker->next;
+ p = sd->repl.lru.walker->data;
+ sd->repl.lru.walker = sd->repl.lru.walker->next;
return p;
}
#include "squid.h"
+/*
+ * submit a request to create a cache object for writing.
+ * The StoreEntry structure is sent as a hint to the filesystem
+ * to what will be stored in this object, to allow the filesystem
+ * to select different polices depending on object size or type.
+ */
+storeIOState *
+storeCreate(StoreEntry *e, STIOCB *file_callback, STIOCB *close_callback, void *callback_data)
+{
+ size_t objsize;
+ sdirno dirn;
+ SwapDir *SD;
+
+ /* This is just done for logging purposes */
+ objsize = objectLen(e);
+ if (objsize != -1)
+ objsize += e->mem_obj->swap_hdr_sz;
+
+ /*
+ * Pick the swapdir
+ * We assume that the header has been packed by now ..
+ */
+ dirn = storeDirSelectSwapDir(e);
+ if (dirn == -1) {
+ debug(20, 2) ("storeCreate: no valid swapdirs for this object\n");
+ return NULL;
+ }
+ debug (20, 2) ("storeCreate: Selected dir '%d' for obj size '%d'\n", dirn, objsize);
+ SD = &Config.cacheSwap.swapDirs[dirn];
+
+ /* Now that we have a fs to use, call its storeCreate function */
+ return(SD->obj.create(SD, e, file_callback, close_callback, callback_data));
+
+ /* Done */
+}
+
+/*
+ * storeOpen() is purely for reading ..
+ */
storeIOState *
-storeOpen(sfileno f, mode_t mode, STIOCB * callback, void *callback_data)
+storeOpen(StoreEntry *e, STFNCB * file_callback, STIOCB * callback,
+ void *callback_data)
{
- SwapDir *SD = &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT];
- assert(mode == O_RDONLY || mode == O_WRONLY);
- return SD->obj.open(f, mode, callback, callback_data);
+ SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn];
+ return SD->obj.open(SD, e, file_callback, callback, callback_data);
}
void
storeClose(storeIOState * sio)
{
- SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT];
+ SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn];
if (sio->flags.closing)
return;
sio->flags.closing = 1;
- SD->obj.close(sio);
+ SD->obj.close(SD, sio);
}
void
storeRead(storeIOState * sio, char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data)
{
- SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT];
- SD->obj.read(sio, buf, size, offset, callback, callback_data);
+ SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn];
+ SD->obj.read(SD, sio, buf, size, offset, callback, callback_data);
}
void
storeWrite(storeIOState * sio, char *buf, size_t size, off_t offset, FREE * free_func)
{
- SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_file_number >> SWAP_DIR_SHIFT];
- SD->obj.write(sio, buf, size, offset, free_func);
+ SwapDir *SD = &Config.cacheSwap.swapDirs[sio->swap_dirn];
+ SD->obj.write(SD, sio, buf, size, offset, free_func);
}
void
-storeUnlink(sfileno f)
+storeUnlink(StoreEntry *e)
{
- SwapDir *SD = &Config.cacheSwap.swapDirs[f >> SWAP_DIR_SHIFT];
- SD->obj.unlink(f);
+ SwapDir *SD = INDEXSD(e->swap_dirn);
+ SD->obj.unlink(SD, e);
}
off_t
/*
- * $Id: store_log.cc,v 1.12 2000/03/24 20:40:29 wessels Exp $
+ * $Id: store_log.cc,v 1.13 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 20 Storage Manager Logging Functions
* AUTHOR: Duane Wessels
mem->log_url = xstrdup(mem->url);
}
reply = mem->reply;
- logfilePrintf(storelog, "%9d.%03d %-7s %08X %4d %9d %9d %9d %s %d/%d %s %s\n",
+ /*
+ * XXX Ok, where should we print the dir number here?
+ * Because if we print it before the swap file number, it'll break
+ * the existing log format.
+ */
+ logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %4d %9d %9d %9d %s %d/%d %s %s\n",
(int) current_time.tv_sec,
(int) current_time.tv_usec / 1000,
storeLogTags[tag],
- e->swap_file_number,
+ e->swap_dirn,
+ e->swap_filen,
reply->sline.status,
(int) reply->date,
(int) reply->last_modified,
/*
- * $Id: store_rebuild.cc,v 1.69 2000/03/06 16:23:35 wessels Exp $
+ * $Id: store_rebuild.cc,v 1.70 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 20 Store Rebuild Routines
* AUTHOR: Duane Wessels
static void storeCleanup(void *);
static int
-storeCleanupDoubleCheck(const StoreEntry * e)
+storeCleanupDoubleCheck(StoreEntry * e)
{
- /* XXX too UFS specific */
- struct stat sb;
- int dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
- if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_UFS)
- (void) 0;
- if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_ASYNCUFS)
- (void) 0;
- else
- return 0;
- if (stat(storeUfsFullPath(e->swap_file_number, NULL), &sb) < 0) {
- debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n");
- debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
- debug(20, 0) ("storeCleanup: PATH %s\n",
- storeUfsFullPath(e->swap_file_number, NULL));
- storeEntryDump(e, 0);
- return -1;
- }
- if (e->swap_file_sz != sb.st_size) {
- debug(20, 0) ("storeCleanup: SIZE MISMATCH\n");
- debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
- debug(20, 0) ("storeCleanup: PATH %s\n",
- storeUfsFullPath(e->swap_file_number, NULL));
- debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n",
- e->swap_file_sz, (int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
- }
- return 0;
+ SwapDir *SD = &Config.cacheSwap.swapDirs[e->swap_dirn];
+ return (SD->dblcheck(SD, e));
}
static void
* Calling storeRelease() has no effect because we're
* still in 'store_rebuilding' state
*/
- if (e->swap_file_number < 0)
+ if (e->swap_filen < 0)
continue;
if (opt_store_doublecheck)
if (storeCleanupDoubleCheck(e))
* Only set the file bit if we know its a valid entry
* otherwise, set it in the validation procedure
*/
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
+ storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
if ((++validnum & 0x3FFFF) == 0)
debug(20, 1) (" %7d Entries Validated so far.\n", validnum);
}
/*
- * $Id: store_swapin.cc,v 1.22 2000/03/06 16:23:35 wessels Exp $
+ * $Id: store_swapin.cc,v 1.23 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 20 Storage Manager Swapin Functions
* AUTHOR: Duane Wessels
#include "squid.h"
static STIOCB storeSwapInFileClosed;
+static STFNCB storeSwapInFileNotify;
void
storeSwapInStart(store_client * sc)
/* We're still reloading and haven't validated this entry yet */
return;
}
- debug(20, 3) ("storeSwapInStart: called for %08X %s \n",
- e->swap_file_number, storeKeyText(e->key));
+ debug(20, 3) ("storeSwapInStart: called for %d %08X %s \n",
+ e->swap_dirn, e->swap_filen, storeKeyText(e->key));
if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) {
debug(20, 1) ("storeSwapInStart: bad swap_status (%s)\n",
swapStatusStr[e->swap_status]);
return;
}
- if (e->swap_file_number < 0) {
- debug(20, 1) ("storeSwapInStart: swap_file_number < 0\n");
+ if (e->swap_filen < 0) {
+ debug(20, 1) ("storeSwapInStart: swap_filen < 0\n");
return;
}
assert(e->mem_obj != NULL);
debug(20, 3) ("storeSwapInStart: Opening fileno %08X\n",
- e->swap_file_number);
- sc->swapin_sio = storeOpen(e->swap_file_number,
- O_RDONLY,
- storeSwapInFileClosed,
- sc);
+ e->swap_filen);
+ sc->swapin_sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed,
+ sc);
cbdataLock(sc->swapin_sio);
}
callback(sc->callback_data, sc->copy_buf, errflag);
}
}
+
+static void
+storeSwapInFileNotify(void *data, int errflag, storeIOState * sio)
+{
+ store_client *sc = data;
+ StoreEntry *e = sc->entry;
+
+ debug(1, 3) ("storeSwapInFileNotify: changing %d/%d to %d/%d\n", e->swap_filen, e->swap_dirn, sio->swap_filen, sio->swap_dirn);
+
+ e->swap_filen = sio->swap_filen;
+ e->swap_dirn = sio->swap_dirn;
+}
/*
- * $Id: store_swapout.cc,v 1.64 2000/04/18 06:06:17 wessels Exp $
+ * $Id: store_swapout.cc,v 1.65 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 20 Storage Manager Swapout Functions
* AUTHOR: Duane Wessels
static off_t storeSwapOutObjectBytesOnDisk(const MemObject *);
static void storeSwapOutStart(StoreEntry * e);
static STIOCB storeSwapOutFileClosed;
+static STIOCB storeSwapOutFileNotify;
/* start swapping object to disk */
static void
tlv *tlv_list;
char *buf;
assert(mem);
- storeLockObject(e);
- storeSwapFileNumberSet(e, storeDirMapAllocate());
+ /* Build the swap metadata, so the filesystem will know how much
+ * metadata there is to store
+ */
+ debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n",
+ storeUrl(e), e->swap_dirn, e->swap_filen);
+ e->swap_status = SWAPOUT_WRITING;
+ tlv_list = storeSwapMetaBuild(e);
+ buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
+ storeSwapTLVFree(tlv_list);
+ mem->swap_hdr_sz = (size_t) swap_hdr_sz;
+ /* Create the swap file */
c = xcalloc(1, sizeof(*c));
c->data = e;
cbdataAdd(c, cbdataXfree, 0);
- mem->swapout.sio = storeOpen(e->swap_file_number,
- O_WRONLY, storeSwapOutFileClosed, c);
+ mem->swapout.sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c);
if (NULL == mem->swapout.sio) {
e->swap_status = SWAPOUT_NONE;
- storeSwapFileNumberSet(e, -1);
cbdataFree(c);
- storeUnlockObject(e);
+ xfree(buf);
return;
}
- e->swap_status = SWAPOUT_WRITING;
+ storeLockObject(e); /* Don't lock until after create, or the replacement
+ * code might get confused */
+ /* Pick up the file number if it was assigned immediately */
+ e->swap_filen = mem->swapout.sio->swap_filen;
+ e->swap_dirn = mem->swapout.sio->swap_dirn;
+ /* write out the swap metadata */
cbdataLock(mem->swapout.sio);
- debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to fileno %08X\n",
- storeUrl(e), e->swap_file_number);
- tlv_list = storeSwapMetaBuild(e);
- buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
- storeSwapTLVFree(tlv_list);
- mem->swap_hdr_sz = (size_t) swap_hdr_sz;
storeWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree);
}
+static void
+storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio)
+{
+ generic_cbdata *c = data;
+ StoreEntry *e = c->data;
+ MemObject *mem = e->mem_obj;
+ assert(e->swap_status == SWAPOUT_WRITING);
+ assert(mem);
+ assert(mem->swapout.sio == sio);
+ assert(errflag == 0);
+ e->swap_filen = mem->swapout.sio->swap_filen;
+ e->swap_dirn = mem->swapout.sio->swap_dirn;
+}
+
void
storeSwapOut(StoreEntry * e)
{
off_t new_mem_lo;
off_t on_disk = 0;
size_t swapout_size;
- char *swap_buf;
ssize_t swap_buf_len;
if (mem == NULL)
return;
lowest_offset = storeLowestMemReaderOffset(e);
debug(20, 7) ("storeSwapOut: lowest_offset = %d\n",
(int) lowest_offset);
+ /*
+ * Grab the swapout_size and check to see whether we're going to defer
+ * the swapout based upon size
+ */
+ swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset);
+ if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) {
+ debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n");
+ return;
+ }
/*
* Careful. lowest_offset can be greater than inmem_hi, such
* as in the case of a range request.
*/
if (mem->inmem_hi < lowest_offset)
new_mem_lo = lowest_offset;
- else if (mem->inmem_hi - lowest_offset > DISK_PAGE_SIZE)
+ else if (mem->inmem_hi - lowest_offset > SM_PAGE_SIZE)
new_mem_lo = lowest_offset;
else
new_mem_lo = mem->inmem_lo;
assert(mem->inmem_lo <= on_disk);
if (!storeSwapOutAble(e))
return;
- swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset);
debug(20, 7) ("storeSwapOut: swapout_size = %d\n",
(int) swapout_size);
if (swapout_size == 0) {
}
if (e->store_status == STORE_PENDING) {
/* wait for a full block to write */
- if (swapout_size < DISK_PAGE_SIZE)
+ if (swapout_size < SM_PAGE_SIZE)
return;
/*
* Wait until we are below the disk FD limit, only if the
if (storeTooManyDiskFilesOpen() && !fwdCheckDeferRead(-1, e))
return;
}
+ /*
+ * Don't start swapping out until its either all in memory, or bigger
+ * than the maximum object size (so we pick a -1 maxobjsize fs)
+ */
+ if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) {
+ debug (20, 5) ("storeSwapOut: Deferring starting swapping out\n");
+ return;
+ }
/* Ok, we have stuff to swap out. Is there a swapout.sio open? */
if (e->swap_status == SWAPOUT_NONE) {
assert(mem->swapout.sio == NULL);
if (NULL == mem->swapout.sio)
return;
do {
- if (swapout_size > DISK_PAGE_SIZE)
- swapout_size = DISK_PAGE_SIZE;
- swap_buf = memAllocate(MEM_DISK_BUF);
- swap_buf_len = stmemCopy(&mem->data_hdr,
- mem->swapout.queue_offset,
- swap_buf,
- swapout_size);
- if (swap_buf_len < 0) {
- debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key));
- storeUnlink(e->swap_file_number);
- storeSwapFileNumberSet(e, -1);
- e->swap_status = SWAPOUT_NONE;
- memFree(swap_buf, MEM_DISK_BUF);
- storeReleaseRequest(e);
- storeSwapOutFileClose(e);
- return;
+ /*
+ * Evil hack time.
+ * We are paging out to disk in page size chunks. however, later on when
+ * we update the queue position, we might not have a page (I *think*),
+ * so we do the actual page update here.
+ */
+
+ if (mem->swapout.memnode == NULL) {
+ /* We need to swap out the first page */
+ mem->swapout.memnode = mem->data_hdr.head;
+ } else {
+ /* We need to swap out the next page */
+ mem->swapout.memnode = mem->swapout.memnode->next;
}
+ /*
+ * Get the length of this buffer. We are assuming(!) that the buffer
+ * length won't change on this buffer, or things are going to be very
+ * strange. I think that after the copy to a buffer is done, the buffer
+ * size should stay fixed regardless so that this code isn't confused,
+ * but we can look at this at a later date or whenever the code results
+ * in bad swapouts, whichever happens first. :-)
+ */
+ swap_buf_len = mem->swapout.memnode->len;
+
debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len);
assert(swap_buf_len > 0);
debug(20, 3) ("storeSwapOut: swapping out %d bytes from %d\n",
swap_buf_len, (int) mem->swapout.queue_offset);
mem->swapout.queue_offset += swap_buf_len;
- storeWrite(mem->swapout.sio, swap_buf, swap_buf_len, -1, memFreeDISK);
+ storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL);
/* the storeWrite() call might generate an error */
if (e->swap_status != SWAPOUT_WRITING)
break;
swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset);
if (e->store_status == STORE_PENDING)
- if (swapout_size < DISK_PAGE_SIZE)
+ if (swapout_size < SM_PAGE_SIZE)
break;
} while (swapout_size > 0);
if (NULL == mem->swapout.sio)
assert(e->swap_status == SWAPOUT_WRITING);
cbdataFree(c);
if (errflag) {
- sfileno bad = e->swap_file_number;
- debug(20, 1) ("storeSwapOutFileClosed: swapfile %08X, errflag=%d\n\t%s\n",
- bad, errflag, xstrerror());
- storeSwapFileNumberSet(e, -1);
- /*
- * yuck. re-set the filemap bit for some errors so that
- * we don't try re-using it over and over
- */
- if (errno == EPERM)
- storeDirMapBitSet(bad);
+ debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n",
+ e->swap_dirn, e->swap_filen, errflag, xstrerror());
if (errflag == DISK_NO_SPACE_LEFT) {
- storeDirDiskFull(bad);
+ storeDirDiskFull(e->swap_dirn);
storeDirConfigure();
storeConfigure();
}
- storeReleaseRequest(e);
+ if (e->swap_filen > 0)
+ storeUnlink(e);
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
e->swap_status = SWAPOUT_NONE;
+ storeReleaseRequest(e);
} else {
/* swapping complete */
- debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %08X\n",
- storeUrl(e), e->swap_file_number);
+ debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n",
+ storeUrl(e), e->swap_dirn, e->swap_filen);
e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz;
e->swap_status = SWAPOUT_DONE;
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
+ storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1);
if (storeCheckCachable(e)) {
storeLog(STORE_LOG_SWAPOUT, e);
storeDirSwapLog(e, SWAP_LOG_ADD);
/*
- * $Id: structs.h,v 1.328 2000/05/02 21:35:24 hno Exp $
+ * $Id: structs.h,v 1.329 2000/05/03 17:15:44 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
void *callback_data;
};
-struct _aio_result_t {
- int aio_return;
- int aio_errno;
-};
-
struct _wordlist {
char *key;
wordlist *next;
#if USE_ICMP
char *pinger;
#endif
-#if USE_UNLINKD
char *unlinkd;
-#endif
} Program;
#if USE_DNSSERVERS
int dnsChildren;
int nclients;
struct {
off_t queue_offset; /* relative to in-mem data */
+ mem_node *memnode; /* which node we're currently paging out */
storeIOState *sio;
} swapout;
HttpReply *reply;
size_t swap_file_sz;
u_short refcount;
u_short flags;
- sfileno swap_file_number;
-#if HEAP_REPLACEMENT
- heap_node *node;
-#else
- dlink_node lru;
+ sdirno swap_dirn;
+ sfileno swap_filen;
+ union {
+#ifdef HEAP_REPLACEMENT
+ heap_node *node;
#endif
+ dlink_node lru;
+ } repl;
u_short lock_count; /* Assume < 65536! */
mem_status_t mem_status:3;
ping_status_t ping_status:3;
};
struct _SwapDir {
- swapdir_t type;
- fileMap *map;
+ char *type;
int cur_size;
int low_size;
int max_size;
char *path;
int index; /* This entry's index into the swapDirs array */
- sfileno suggest;
+ int suggest;
+ size_t max_objsize;
+ union {
+#ifdef HEAP_REPLACEMENT
+ struct {
+ heap *heap;
+ } heap;
+#endif
+ struct {
+ dlink_list list;
+ dlink_node *walker;
+ } lru;
+ } repl;
int removals;
int scanned;
struct {
unsigned int selected:1;
unsigned int read_only:1;
} flags;
- STINIT *init;
- STNEWFS *newfs;
- struct {
+ STINIT *init; /* Initialise the fs */
+ STNEWFS *newfs; /* Create a new fs */
+ STDUMP *dump; /* Dump fs config snippet */
+ STFREE *freefs; /* Free the fs data */
+ STDBLCHECK *dblcheck; /* Double check the obj integrity */
+ STSTATFS *statfs; /* Dump fs statistics */
+ STMAINTAINFS *maintainfs; /* Replacement maintainence */
+ STCHECKOBJ *checkobj; /* Check if the fs will store an object */
+ /* These two are notifications */
+ STREFOBJ *refobj; /* Reference this object */
+ STUNREFOBJ *unrefobj; /* Unreference this object */
+ STCALLBACK *callback; /* Handle pending callbacks */
+ STSYNC *sync; /* Sync the directory */
+ struct {
+ STOBJCREATE *create;
STOBJOPEN *open;
STOBJCLOSE *close;
STOBJREAD *read;
void *state;
} clean;
} log;
-#if !HEAP_REPLACEMENT
- dlink_list lru_list;
- dlink_node *lru_walker;
-#endif
- union {
- struct {
- int l1;
- int l2;
- int swaplog_fd;
- } ufs;
-#if USE_DISKD
- struct {
- int l1;
- int l2;
- int swaplog_fd;
- int smsgid;
- int rmsgid;
- int wfd;
- int away;
- struct {
- char *buf;
- link_list *stack;
- int id;
- } shm;
- } diskd;
-#endif
- } u;
+ void *fsdata;
};
struct _request_flags {
};
struct _storeIOState {
- sfileno swap_file_number;
+ sdirno swap_dirn;
+ sfileno swap_filen;
+ StoreEntry *e; /* Need this so the FS layers can play god */
mode_t mode;
size_t st_size; /* do stat(2) after read open */
- off_t offset; /* current offset pointer */
+ off_t offset; /* current on-disk offset pointer */
+ STFNCB *file_callback; /* called on delayed sfileno assignments */
STIOCB *callback;
void *callback_data;
struct {
struct {
unsigned int closing:1; /* debugging aid */
} flags;
- union {
- struct {
- int fd;
- struct {
- unsigned int close_request:1;
- unsigned int reading:1;
- unsigned int writing:1;
- } flags;
- } ufs;
- struct {
- int fd;
- struct {
- unsigned int close_request:1;
- unsigned int reading:1;
- unsigned int writing:1;
- unsigned int opening:1;
- } flags;
- const char *read_buf;
- link_list *pending_writes;
- link_list *pending_reads;
- } aufs;
-#if USE_DISKD
- struct {
- int id;
- struct {
- unsigned int reading:1;
- unsigned int writing:1;
- } flags;
- char *read_buf;
- } diskd;
-#endif
- } type;
+ void *fsstate;
};
struct _request_t {
tlv *next;
};
+/*
+ * Do we need to have the dirn in here? I don't think so, since we already
+ * know the dirn ..
+ */
struct _storeSwapLogData {
char op;
- sfileno swap_file_number;
+ sfileno swap_filen;
time_t timestamp;
time_t lastref;
time_t expires;
int zero_object_sz;
};
+/*
+ * This defines an fs type
+ */
+
+struct _storefs_entry {
+ char *typestr;
+ STFSPARSE *parsefunc;
+ STFSRECONFIGURE *reconfigurefunc;
+ STFSSHUTDOWN *donefunc;
+};
+
+/*
+ * Async disk IO - this defines a async disk io queue
+ */
+
+struct _diskd_queue {
+ int smsgid; /* send sysvmsg id */
+ int rmsgid; /* recv sysvmsg id */
+ int wfd; /* queue file descriptor ? */
+ int away; /* number of requests away */
+ int sent_count; /* number of messages sent */
+ int recv_count; /* number of messages received */
+ struct {
+ char *buf; /* shm buffer */
+ link_list *stack;
+ int id; /* sysvshm id */
+ } shm;
+};
+
struct _Logfile {
int fd;
char path[MAXPATHLEN];
/*
- * $Id: typedefs.h,v 1.101 2000/03/14 23:07:51 wessels Exp $
+ * $Id: typedefs.h,v 1.102 2000/05/03 17:15:44 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
typedef unsigned int ping_status_t;
typedef unsigned int swap_status_t;
typedef int sfileno;
+typedef int sdirno;
typedef struct {
size_t bytes;
typedef struct _acl_list acl_list;
typedef struct _acl_access acl_access;
typedef struct _aclCheck_t aclCheck_t;
-typedef struct _aio_result_t aio_result_t;
typedef struct _wordlist wordlist;
typedef struct _intlist intlist;
typedef struct _intrange intrange;
typedef struct _generic_cbdata generic_cbdata;
typedef struct _storeIOState storeIOState;
typedef struct _link_list link_list;
+typedef struct _storefs_entry storefs_entry_t;
+typedef struct _diskd_queue diskd_queue;
typedef struct _Logfile Logfile;
#if SQUID_SNMP
typedef struct _delaySpec delaySpec;
#endif
-/* define AIOCB even without USE_ASYNC_IO */
-typedef void AIOCB(int fd, void *, int aio_return, int aio_errno);
typedef void CWCB(int fd, char *, size_t size, int flag, void *data);
typedef void CNCB(int fd, int status, void *);
typedef void FOCB(void *, int fd, int errcode);
typedef void EVH(void *);
typedef void PF(int, void *);
-typedef void DRCB(int fd, const char *buf, int size, int errflag, void *data);
-typedef void DWCB(int, int, size_t, void *);
+
+/* disk.c / diskd.c callback typedefs */
+typedef void DRCB(int, const char *buf, int size, int errflag, void *data);
+ /* Disk read CB */
+typedef void DWCB(int, int, size_t, void *); /* disk write CB */
+typedef void DOCB(int, int errflag, void *data); /* disk open CB */
+typedef void DCCB(int, int errflag, void *data); /* disk close CB */
+typedef void DUCB(int errflag, void *data); /* disk unlink CB */
+typedef void DTCB(int errflag, void *data); /* disk trunc CB */
+
typedef void FQDNH(const char *, void *);
typedef void IDCB(const char *ident, void *data);
typedef void IPH(const ipcache_addrs *, void *);
typedef int DEFER(int fd, void *data);
typedef void STIOCB(void *their_data, int errflag, storeIOState *);
+typedef void STFNCB(void *their_data, int errflag, storeIOState *);
typedef void STRCB(void *their_data, const char *buf, ssize_t len);
typedef void SIH(storeIOState *, void *); /* swap in */
typedef void STINIT(SwapDir *);
typedef void STNEWFS(SwapDir *);
-typedef storeIOState *STOBJOPEN(sfileno, mode_t, STIOCB *, void *);
-typedef void STOBJCLOSE(storeIOState *);
-typedef void STOBJREAD(storeIOState *, char *, size_t, off_t, STRCB *, void *);
-typedef void STOBJWRITE(storeIOState *, char *, size_t, off_t, FREE *);
-typedef void STOBJUNLINK(sfileno);
+typedef void STDUMP(StoreEntry *, const char *, SwapDir *);
+typedef void STFREE(SwapDir *);
+typedef int STDBLCHECK(SwapDir *, StoreEntry *);
+typedef void STSTATFS(SwapDir *, StoreEntry *);
+typedef void STMAINTAINFS(SwapDir *);
+typedef int STCHECKOBJ(SwapDir *, const StoreEntry *);
+typedef void STREFOBJ(SwapDir *, StoreEntry *);
+typedef void STUNREFOBJ(SwapDir *, StoreEntry *);
+typedef void STSETUP(storefs_entry_t *);
+typedef void STDONE(void);
+typedef void STCALLBACK(SwapDir *);
+typedef void STSYNC(SwapDir *);
+
+typedef storeIOState *STOBJCREATE(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *);
+typedef storeIOState *STOBJOPEN(SwapDir *, StoreEntry *, STFNCB *, STIOCB *, void *);
+typedef void STOBJCLOSE(SwapDir *, storeIOState *);
+typedef void STOBJREAD(SwapDir *, storeIOState *, char *, size_t, off_t, STRCB *, void *);
+typedef void STOBJWRITE(SwapDir *, storeIOState *, char *, size_t, off_t, FREE *);
+typedef void STOBJUNLINK(SwapDir *, StoreEntry *);
+
typedef void STLOGOPEN(SwapDir *);
typedef void STLOGCLOSE(SwapDir *);
typedef void STLOGWRITE(const SwapDir *, const StoreEntry *, int);
typedef int STLOGCLEANOPEN(SwapDir *);
typedef void STLOGCLEANWRITE(const StoreEntry *, SwapDir *);
+/* Store dir configuration routines */
+/* SwapDir *sd, char *path ( + char *opt later when the strtok mess is gone) */
+typedef void STFSPARSE(SwapDir *, int, char *);
+typedef void STFSRECONFIGURE(SwapDir *, int, char *);
+typedef void STFSSTARTUP(void);
+typedef void STFSSHUTDOWN(void);
+
typedef double hbase_f(double);
typedef void StatHistBinDumper(StoreEntry *, int idx, double val, double size, int count);
/*
- * $Id: unlinkd.cc,v 1.35 2000/05/02 20:13:58 hno Exp $
+ * $Id: unlinkd.cc,v 1.36 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 12 Unlink Daemon
* AUTHOR: Duane Wessels
/* This code gets linked to Squid */
-#if USE_UNLINKD
static int unlinkd_wfd = -1;
static int unlinkd_rfd = -1;
-#endif
#define UNLINKD_QUEUE_LIMIT 20
void
unlinkdUnlink(const char *path)
{
-#if USE_UNLINKD
char buf[MAXPATHLEN];
int l;
int x;
}
Counter.unlink.requests++;
queuelen++;
-#endif
}
void
unlinkdClose(void)
{
-#if USE_UNLINKD
assert(unlinkd_wfd > -1);
debug(12, 1) ("Closing unlinkd pipe on FD %d\n", unlinkd_wfd);
file_close(unlinkd_wfd);
file_close(unlinkd_rfd);
unlinkd_wfd = -1;
unlinkd_rfd = -1;
-#endif
}
void
unlinkdInit(void)
{
-#if USE_UNLINKD
int x;
char *args[2];
struct timeval slp;
if (FD_PIPE == fd_table[unlinkd_wfd].type)
commUnsetNonBlocking(unlinkd_wfd);
debug(12, 1) ("Unlinkd pipe opened on FD %d\n", unlinkd_wfd);
-#else
- debug(12, 1) ("Unlinkd is disabled\n");
-#endif
}
#endif /* ndef UNLINK_DAEMON */
/*
- * $Id: url.cc,v 1.122 2000/05/02 20:22:55 hno Exp $
+ * $Id: url.cc,v 1.123 2000/05/03 17:15:44 adrian Exp $
*
* DEBUG: section 23 URL Parsing
* AUTHOR: Duane Wessels
port = urlDefaultPort(protocol);
/* Is there any login informaiton? */
if ((t = strrchr(host, '@'))) {
- strcpy(login, host);
+ strcpy((char *)login, (char *)host);
t = strrchr(login, '@');
*t = 0;
- strcpy(host, t + 1);
+ strcpy((char *)host, t + 1);
}
if ((t = strrchr(host, ':'))) {
*t++ = '\0';
projects / thirdparty / squid.git / commitdiff
