]> git.ipfire.org Git - thirdparty/squid.git/commitdiff
Add storeid_file_rewrite helper
authorAlan Mizrahi <alan@mizrahi.com.ve>
Tue, 9 Jul 2013 11:15:51 +0000 (05:15 -0600)
committerAmos Jeffries <squid3@treenet.co.nz>
Tue, 9 Jul 2013 11:15:51 +0000 (05:15 -0600)
Based on work by Eliezer Croitoru <eliezer@ngtech.co.il>

This program acts as a Store-ID helper program, rewriting URLs passed
by Squid into storage-ids that can be used to achieve better caching
for websites that use different URLs for the same content.

It takes a text file with two tab separated columns.
 Column 1: Regular expression to match against the URL
 Column 2: Rewrite rule to generate a Store-ID

Rewrite rules are matched in the same order as they appear in the file.
So for best performance, sort it in order of frequency of occurrence.

configure.ac
doc/release-notes/release-3.4.sgml
helpers/Makefile.am
helpers/storeid_rewrite/Makefile.am [new file with mode: 0644]
helpers/storeid_rewrite/file/Makefile.am [new file with mode: 0644]
helpers/storeid_rewrite/file/required.m4 [new file with mode: 0755]
helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in [new file with mode: 0644]
helpers/storeid_rewrite/modules.m4 [new file with mode: 0644]

index 3635c26c56fb610cd7f9d1db75be1598a6b0dc50..3379cadd1c31a0a3e532d14735b3485f25a9d507 100644 (file)
@@ -1968,6 +1968,18 @@ AC_ARG_ENABLE(url-rewrite-helpers,
 ])
 m4_include([helpers/url_rewrite/modules.m4])
 
+dnl Select storeid_rewrite helpers to build
+AC_ARG_ENABLE(storeid-rewrite-helpers,
+  AS_HELP_STRING([--enable-storeid-rewrite-helpers="list of helpers"],
+                 [This option selects which Store-ID rewrite helpers to
+                  build and install as part of the normal build
+                  process. The default is to attempt the build of all possible
+                  helpers. Use --disable-storeid-rewrite-helpers to build none.
+                  For a list of available helpers see the
+                  helpers/storeid_rewrite directory.]),[
+])
+m4_include([helpers/storeid_rewrite/modules.m4])
+
 AC_ARG_WITH(valgrind-debug,
   AS_HELP_STRING([--with-valgrind-debug],
                  [Include debug instrumentation for use with valgrind]),
@@ -3465,6 +3477,8 @@ AC_CONFIG_FILES([\
        helpers/url_rewrite/Makefile \
        helpers/url_rewrite/fake/Makefile \
        helpers/ssl/Makefile \
+       helpers/storeid_rewrite/Makefile \
+       helpers/storeid_rewrite/file/Makefile \
        tools/Makefile
        tools/purge/Makefile
 ])
index 99b53dae8a9472632bfe26910f839a640cfe226d..a44ab6bfa22cc7b135ac1446640068b305f060ab 100644 (file)
@@ -120,6 +120,10 @@ Most user-facing changes are reflected in squid.conf (see below).
    ID key and not the transaction URL. So using the Store-ID feature to alter the value
    affects which <em>refresh_pattern</em> directive will be matched.
 
+<p>Store-ID helpers bundled with Squid can be built with the --enable-storeid-rewrite-helpers
+   options which is added in this version. Currently there is a <em>file</em> helper
+   provided.
+
 
 <sect1>TPROXY Support for OpenBSD 5.1+ and FreeBSD 9+
 <p>Details at <url url="http://wiki.squid-cache.org/ConfigExamples/Intercept/OpenBsdPf">.
@@ -290,6 +294,12 @@ This section gives an account of those changes in three categories:
 <sect1>New options<label id="newoptions">
 <p>
 <descrip>
+       <tag>--enable-storeid-rewrite-helpers</tag>
+       <p>New option to control which Store-ID helpers are built. As with other
+          helper options use --disable-* to prevent any helpers building and
+          omit to get all helper auto-detected.
+       <p>Currenly only a helper using <em>file</em> for backend is provided.
+
        <tag>--with-nat-pf</tag>
        <p>New option to alter the behaviour of <em>http_port ... intercept</em> option
           in squid.conf.
index 37b765fb0b409a49477fe7fe3a4ce82e5bb33d57..cf8e90b8e85c115cdaf37d09722491b0a335a945 100644 (file)
@@ -8,7 +8,8 @@ DIST_SUBDIRS = \
        negotiate_auth \
        ntlm_auth \
        url_rewrite \
-       ssl
+       ssl \
+       storeid_rewrite
 
 SUBDIRS = \
        basic_auth \
@@ -16,7 +17,8 @@ SUBDIRS = \
        external_acl \
        log_daemon \
        negotiate_auth \
-       url_rewrite
+       url_rewrite \
+       storeid_rewrite
 
 if ENABLE_AUTH_NTLM
 SUBDIRS += ntlm_auth
diff --git a/helpers/storeid_rewrite/Makefile.am b/helpers/storeid_rewrite/Makefile.am
new file mode 100644 (file)
index 0000000..c48e019
--- /dev/null
@@ -0,0 +1,3 @@
+
+DIST_SUBDIRS   = file
+SUBDIRS                = $(STOREID_REWRITE_HELPERS)
diff --git a/helpers/storeid_rewrite/file/Makefile.am b/helpers/storeid_rewrite/file/Makefile.am
new file mode 100644 (file)
index 0000000..0a280a0
--- /dev/null
@@ -0,0 +1,15 @@
+include $(top_srcdir)/src/Common.am
+
+libexec_SCRIPTS        = storeid_file_rewrite
+CLEANFILES += storeid_file_rewrite storeid_file_rewrite.8
+man_MANS = storeid_file_rewrite.8
+EXTRA_DIST= \
+       storeid_file_rewrite.8 \
+       storeid_file_rewrite.pl.in \
+       required.m4
+
+storeid_file_rewrite.8: storeid_file_rewrite
+       pod2man storeid_file_rewrite storeid_file_rewrite.8
+
+storeid_file_rewrite: storeid_file_rewrite.pl.in
+       $(subst_perlshell)
diff --git a/helpers/storeid_rewrite/file/required.m4 b/helpers/storeid_rewrite/file/required.m4
new file mode 100755 (executable)
index 0000000..72c3e3e
--- /dev/null
@@ -0,0 +1,3 @@
+if test "x$PERL" != "x" -a "x$POD2MAN" != "x"; then
+  BUILD_HELPER="file"
+fi
diff --git a/helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in b/helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in
new file mode 100644 (file)
index 0000000..342858c
--- /dev/null
@@ -0,0 +1,92 @@
+#!@PERL@
+use strict;
+use warnings;
+$|=1;
+
+=pod
+
+=head1 NAME
+
+storeid_file_rewrite - File based Store-ID helper for Squid
+
+=head1 SYNOPSIS
+
+storeid_file_rewrite filepath
+
+=head1 DESCRIPTOIN
+
+This program acts as a store_id helper program, rewriting URLs passed
+by Squid into storage-ids that can be used to achieve better caching
+for websites that use different URLs for the same content.
+
+It takes a text file with two tab separated columns.
+Column 1: Regular expression to match against the URL
+Column 2: Rewrite rule to generate a Store-ID
+Eg:
+^http:\/\/[^\.]+\.dl\.sourceforge\.net\/(.*)   http://dl.sourceforge.net.squid.internal/$1
+
+Rewrite rules are matched in the same order as they appear in the rules file.
+So for best performance, sort it in order of frequency of occurrence.
+
+For more information please see http://wiki.squid-cache.org/Features/StoreID
+
+=cut
+
+my @rules; # array of [regex, replacement string]
+
+die "Usage: $0 <rewrite-file>\n" unless $#ARGV == 0;
+
+# read config file
+open RULES, $ARGV[0] or die "Error opening $ARGV[0]: $!";
+while (<RULES>) {
+       chomp;
+       next if /^\s*#?$/;
+       if (/^\s*([^\t]+?)\s*\t+\s*([^\t]+?)\s*$/) {
+               push(@rules, [qr/$1/, $2]);
+       } else {
+               print STDERR "$0: Parse error in $ARGV[0] (line $.)\n";
+       }
+}
+close RULES;
+
+# read urls from squid and do the replacement
+URL: while (<STDIN>) {
+       chomp;
+       last if $_ eq 'quit';
+       
+       foreach my $rule (@rules) {
+               if (my @match = /$rule->[0]/) {
+                       $_ = $rule->[1];
+                       
+                       for (my $i=1; $i<=scalar(@match); $i++) {
+                               s/\$$i/$match[$i-1]/g;
+                       }
+                       print "OK store-id=$_\n";
+                       next URL;
+               }
+       }
+       print "ERR\n";
+}
+
+=pod
+
+=head1 COPYRIGHT
+
+Copyright (C) 2013 Alan Mizrahi <alan@mizrahi.com.ve>
+Based on code from Eliezer Croitoru <eliezer@ngtech.co.il>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307, USA.
+
+=cut
diff --git a/helpers/storeid_rewrite/modules.m4 b/helpers/storeid_rewrite/modules.m4
new file mode 100644 (file)
index 0000000..755e1dd
--- /dev/null
@@ -0,0 +1,40 @@
+# This file is supposed to run all the tests required to identify which
+# configured modules are able to be built in this environment
+
+# FIXME: de-duplicate $enable_storeid_rewrite_helpers list containing double entries.
+
+#define list of modules to build
+if test "x${enable_storeid_rewrite_helpers:=yes}" = "xyes" ; then
+    SQUID_LOOK_FOR_MODULES([$srcdir/helpers/storeid_rewrite],[enable_storeid_rewrite_helpers])
+fi
+
+enable_storeid_rewrite_helpers="`echo $enable_storeid_rewrite_helpers| sed -e 's/,/ /g;s/  */ /g'`"
+AC_MSG_NOTICE([Store-ID rewrite helper candidates: $enable_storeid_rewrite_helpers])
+STOREID_REWRITE_HELPERS=""
+if test "x$enable_storeid_rewrite_helpers" != "xno" ; then
+  for helper in $enable_storeid_rewrite_helpers; do
+    dir="$srcdir/helpers/storeid_rewrite/$helper"
+
+    # modules converted to autoconf macros already
+    # NP: we only need this list because m4_include() does not accept variables
+    if test "x$helper" = "xfile" ; then
+      m4_include([helpers/storeid_rewrite/file/required.m4])
+
+    # modules not yet converted to autoconf macros (or third party drop-in's)
+    elif test -f "$dir/config.test" && sh "$dir/config.test" "$squid_host_os"; then
+      BUILD_HELPER="$helper"
+    fi
+
+    if test -d "$srcdir/helpers/storeid_rewrite/$helper"; then
+      if test "$BUILD_HELPER" != "$helper"; then
+        AC_MSG_NOTICE([Store-ID rewrite helper $helper ... found but cannot be built])
+      else
+        STOREID_REWRITE_HELPERS="$STOREID_REWRITE_HELPERS $BUILD_HELPER"
+      fi
+    else
+      AC_MSG_ERROR([Store-ID rewrite helper $helper ... not found])
+    fi
+  done
+fi
+AC_MSG_NOTICE([Store-ID rewrite helpers to be built: $STOREID_REWRITE_HELPERS])
+AC_SUBST(STOREID_REWRITE_HELPERS)