From 0d5ee502d62b98f4578f0562153877274ae40e3c Mon Sep 17 00:00:00 2001 From: Alan Mizrahi Date: Tue, 9 Jul 2013 05:15:51 -0600 Subject: [PATCH] Add storeid_file_rewrite helper Based on work by Eliezer Croitoru This program acts as a Store-ID helper program, rewriting URLs passed by Squid into storage-ids that can be used to achieve better caching for websites that use different URLs for the same content. It takes a text file with two tab separated columns. Column 1: Regular expression to match against the URL Column 2: Rewrite rule to generate a Store-ID Rewrite rules are matched in the same order as they appear in the file. So for best performance, sort it in order of frequency of occurrence. --- configure.ac | 14 +++ doc/release-notes/release-3.4.sgml | 10 ++ helpers/Makefile.am | 6 +- helpers/storeid_rewrite/Makefile.am | 3 + helpers/storeid_rewrite/file/Makefile.am | 15 +++ helpers/storeid_rewrite/file/required.m4 | 3 + .../file/storeid_file_rewrite.pl.in | 92 +++++++++++++++++++ helpers/storeid_rewrite/modules.m4 | 40 ++++++++ 8 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 helpers/storeid_rewrite/Makefile.am create mode 100644 helpers/storeid_rewrite/file/Makefile.am create mode 100755 helpers/storeid_rewrite/file/required.m4 create mode 100644 helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in create mode 100644 helpers/storeid_rewrite/modules.m4 diff --git a/configure.ac b/configure.ac index 3635c26c56..3379cadd1c 100644 --- a/configure.ac +++ b/configure.ac @@ -1968,6 +1968,18 @@ AC_ARG_ENABLE(url-rewrite-helpers, ]) m4_include([helpers/url_rewrite/modules.m4]) +dnl Select storeid_rewrite helpers to build +AC_ARG_ENABLE(storeid-rewrite-helpers, + AS_HELP_STRING([--enable-storeid-rewrite-helpers="list of helpers"], + [This option selects which Store-ID rewrite helpers to + build and install as part of the normal build + process. The default is to attempt the build of all possible + helpers. Use --disable-storeid-rewrite-helpers to build none. + For a list of available helpers see the + helpers/storeid_rewrite directory.]),[ +]) +m4_include([helpers/storeid_rewrite/modules.m4]) + AC_ARG_WITH(valgrind-debug, AS_HELP_STRING([--with-valgrind-debug], [Include debug instrumentation for use with valgrind]), @@ -3465,6 +3477,8 @@ AC_CONFIG_FILES([\ helpers/url_rewrite/Makefile \ helpers/url_rewrite/fake/Makefile \ helpers/ssl/Makefile \ + helpers/storeid_rewrite/Makefile \ + helpers/storeid_rewrite/file/Makefile \ tools/Makefile tools/purge/Makefile ]) diff --git a/doc/release-notes/release-3.4.sgml b/doc/release-notes/release-3.4.sgml index 99b53dae8a..a44ab6bfa2 100644 --- a/doc/release-notes/release-3.4.sgml +++ b/doc/release-notes/release-3.4.sgml @@ -120,6 +120,10 @@ Most user-facing changes are reflected in squid.conf (see below). ID key and not the transaction URL. So using the Store-ID feature to alter the value affects which refresh_pattern directive will be matched. +

Store-ID helpers bundled with Squid can be built with the --enable-storeid-rewrite-helpers + options which is added in this version. Currently there is a file helper + provided. + TPROXY Support for OpenBSD 5.1+ and FreeBSD 9+

Details at . @@ -290,6 +294,12 @@ This section gives an account of those changes in three categories: New options

+ --enable-storeid-rewrite-helpers +

New option to control which Store-ID helpers are built. As with other + helper options use --disable-* to prevent any helpers building and + omit to get all helper auto-detected. +

Currenly only a helper using file for backend is provided. + --with-nat-pf

New option to alter the behaviour of http_port ... intercept option in squid.conf. diff --git a/helpers/Makefile.am b/helpers/Makefile.am index 37b765fb0b..cf8e90b8e8 100644 --- a/helpers/Makefile.am +++ b/helpers/Makefile.am @@ -8,7 +8,8 @@ DIST_SUBDIRS = \ negotiate_auth \ ntlm_auth \ url_rewrite \ - ssl + ssl \ + storeid_rewrite SUBDIRS = \ basic_auth \ @@ -16,7 +17,8 @@ SUBDIRS = \ external_acl \ log_daemon \ negotiate_auth \ - url_rewrite + url_rewrite \ + storeid_rewrite if ENABLE_AUTH_NTLM SUBDIRS += ntlm_auth diff --git a/helpers/storeid_rewrite/Makefile.am b/helpers/storeid_rewrite/Makefile.am new file mode 100644 index 0000000000..c48e019115 --- /dev/null +++ b/helpers/storeid_rewrite/Makefile.am @@ -0,0 +1,3 @@ + +DIST_SUBDIRS = file +SUBDIRS = $(STOREID_REWRITE_HELPERS) diff --git a/helpers/storeid_rewrite/file/Makefile.am b/helpers/storeid_rewrite/file/Makefile.am new file mode 100644 index 0000000000..0a280a0315 --- /dev/null +++ b/helpers/storeid_rewrite/file/Makefile.am @@ -0,0 +1,15 @@ +include $(top_srcdir)/src/Common.am + +libexec_SCRIPTS = storeid_file_rewrite +CLEANFILES += storeid_file_rewrite storeid_file_rewrite.8 +man_MANS = storeid_file_rewrite.8 +EXTRA_DIST= \ + storeid_file_rewrite.8 \ + storeid_file_rewrite.pl.in \ + required.m4 + +storeid_file_rewrite.8: storeid_file_rewrite + pod2man storeid_file_rewrite storeid_file_rewrite.8 + +storeid_file_rewrite: storeid_file_rewrite.pl.in + $(subst_perlshell) diff --git a/helpers/storeid_rewrite/file/required.m4 b/helpers/storeid_rewrite/file/required.m4 new file mode 100755 index 0000000000..72c3e3e426 --- /dev/null +++ b/helpers/storeid_rewrite/file/required.m4 @@ -0,0 +1,3 @@ +if test "x$PERL" != "x" -a "x$POD2MAN" != "x"; then + BUILD_HELPER="file" +fi diff --git a/helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in b/helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in new file mode 100644 index 0000000000..342858cd17 --- /dev/null +++ b/helpers/storeid_rewrite/file/storeid_file_rewrite.pl.in @@ -0,0 +1,92 @@ +#!@PERL@ +use strict; +use warnings; +$|=1; + +=pod + +=head1 NAME + +storeid_file_rewrite - File based Store-ID helper for Squid + +=head1 SYNOPSIS + +storeid_file_rewrite filepath + +=head1 DESCRIPTOIN + +This program acts as a store_id helper program, rewriting URLs passed +by Squid into storage-ids that can be used to achieve better caching +for websites that use different URLs for the same content. + +It takes a text file with two tab separated columns. +Column 1: Regular expression to match against the URL +Column 2: Rewrite rule to generate a Store-ID +Eg: +^http:\/\/[^\.]+\.dl\.sourceforge\.net\/(.*) http://dl.sourceforge.net.squid.internal/$1 + +Rewrite rules are matched in the same order as they appear in the rules file. +So for best performance, sort it in order of frequency of occurrence. + +For more information please see http://wiki.squid-cache.org/Features/StoreID + +=cut + +my @rules; # array of [regex, replacement string] + +die "Usage: $0 \n" unless $#ARGV == 0; + +# read config file +open RULES, $ARGV[0] or die "Error opening $ARGV[0]: $!"; +while () { + chomp; + next if /^\s*#?$/; + if (/^\s*([^\t]+?)\s*\t+\s*([^\t]+?)\s*$/) { + push(@rules, [qr/$1/, $2]); + } else { + print STDERR "$0: Parse error in $ARGV[0] (line $.)\n"; + } +} +close RULES; + +# read urls from squid and do the replacement +URL: while () { + chomp; + last if $_ eq 'quit'; + + foreach my $rule (@rules) { + if (my @match = /$rule->[0]/) { + $_ = $rule->[1]; + + for (my $i=1; $i<=scalar(@match); $i++) { + s/\$$i/$match[$i-1]/g; + } + print "OK store-id=$_\n"; + next URL; + } + } + print "ERR\n"; +} + +=pod + +=head1 COPYRIGHT + +Copyright (C) 2013 Alan Mizrahi +Based on code from Eliezer Croitoru + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + +=cut diff --git a/helpers/storeid_rewrite/modules.m4 b/helpers/storeid_rewrite/modules.m4 new file mode 100644 index 0000000000..755e1dd573 --- /dev/null +++ b/helpers/storeid_rewrite/modules.m4 @@ -0,0 +1,40 @@ +# This file is supposed to run all the tests required to identify which +# configured modules are able to be built in this environment + +# FIXME: de-duplicate $enable_storeid_rewrite_helpers list containing double entries. + +#define list of modules to build +if test "x${enable_storeid_rewrite_helpers:=yes}" = "xyes" ; then + SQUID_LOOK_FOR_MODULES([$srcdir/helpers/storeid_rewrite],[enable_storeid_rewrite_helpers]) +fi + +enable_storeid_rewrite_helpers="`echo $enable_storeid_rewrite_helpers| sed -e 's/,/ /g;s/ */ /g'`" +AC_MSG_NOTICE([Store-ID rewrite helper candidates: $enable_storeid_rewrite_helpers]) +STOREID_REWRITE_HELPERS="" +if test "x$enable_storeid_rewrite_helpers" != "xno" ; then + for helper in $enable_storeid_rewrite_helpers; do + dir="$srcdir/helpers/storeid_rewrite/$helper" + + # modules converted to autoconf macros already + # NP: we only need this list because m4_include() does not accept variables + if test "x$helper" = "xfile" ; then + m4_include([helpers/storeid_rewrite/file/required.m4]) + + # modules not yet converted to autoconf macros (or third party drop-in's) + elif test -f "$dir/config.test" && sh "$dir/config.test" "$squid_host_os"; then + BUILD_HELPER="$helper" + fi + + if test -d "$srcdir/helpers/storeid_rewrite/$helper"; then + if test "$BUILD_HELPER" != "$helper"; then + AC_MSG_NOTICE([Store-ID rewrite helper $helper ... found but cannot be built]) + else + STOREID_REWRITE_HELPERS="$STOREID_REWRITE_HELPERS $BUILD_HELPER" + fi + else + AC_MSG_ERROR([Store-ID rewrite helper $helper ... not found]) + fi + done +fi +AC_MSG_NOTICE([Store-ID rewrite helpers to be built: $STOREID_REWRITE_HELPERS]) +AC_SUBST(STOREID_REWRITE_HELPERS) -- 2.47.3