pjproject_bundled: Improve reliability of pjproject download

The download process now has a timeout which will cause wget to retry
if it stops retrieving data for 5 seconds and fetch and curl to timeout
if the whole retrieval take smore than 30 seconds.

If the tarball retrieval works, the MD5SUM file is retrieved from
the downloads site and the md5 checksum is verified.

If either the tarball retrieval or MD5SUM retrieval fails, or the
checksums don't match, the entire process is retried once.  If it
fails again, any incomplete tarball is deleted.

.DELETE_ON_ERROR: was also added to the Makefile.  Not only does
this delete the tarball on failure, it till also delete corrupted
library files from the pjproject source directory should they
fail to build correctly.

Tested all the way back to FreeBSD 9, CentOS 6, Debian 6 and
Ubuntu 14.

Change-Id: Iea7d33b96a31622ab1b6e54baebaf271959514e1

diff --git a/configure b/configure
index 0446c4571884e578b46dcfd1c40f022c8f84afc3..96f095983acf782fe606f5cc6366f4ea7e85b630 100755 (executable)
--- a/configure
+++ b/configure
@@ -1195,6 +1195,7 @@ PBX_BISON
 OPENSSL
 SHA1SUM
 LDCONFIG
+DOWNLOAD_TIMEOUT
 DOWNLOAD_TO_STDOUT
 DOWNLOAD
 FETCH
@@ -1221,6 +1222,8 @@ COMPRESS
 FIND
 PYTHON
 FLEX
+CUT
+CAT
 CMP
 BISON
 GNU_LD
@@ -6747,6 +6750,88 @@ $as_echo "no" >&6; }
 fi
 
 
+# Extract the first word of "cat", so it can be a program name with args.
+set dummy cat; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_CAT+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $CAT in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_CAT="$CAT" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_CAT="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_path_CAT" && ac_cv_path_CAT=":"
+  ;;
+esac
+fi
+CAT=$ac_cv_path_CAT
+if test -n "$CAT"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CAT" >&5
+$as_echo "$CAT" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+# Extract the first word of "cut", so it can be a program name with args.
+set dummy cut; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_CUT+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $CUT in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_CUT="$CUT" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_CUT="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_path_CUT" && ac_cv_path_CUT=":"
+  ;;
+esac
+fi
+CUT=$ac_cv_path_CUT
+if test -n "$CUT"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CUT" >&5
+$as_echo "$CUT" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
 # Extract the first word of "flex", so it can be a program name with args.
 set dummy flex; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -7817,9 +7902,11 @@ fi
 if test "${WGET}" != ":" ; then
   DOWNLOAD=${WGET}
   DOWNLOAD_TO_STDOUT="${WGET} -O-"
+  DOWNLOAD_TIMEOUT='--timeout=$1'
 else if test "${CURL}" != ":" ; then
   DOWNLOAD="${CURL} -O --progress-bar -w \"%{url_effective}\n\""
   DOWNLOAD_TO_STDOUT="${CURL} -L --progress-bar -w \"%{url_effective}\n\""
+  DOWNLOAD_TIMEOUT='--max-time $(or $2,$1)'
 else
   # Extract the first word of "fetch", so it can be a program name with args.
 set dummy fetch; ac_word=$2
@@ -7864,11 +7951,14 @@ fi
 
   DOWNLOAD=${FETCH}
   DOWNLOAD_TO_STDOUT="${FETCH} -o-"
+  DOWNLOAD_TIMEOUT='--timeout=$(or $2,$1)'
 fi
 fi
 
 
 
+
+
 # Extract the first word of "ldconfig", so it can be a program name with args.
 set dummy ldconfig; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -9190,8 +9280,14 @@ $as_echo "configuring" >&6; }
        if test "${NM}" = ":" ; then
                as_fn_error $? "nm is required to build bundled pjproject" "$LINENO" 5
        fi
+       if test "${MD5}" = ":" ; then
+               as_fn_error $? "md5dum is required to build bundled pjproject" "$LINENO" 5
+       fi
+       if test "${CAT}" = ":" ; then
+               as_fn_error $? "cat is required to build bundled pjproject" "$LINENO" 5
+       fi
 
-       export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT
+       export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT DOWNLOAD_TIMEOUT DOWNLOAD MD5 CAT
        ${GNU_MAKE} --quiet --no-print-directory -C ${PJPROJECT_DIR} EXTERNALS_CACHE_DIR=${EXTERNALS_CACHE_DIR} configure
        if test $? -ne 0 ; then
                { $as_echo "$as_me:${as_lineno-$LINENO}: result: failed" >&5

diff --git a/configure.ac b/configure.ac
index e4a20cfa5a0b5baa0787ab3959a8c6586fd9e356..ec7536a77c42543683d2c43c16f5ab5593968747 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -264,6 +264,7 @@ AC_SUBST(GNU_LD)
 
 AC_PATH_PROG([BISON], [bison], :)
 AC_PATH_PROG([CMP], [cmp], :)
+AC_PATH_PROG([CAT], [cat], :)
 AC_PATH_PROG([FLEX], [flex], :)
 AC_PATH_PROG([GREP], [grep], :)
 AC_PATH_PROG([PYTHON], [python], :)
@@ -294,18 +295,23 @@ AC_PATH_PROG([NM], [nm], :)
 if test "${WGET}" != ":" ; then
   DOWNLOAD=${WGET}
   DOWNLOAD_TO_STDOUT="${WGET} -O-"
+  DOWNLOAD_TIMEOUT='--timeout=$1'
 else if test "${CURL}" != ":" ; then
   DOWNLOAD="${CURL} -O --progress-bar -w \"%{url_effective}\n\""
   DOWNLOAD_TO_STDOUT="${CURL} -L --progress-bar -w \"%{url_effective}\n\""
+  DOWNLOAD_TIMEOUT='--max-time $(or $2,$1)'
 else
   AC_PATH_PROG([FETCH], [fetch], [:])
   DOWNLOAD=${FETCH}
   DOWNLOAD_TO_STDOUT="${FETCH} -o-"
+  DOWNLOAD_TIMEOUT='--timeout=$(or $2,$1)'
 fi
 fi
 
 AC_SUBST(DOWNLOAD)
 AC_SUBST(DOWNLOAD_TO_STDOUT)
+AC_SUBST(DOWNLOAD_TIMEOUT)
+
 AC_PATH_PROG([LDCONFIG], [ldconfig], :)
 AC_PATH_PROG([SHA1SUM], [sha1sum], $ac_aux_dir/build_tools/sha1sum-sh)
 AC_PATH_PROG([OPENSSL], [openssl], :)

diff --git a/makeopts.in b/makeopts.in
index fed5030e8e717e99456bb7ab15011d78a82b1eaf..e5ab0be96a0b0d9031e0a1d01510a40fa9c8f697 100644 (file)
--- a/makeopts.in
+++ b/makeopts.in
@@ -28,6 +28,7 @@ WGET=@WGET@
 FETCH=@FETCH@
 DOWNLOAD=@DOWNLOAD@
 DOWNLOAD_TO_STDOUT=@DOWNLOAD_TO_STDOUT@
+DOWNLOAD_MAX_TIMEOUT=@DOWNLOAD_MAX_TIMEOUT@
 SOUNDS_CACHE_DIR=@SOUNDS_CACHE_DIR@
 EXTERNALS_CACHE_DIR=@EXTERNALS_CACHE_DIR@
 RUBBER=@RUBBER@
@@ -46,6 +47,7 @@ TAR=@TAR@
 PATCH=@PATCH@
 SED=@SED@
 NM=@NM@
+CAT=@CAT@
 
 BUILD_PLATFORM=@BUILD_PLATFORM@
 BUILD_CPU=@BUILD_CPU@

diff --git a/third-party/Makefile.rules b/third-party/Makefile.rules
index 4f804dd0e3cd67ceca21e5691b1a39462293c7e2..f8b72ba6ddb4704d00697344cc34ed4c0e422a5b 100644 (file)
--- a/third-party/Makefile.rules
+++ b/third-party/Makefile.rules
@@ -26,4 +26,8 @@ export TAR
 export PATCH
 export SED
 export NM
+export MD5
+export CAT
 export DOWNLOAD
+export DOWNLOAD_TO_STDOUT
+export DOWNLOAD_TIMEOUT

diff --git a/third-party/pjproject/Makefile b/third-party/pjproject/Makefile
index 106938b6cc9ea48a8a6f2e8269fe1fec8bff2526..ff3266c16fe2eba58506942a3a3971cd18858d4e 100644 (file)
--- a/third-party/pjproject/Makefile
+++ b/third-party/pjproject/Makefile
@@ -75,14 +75,31 @@ include ../Makefile.rules
 include Makefile.rules
 
 ECHO_PREFIX := $(ECHO_PREFIX) echo '[pjproject] '
+SHELL_ECHO_PREFIX := echo '[pjproject] '
 
 _all: $(TARGETS)
 
+define download_from_pjproject
+       ($(SHELL_ECHO_PREFIX) Downloading $(PJPROJECT_URL)/$(@F) to $@ ;\
+       $(DOWNLOAD_TO_STDOUT) $(call DOWNLOAD_TIMEOUT,5,10) $(PJPROJECT_URL)/$(@F) > $@ &&\
+       $(SHELL_ECHO_PREFIX) Downloading $(PJPROJECT_URL)/MD5SUM to $(PJMD5SUM) &&\
+       $(DOWNLOAD_TO_STDOUT) $(call DOWNLOAD_TIMEOUT,5,10) $(PJPROJECT_URL)/MD5SUM.TXT > $(PJMD5SUM) &&\
+       ($(SHELL_ECHO_PREFIX) Verifying $@ &&\
+       tarball_sum=$$($(CAT) $@ | $(MD5) | $(SED) -n -r -e "s/^([^ ]+)\s+.*/\1/gp") ;\
+       required_sum=$$($(SED) -n -r -e "s/^([^ ]+)\s+$(@F)/\1/gp" $(PJMD5SUM)) ;\
+       if [ "$$tarball_sum" != "$$required_sum" ] ; then $(SHELL_ECHO_PREFIX) Verify failed ; exit 1 ; fi) &&\
+       $(SHELL_ECHO_PREFIX) Verify successful ; exit 0)
+endef
+
+.DELETE_ON_ERROR:
+
 DOWNLOAD_DIR := $(or $(EXTERNALS_CACHE_DIR),$(TMPDIR),$(wildcard /tmp),.)
+TARBALL = $(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2
+PJMD5SUM = $(patsubst %.tar.bz2,%.md5,$(TARBALL))
 
-$(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2: ../versions.mak
-       $(ECHO_PREFIX) Downloading $(PJPROJECT_URL)/$(@F) to $@
-       $(CMD_PREFIX) $(DOWNLOAD_TO_STDOUT) $(PJPROJECT_URL)/$(@F) > $@
+$(TARBALL): ../versions.mak
+       $(CMD_PREFIX) $(download_from_pjproject) || (rm -rf $@ ;\
+       $(SHELL_ECHO_PREFIX) Retrying download ; sleep 3 ; $(download_from_pjproject))
 
 source/.unpacked: $(DOWNLOAD_DIR)/pjproject-$(PJPROJECT_VERSION).tar.bz2
        $(ECHO_PREFIX) Unpacking $<

diff --git a/third-party/pjproject/configure.m4 b/third-party/pjproject/configure.m4
index 870468257b3bca1e112ddef8fce99d049ad8102a..be03b45b533dd67fe69925871baf5c5bacf8cf9f 100644 (file)
--- a/third-party/pjproject/configure.m4
+++ b/third-party/pjproject/configure.m4
@@ -28,8 +28,14 @@ AC_DEFUN([_PJPROJECT_CONFIGURE],
        if test "${NM}" = ":" ; then
                AC_MSG_ERROR(nm is required to build bundled pjproject)
        fi
+       if test "${MD5}" = ":" ; then
+               AC_MSG_ERROR(md5sum is required to build bundled pjproject)
+       fi
+       if test "${CAT}" = ":" ; then
+               AC_MSG_ERROR(cat is required to build bundled pjproject)
+       fi
 
-       export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT
+       export TAR PATCH SED NM EXTERNALS_CACHE_DIR DOWNLOAD_TO_STDOUT DOWNLOAD_TIMEOUT DOWNLOAD MD5 CAT
        ${GNU_MAKE} --quiet --no-print-directory -C ${PJPROJECT_DIR} EXTERNALS_CACHE_DIR=${EXTERNALS_CACHE_DIR} configure
        if test $? -ne 0 ; then
                AC_MSG_RESULT(failed)