Renewed downloadscript.

For this we need to touch every lfs/-script. :(

diff --git a/lfs/Config b/lfs/Config
index 614a5b2c26d4aa084bd3d28bb4843cc5207e7956..17af2eb70639080a76d50904d5a2cc3e4ba09c3a 100644 (file)
--- a/lfs/Config
+++ b/lfs/Config
@@ -33,7 +33,7 @@
 # URLs that are common sources of downloads.  If you're having trouble with
 # a site you should change its URL to that of a suitable mirror site.
 #
-URL_IPFIRE  = http://source.ipfire.org/source-3.0
+URL_IPFIRE  = http://source.ipfire.org/download
 URL_TOOLCHAIN = http://source.ipfire.org/toolchains
 
 # Default compiler optimizations.
@@ -56,15 +56,17 @@ DIR_SRC = $(LFS)/usr/src
 # save all the files that are downloaded.  DIR_INFO contains the
 # file lists of installed packages.
 #
-DIR_DL      = $(LFS_BASEDIR)/cache
+DIR_DL      = $(LFS_BASEDIR)/cache/tarballs
 DIR_CONF    = $(LFS_BASEDIR)/config
 DIR_INFO    = $(LFS_BASEDIR)/log_$(MACHINE)
 DIR_TOOLS   = $(LFS_BASEDIR)/tools
 DIR_TMP     = /tmp
-DIR_PATCHES = $(DIR_DL)
+DIR_PATCHES = $(DIR_DL)/../patches
 DIR_CONFIG  = $(DIR_CONF)
 DIR_SOURCE  = $(DIR_SRC)/src
 
+VPATH = $(DIR_DL):$(DIR_PATCHES)
+
 HOST_GCC="/usr/bin/gcc"
 KGCC=$(HOST_GCC)
 
@@ -186,8 +188,8 @@ endef
 endif
 
 define LOAD
-       BASEDIR=$(LFS_BASEDIR) DIR_TMP=$(DIR_TMP) DIR_DL=$(DIR_DL) \
-               sh $(DIR_TOOLS)/downloader $(URL_IPFIRE) $(notdir $@)
+       DIR_TMP=$(DIR_TMP) DIR_DL=$(DIR_DL) DIR_PATCHES=$(DIR_PATCHES) \
+               python $(DIR_TOOLS)/downloader-ng $(URL_IPFIRE)/$(notdir $@)
 endef
 
 define PAK

diff --git a/tools/downloader b/tools/downloader
index 15f62aa979ac5921d935a646482588127656fded..9bcf3eb2531dc4284fe1dd3449073725881eb238 100644 (file)
--- a/tools/downloader
+++ b/tools/downloader
@@ -1,8 +1,8 @@
-#!/bin/bash
+#!/usr/bin/python
 ###############################################################################
 #                                                                             #
 # IPFire.org - A linux based firewall                                         #
-# Copyright (C) 2007  Michael Tremer & Christian Schmidt                      #
+# Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
 #                                                                             #
 # This program is free software: you can redistribute it and/or modify        #
 # it under the terms of the GNU General Public License as published by        #
@@ -25,33 +25,119 @@
 #                                                                             #
 ###############################################################################
 
-# BASEDIR, DIR_TMP, DIR_DL are given by the environment
-URL="$1"
-FILE="$(basename $2)"
-
-MAX=2
-COUNTER=0
-
-WGET_OPTS="-c"
-
-while [ "${COUNTER}" -lt "${MAX}" ]; do
-       cd ${DIR_TMP}
-       if wget ${WGET_OPTS} ${URL}/${FILE}{,.md5}; then
-               if [ `awk '{ print $1 }' ${DIR_TMP}/${FILE}.md5` = \
-                               `md5sum ${DIR_TMP}/${FILE} | awk '{ print $1 }'` ]; then
-                       mv ${DIR_TMP}/${FILE} ${DIR_TMP}/${FILE}.md5 ${DIR_DL}
-                       exit 0
-               else
-                       rm -f ${FILE}* 2>/dev/null
-               fi
-       else
-               rm -f ${FILE}* 2>/dev/null
-               exit 1
-       fi
-       COUNTER=$(( ${COUNTER} + 1 ))
-done
-
-if [ "${COUNTER}" -eq "${MAX}" ]; then
-       echo "Download failed." 1>&2
-       exit 1
-fi
+import os
+import sys
+
+import hashlib
+import shutil
+
+from urlgrabber.grabber import URLGrabber, URLGrabError
+import urlgrabber.progress as progress
+
+###############################################################################
+# Sample header                                                               #
+###############################################################################
+#X-Object: FileObject
+#X-MD5: d79f553e7916ea21c556329eacfeaa16
+#X-SHA1: bb20efc7750fe0d6172c5945572bf036fe59d3dd
+
+###############################################################################
+# Environment                                                                 #
+###############################################################################
+
+BSIZE = 1024   #1k
+gobj = None
+
+try:
+       DIR_DL = os.environ["DIR_DL"] or os.getcwd()
+       DIR_PATCHES = os.environ["DIR_PATCHES"] or os.getcwd()
+       DIR_TMP = os.environ["DIR_TMP"] or None
+except KeyError:
+       sys.stderr.write("DIR_TMP, DIR_DL and DIR_PATCHES have to be set "
+                        "in environment.\n")
+       sys.exit(1)
+
+url = sys.argv[1]
+filename = os.path.basename(url)
+
+#if os.access(os.path.join(DIR_PATCHES), filename), R_OK) or
+#      os.access(os.path.join(DIR_DL), filename), R_OK):
+#      sys.stdout.write("Already downloaded: %s\n" % (filename,))      
+#      sys.exit(0)
+
+#sys.stdout.write("Downloading %s\n" % (url,))
+
+###############################################################################
+# Create urlgrabber instance                                                  #
+###############################################################################
+g = URLGrabber( user_agent = "IPFireSourceGrabber/3.x",
+               progress_obj = progress.TextMeter(fo=sys.stdout) )
+try:
+       gobj = g.urlopen(url)
+except URLGrabError, e:
+       if e.errno == 14: # 404
+               sys.stderr.write("404 - File not found.\n")
+               sys.exit(1)
+
+if not gobj:
+       sys.stderr.write("Unknown happended: %s\n" % (url,))
+       sys.exit(1)
+
+# Init hash
+hobj = hashlib.sha1()
+
+###############################################################################
+# Parse header                                                                #
+###############################################################################
+try:
+       object_type = gobj.hdr["X-Object"]
+
+       #hash_md5 = gobj.hdr["X-MD5"]
+       hash_sha1 = gobj.hdr["X-SHA1"]
+       
+       hash = hash_sha1
+except KeyError:
+       sys.stderr.write("No header data found. Can't check file.\n")
+       #for line in gobj.hdr.items(): sys.stderr.write("  %s\n" % (line,))
+       sys.exit(1)
+
+###############################################################################
+# Setup fileobjects                                                           #
+###############################################################################
+if object_type == "FileObject":
+       dest_file = os.path.join(DIR_DL,filename)
+elif object_type == "PatchObject":
+       dest_file = os.path.join(DIR_PATCHES,filename)
+
+if DIR_TMP:
+       temp_file = os.path.join(DIR_TMP, filename)
+else:
+       temp_file = dest_file
+
+fobj = open(temp_file, "wb")
+
+###############################################################################
+# Download                                                                    #
+###############################################################################
+buf = gobj.read(BSIZE)
+while len(buf) > 0:
+       hobj.update(buf)
+       fobj.write(buf)
+       buf = gobj.read(BSIZE)
+
+gobj.close() # Close connection to server.
+fobj.close() # Also close local file.
+
+###############################################################################
+# Check hash                                                                  #
+###############################################################################
+if hash == hobj.hexdigest():
+       shutil.move(temp_file, dest_file)
+else:
+       os.unlink(temp_file)
+       sys.stderr.write("Hash mismatch: %s\n" % (filename,))
+       sys.exit(1)
+
+sys.exit(0)
+
+###############################################################################

diff --git a/tools/make-interactive b/tools/make-interactive
index 89875a21b51945e78518502b1a5d0da8ee928c67..f730f858eb3f339c31fb4b5b2964309a3678122c 100644 (file)
--- a/tools/make-interactive
+++ b/tools/make-interactive
@@ -99,22 +99,20 @@ clean)
        ;;
        
 downloadsrc)
-       LOGFILE=$BASEDIR/log_${MACHINE}/_build.preparation.log
+       LOGFILE=$BASEDIR/log_${MACHINE}/_build.00-preparation.log
        
        if [ ! -d $BASEDIR/cache ]; then
-               mkdir $BASEDIR/cache
+               mkdir -p $BASEDIR/cache/{tarballs,patches}
        fi
        mkdir -p $BASEDIR/log_${MACHINE}
-       echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE
+       echo -e "${BOLD}Preload all source files${NORMAL}"
        cd $BASEDIR/lfs
        for i in *; do
                if [ -f "$i" -a "$i" != "Config" ]; then
-                       echo -ne "Loading $i"
-                       make -s -f $i LFS_BASEDIR=$BASEDIR MESSAGE="$i\t" download >> $LOGFILE 2>&1
+                       make -s -f $i LFS_BASEDIR=$BASEDIR MESSAGE="$i\t" download 2>> $LOGFILE
                        if [ $? -ne 0 ]; then
+                               echo -ne "Couldn't download all files for ${BOLD}${i}${NORMAL}"
                                beautify message FAIL
-                       else
-                               beautify message DONE
                        fi
                fi
        done