From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 4 Jun 2008 18:51:03 +0000 (+0000)
Subject: Renewed downloadscript.
X-Git-Tag: v3.0-alpha1~983
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=fa0dac0fdcea79060a6350d5f0a14abb8fc52900;p=ipfire-3.x.git

Renewed downloadscript.

For this we need to touch every lfs/-script. :(
---

diff --git a/lfs/Config b/lfs/Config
index 614a5b2c2..17af2eb70 100644
--- a/lfs/Config
+++ b/lfs/Config
@@ -33,7 +33,7 @@
 # URLs that are common sources of downloads.  If you're having trouble with
 # a site you should change its URL to that of a suitable mirror site.
 #
-URL_IPFIRE  = http://source.ipfire.org/source-3.0
+URL_IPFIRE  = http://source.ipfire.org/download
 URL_TOOLCHAIN = http://source.ipfire.org/toolchains
 
 # Default compiler optimizations.
@@ -56,15 +56,17 @@ DIR_SRC = $(LFS)/usr/src
 # save all the files that are downloaded.  DIR_INFO contains the
 # file lists of installed packages.
 #
-DIR_DL      = $(LFS_BASEDIR)/cache
+DIR_DL      = $(LFS_BASEDIR)/cache/tarballs
 DIR_CONF    = $(LFS_BASEDIR)/config
 DIR_INFO    = $(LFS_BASEDIR)/log_$(MACHINE)
 DIR_TOOLS   = $(LFS_BASEDIR)/tools
 DIR_TMP     = /tmp
-DIR_PATCHES = $(DIR_DL)
+DIR_PATCHES = $(DIR_DL)/../patches
 DIR_CONFIG  = $(DIR_CONF)
 DIR_SOURCE  = $(DIR_SRC)/src
 
+VPATH = $(DIR_DL):$(DIR_PATCHES)
+
 HOST_GCC="/usr/bin/gcc"
 KGCC=$(HOST_GCC)
 
@@ -186,8 +188,8 @@ endef
 endif
 
 define LOAD
-	BASEDIR=$(LFS_BASEDIR) DIR_TMP=$(DIR_TMP) DIR_DL=$(DIR_DL) \
-		sh $(DIR_TOOLS)/downloader $(URL_IPFIRE) $(notdir $@)
+	DIR_TMP=$(DIR_TMP) DIR_DL=$(DIR_DL) DIR_PATCHES=$(DIR_PATCHES) \
+		python $(DIR_TOOLS)/downloader-ng $(URL_IPFIRE)/$(notdir $@)
 endef
 
 define PAK
diff --git a/tools/downloader b/tools/downloader
index 15f62aa97..9bcf3eb25 100644
--- a/tools/downloader
+++ b/tools/downloader
@@ -1,8 +1,8 @@
-#!/bin/bash
+#!/usr/bin/python
 ###############################################################################
 #                                                                             #
 # IPFire.org - A linux based firewall                                         #
-# Copyright (C) 2007  Michael Tremer & Christian Schmidt                      #
+# Copyright (C) 2008  Michael Tremer & Christian Schmidt                      #
 #                                                                             #
 # This program is free software: you can redistribute it and/or modify        #
 # it under the terms of the GNU General Public License as published by        #
@@ -25,33 +25,119 @@
 #                                                                             #
 ###############################################################################
 
-# BASEDIR, DIR_TMP, DIR_DL are given by the environment
-URL="$1"
-FILE="$(basename $2)"
-
-MAX=2
-COUNTER=0
-
-WGET_OPTS="-c"
-
-while [ "${COUNTER}" -lt "${MAX}" ]; do
-	cd ${DIR_TMP}
-	if wget ${WGET_OPTS} ${URL}/${FILE}{,.md5}; then
-		if [ `awk '{ print $1 }' ${DIR_TMP}/${FILE}.md5` = \
-				`md5sum ${DIR_TMP}/${FILE} | awk '{ print $1 }'` ]; then
-			mv ${DIR_TMP}/${FILE} ${DIR_TMP}/${FILE}.md5 ${DIR_DL}
-			exit 0
-		else
-			rm -f ${FILE}* 2>/dev/null
-		fi
-	else
-		rm -f ${FILE}* 2>/dev/null
-		exit 1
-	fi
-	COUNTER=$(( ${COUNTER} + 1 ))
-done
-
-if [ "${COUNTER}" -eq "${MAX}" ]; then
-	echo "Download failed." 1>&2
-	exit 1
-fi
+import os
+import sys
+
+import hashlib
+import shutil
+
+from urlgrabber.grabber import URLGrabber, URLGrabError
+import urlgrabber.progress as progress
+
+###############################################################################
+# Sample header                                                               #
+###############################################################################
+#X-Object: FileObject
+#X-MD5: d79f553e7916ea21c556329eacfeaa16
+#X-SHA1: bb20efc7750fe0d6172c5945572bf036fe59d3dd
+
+###############################################################################
+# Environment                                                                 #
+###############################################################################
+
+BSIZE = 1024	#1k
+gobj = None
+
+try:
+	DIR_DL = os.environ["DIR_DL"] or os.getcwd()
+	DIR_PATCHES = os.environ["DIR_PATCHES"] or os.getcwd()
+	DIR_TMP = os.environ["DIR_TMP"] or None
+except KeyError:
+	sys.stderr.write("DIR_TMP, DIR_DL and DIR_PATCHES have to be set "
+			 "in environment.\n")
+	sys.exit(1)
+
+url = sys.argv[1]
+filename = os.path.basename(url)
+
+#if os.access(os.path.join(DIR_PATCHES), filename), R_OK) or
+#	os.access(os.path.join(DIR_DL), filename), R_OK):
+#	sys.stdout.write("Already downloaded: %s\n" % (filename,))	
+#	sys.exit(0)
+
+#sys.stdout.write("Downloading %s\n" % (url,))
+
+###############################################################################
+# Create urlgrabber instance                                                  #
+###############################################################################
+g = URLGrabber( user_agent = "IPFireSourceGrabber/3.x",
+		progress_obj = progress.TextMeter(fo=sys.stdout) )
+try:
+	gobj = g.urlopen(url)
+except URLGrabError, e:
+	if e.errno == 14: # 404
+		sys.stderr.write("404 - File not found.\n")
+		sys.exit(1)
+
+if not gobj:
+	sys.stderr.write("Unknown happended: %s\n" % (url,))
+	sys.exit(1)
+
+# Init hash
+hobj = hashlib.sha1()
+
+###############################################################################
+# Parse header                                                                #
+###############################################################################
+try:
+	object_type = gobj.hdr["X-Object"]
+
+	#hash_md5 = gobj.hdr["X-MD5"]
+	hash_sha1 = gobj.hdr["X-SHA1"]
+	
+	hash = hash_sha1
+except KeyError:
+	sys.stderr.write("No header data found. Can't check file.\n")
+	#for line in gobj.hdr.items(): sys.stderr.write("  %s\n" % (line,))
+	sys.exit(1)
+
+###############################################################################
+# Setup fileobjects                                                           #
+###############################################################################
+if object_type == "FileObject":
+	dest_file = os.path.join(DIR_DL,filename)
+elif object_type == "PatchObject":
+	dest_file = os.path.join(DIR_PATCHES,filename)
+
+if DIR_TMP:
+	temp_file = os.path.join(DIR_TMP, filename)
+else:
+	temp_file = dest_file
+
+fobj = open(temp_file, "wb")
+
+###############################################################################
+# Download                                                                    #
+###############################################################################
+buf = gobj.read(BSIZE)
+while len(buf) > 0:
+	hobj.update(buf)
+	fobj.write(buf)
+	buf = gobj.read(BSIZE)
+
+gobj.close() # Close connection to server.
+fobj.close() # Also close local file.
+
+###############################################################################
+# Check hash                                                                  #
+###############################################################################
+if hash == hobj.hexdigest():
+	shutil.move(temp_file, dest_file)
+else:
+	os.unlink(temp_file)
+	sys.stderr.write("Hash mismatch: %s\n" % (filename,))
+	sys.exit(1)
+
+sys.exit(0)
+
+###############################################################################
diff --git a/tools/make-interactive b/tools/make-interactive
index 89875a21b..f730f858e 100644
--- a/tools/make-interactive
+++ b/tools/make-interactive
@@ -99,22 +99,20 @@ clean)
 	;;
 	
 downloadsrc)
-	LOGFILE=$BASEDIR/log_${MACHINE}/_build.preparation.log
+	LOGFILE=$BASEDIR/log_${MACHINE}/_build.00-preparation.log
 	
 	if [ ! -d $BASEDIR/cache ]; then
-		mkdir $BASEDIR/cache
+		mkdir -p $BASEDIR/cache/{tarballs,patches}
 	fi
 	mkdir -p $BASEDIR/log_${MACHINE}
-	echo -e "${BOLD}Preload all source files${NORMAL}" | tee -a $LOGFILE
+	echo -e "${BOLD}Preload all source files${NORMAL}"
 	cd $BASEDIR/lfs
 	for i in *; do
 		if [ -f "$i" -a "$i" != "Config" ]; then
-			echo -ne "Loading $i"
-			make -s -f $i LFS_BASEDIR=$BASEDIR MESSAGE="$i\t" download >> $LOGFILE 2>&1
+			make -s -f $i LFS_BASEDIR=$BASEDIR MESSAGE="$i\t" download 2>> $LOGFILE
 			if [ $? -ne 0 ]; then
+				echo -ne "Couldn't download all files for ${BOLD}${i}${NORMAL}"
 				beautify message FAIL
-			else
-				beautify message DONE
 			fi
 		fi
 	done