#!/bin/bash ############################################################################### # # # IPFire.org - A linux based firewall # # Copyright (C) 2008 Michael Tremer & Christian Schmidt # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # # the Free Software Foundation, either version 3 of the License, or # # (at your option) any later version. # # # # This program is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # # GNU General Public License for more details. # # # # You should have received a copy of the GNU General Public License # # along with this program. If not, see . # # # ############################################################################### ############################################################################### # # System variables # ############################################################################### PWD=$(pwd) BASENAME=$(basename $0) # Debian specific settings if [ ! -e /etc/debian_version ]; then FULLPATH=$(which $0) else if [ -x /usr/bin/realpath ]; then FULLPATH=$(/usr/bin/realpath $0) else echo "ERROR: Need to do apt-get install realpath" exit 1 fi fi BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g") export BASEDIR . $BASEDIR/tools/make-constants # Load this very early . $BASEDIR/tools/make-beautify mkdir $BASEDIR/log_${TARGET}/ 2>/dev/null ################################################################################ # # # Necessary shell functions # # # ################################################################################ . $BASEDIR/tools/make-buildspy . $BASEDIR/tools/make-check . $BASEDIR/tools/make-batch . $BASEDIR/tools/make-compilers . $BASEDIR/tools/make-git . $BASEDIR/tools/make-packages . $BASEDIR/tools/make-rootfiles . $BASEDIR/tools/make-vm . $BASEDIR/tools/make-cron evaluate() { RETVAL=$? if [ "$RETVAL" -eq "0" ]; then beautify message DONE else beautify message FAIL fi (exit $RETVAL) } stdumount() { sleep 0.3 # Wait one second for finish of processes for fs in $(mount | grep $BASEDIR/build_${TARGET} | awk '{print $3}'); do umount $fs 2>/dev/null; done } # End of stdumount() exiterror() { stdumount dialogerror $* build_spy state error if [ -n "$LOGFILE" ]; then echo "$(date -u '+%b %e %T') ERROR: $*" >> $LOGFILE build_spy log $(tail -n16 $LOGFILE | $BASEDIR/tools/base64) fi build_spy exit touch $FAILED rm -f $RUNNING 2>/dev/null exit 1 } # End of exiterror() logger() { local logfile=$LOGFILE case "$1" in --distcc) logfile=$BASEDIR/log_$TARGET/_build.00-distcc.log shift ;; esac echo "$(date -u '+%b %e %T') $*" >> $logfile } ################################################################################ # This is the function that sets the environment of a chroot and enters it # ################################################################################ entershell() { PATH=${TOOLS_DIR}/usr/bin:/bin:/usr/bin:/sbin:/usr/sbin:${TOOLS_DIR}/bin:/usr/${MACHINE_REAL}-linux/bin if [ ! -e $LFS/usr/src/lfs/ ]; then exiterror "No such file or directory: $LFS/usr/src/lfs/" fi COMMAND=${@-bash} # Run command, given as parameters. echo -ne "Entering ${BOLD}$MACHINE${NORMAL} LFS chroot, type exit to return to host environment\n" chroot $LFS $TOOLS_DIR/bin/env -i \ HOME=/root \ TERM=$TERM \ PS1="${BOLD}[chroot-${TARGET}(${MACHINE})]${NORMAL} \u:\w\$ " \ PATH=$PATH \ CONFIG_ROOT=${CONFIG_ROOT} \ VERSION=${VERSION} \ NAME=${NAME} \ SNAME=${SNAME} \ SLOGAN="$SLOGAN" \ CCACHE_DIR=/usr/src/ccache \ CCACHE_PREFIX=${CCACHE_PREFIX} \ CCACHE_HASHDIR=${CCACHE_HASHDIR} \ DISTCC_DIR=/usr/src/distcc \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ INSTALLER_DIR=$INSTALLER_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ CFLAGS="$CFLAGS" \ CXXFLAGS="$CXXFLAGS" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ LOGFILE=$(echo $LOGFILE | sed "s,$BASEDIR,/usr/src,g") \ $COMMAND if [ $? -ne 0 ]; then exiterror "chroot error" else stdumount fi } # End of entershell() ################################################################################ # Common checking before entering the chroot and compilling # # Return:0 caller can continue # # :1 skip (nothing to do) # # or fail if no script file found # ################################################################################ lfsmakecommoncheck() { logger "Building $*" build_spy package $1 # Script present? if [ ! -f $BASEDIR/lfs/$1 ]; then exiterror "No such file or directory: $BASEDIR/lfs/$1" fi local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) beautify make_pkg "$PKG_VER $*" # Script slipped? local i for i in $SKIP_PACKAGE_LIST; do if [ "$i" == "$1" ]; then beautify result SKIP return 1 fi done # On debug build, we don't strip if [ "$STAGE" != "toolchain" -a \ "$1" == "strip" -a "$BUILD_DEBUG" == "1" ]; then beautify result SKIP return 1 fi # Don't create addons? local EXTRA=$(grep ^EXTRA $BASEDIR/lfs/$1 | awk '{print $3}') if [ "$EXTRA" == "yes" ] && [ "$BUILD_EXTRAS" == "0" ]; then beautify result SKIP return 1 fi # Don't create debugging tools? local DEBUG=$(grep ^DEBUG $BASEDIR/lfs/$1 | awk '{print $3}') if [ "$DEBUG" == "yes" ] && [ "$BUILD_DEBUG" == "0" ]; then beautify result SKIP return 1 fi # Searching for dependencies local found message for dep in $(pkg_deps $1); do found=0 for package in $PACKAGES_DONE; do if [ "$dep" = "$package" ]; then found=1 break fi done [ "$found" = "0" ] && message="$message $dep" done if [ "$message" != "" ]; then message="Deps are missing: $message" if [ "$EXTRA" = "yes" ]; then echo -ne "${WARN}${message}${NORMAL}" beautify message SKIP return 1 else exiterror "$message" fi fi cd $BASEDIR/lfs && make -s -f $* MACHINE=$MACHINE LFS_BASEDIR=$BASEDIR MESSAGE="$1\t " download >> $LOGFILE 2>&1 if [ $? -ne 0 ]; then exiterror "Download error in $1" fi distcc_reload & return 0 # pass all! } # End of lfsmakecommoncheck() ################################################################################ # This is the function that builds every package in stage "toolchain" # ################################################################################ toolchain_make() { lfsmakecommoncheck $* [ $? == 1 ] && return 0 local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) local EXTRA_MAKE=$EXTRA_MAKE local PKG_TIME_START=$(date +%s) cd $BASEDIR/lfs && $EXTRA_MAKE make -f $* \ CONFIG_ROOT=$CONFIG_ROOT \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ TARGET="$TARGET" \ LFS_BASEDIR=$BASEDIR \ LFS=$LFS \ INSTALLER_DIR=$INSTALLER_DIR \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ install >> $LOGFILE 2>&1 local COMPILE_SUCCESS=$? local PKG_TIME_END=$(date +%s) if [ $COMPILE_SUCCESS -ne 0 ]; then beautify result FAIL $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE exiterror "Building $*" else beautify result DONE $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE PACKAGES_DONE="$PACKAGES_DONE $1" fi return 0 } # End of toolchain_make() ################################################################################ # This is the function that builds every package in stage "base" and "ipfire" # ################################################################################ ipfire_make() { lfsmakecommoncheck $* [ $? == 1 ] && return 0 local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) local EXTRA_MAKE=$EXTRA_MAKE # When cross-compiling, make sure the kernel is compiled for the target [ "$MACHINE" != "$MACHINE_REAL" -a "$1" == "linux" ] && unset EXTRA_MAKE # Also, make sure external kernel modules are compiled 64bit if grep -qEi 'KERNEL_MOD = yes' $1 ; then unset EXTRA_MAKE fi local PKG_TIME_START=$(date +%s) chroot $LFS $TOOLS_DIR/bin/env -i \ HOME=/root \ TERM=$TERM \ PS1='\u:\w\$ ' \ PATH=$PATH \ CONFIG_ROOT=${CONFIG_ROOT} \ VERSION=${VERSION} \ NAME=${NAME} \ SNAME=${SNAME} \ SLOGAN="$SLOGAN" \ CCACHE_DIR=/usr/src/ccache \ CCACHE_PREFIX=${CCACHE_PREFIX} \ CCACHE_HASHDIR=${CCACHE_HASHDIR} \ DISTCC_DIR=/usr/src/distcc \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ INSTALLER_DIR=$INSTALLER_DIR \ CDROM_DIR=$CDROM_DIR \ IMAGES_DIR=$IMAGES_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ CFLAGS="$CFLAGS" \ CXXFLAGS="$CXXFLAGS" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ TARGET="$TARGET" \ BUILD_DEBUG=$BUILD_DEBUG \ BUILD_EXTRAS=$BUILD_EXTRAS \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ EMB=$EMB \ IMAGENAME=$IMAGENAME \ LOGFILE=$(echo $LOGFILE | sed "s,$BASEDIR,/usr/src,g") \ bash -x -c "cd /usr/src/lfs && \ $EXTRA_MAKE make -f $* LFS_BASEDIR=/usr/src install" >>$LOGFILE 2>&1 local COMPILE_SUCCESS=$? local PKG_TIME_END=$(date +%s) if [ $COMPILE_SUCCESS -ne 0 ]; then beautify result FAIL $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE exiterror "Building $*" else beautify result DONE $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE PACKAGES_DONE="$PACKAGES_DONE $1" fi return 0 } # End of ipfire_make() ################################################################################ # This prepares the build environment # ################################################################################ prepareenv() { LOGFILE=$BASEDIR/log_${TARGET}/_build.00-preparation.log export LOGFILE mkdir -p $BASEDIR/log_${TARGET}/01_toolchain 2>/dev/null mkdir -p $BASEDIR/log_${TARGET}/02_base 2>/dev/null mkdir -p $BASEDIR/log_${TARGET}/03_${SNAME} 2>/dev/null mkdir -p $BASEDIR/log_${TARGET}/04_misc 2>/dev/null mkdir -p $BASEDIR/log_${TARGET}/05_installer 2>/dev/null mkdir -p $BASEDIR/log_${TARGET}/06_packages 2>/dev/null ############################################################################# # Are we running the right shell? # ############################################################################# if [ ! "$BASH" ]; then exiterror "BASH environment variable is not set. You're probably running the wrong shell." fi if [ -z "${BASH_VERSION}" ]; then exiterror "Not running BASH shell." fi ############################################################################# # Trap on emergency exit # ############################################################################# trap "exiterror 'Build process interrupted'" SIGINT SIGTERM SIGKILL SIGSTOP SIGQUIT ############################################################################# # Resetting our nice level # ############################################################################# echo -ne "Resetting our nice level to $NICE" renice $NICE $$ > /dev/null if [ $(nice) != "$NICE" ]; then beautify message FAIL exiterror "Failed to set correct nice level" else beautify message DONE fi # Set SCHED_BATCH if [ -x /usr/bin/schedtool ]; then /usr/bin/schedtool -B $$ if [ $? -ne 0 ]; then echo -ne "Setting kernel schedular to SCHED_BATCH" beautify message FAIL fi fi ############################################################################## # Checking if running as root user # ############################################################################## if [ $(id -u) != 0 ]; then echo -ne "Checking if we're running as root user" beautify message FAIL exiterror "Not building as root" fi ############################################################################## # Checking for necessary temporary space # ############################################################################## BASE_DEV=$(df -P -k $BASEDIR | tail -n 1 | awk '{ print $1 }') BASE_ASPACE=$(df -P -k $BASEDIR | tail -n 1 | awk '{ print $4 }') if (( 2048000 > $BASE_ASPACE )); then BASE_USPACE=$(du -skx $BASEDIR | awk '{print $1}') if (( 2048000 - $BASE_USPACE > $BASE_ASPACE )); then echo -ne "Checking for necessary space on disk $BASE_DEV" beautify message FAIL exiterror "Not enough temporary space available, need at least 2GB on $BASE_DEV" fi fi ############################################################################## # Embedded build # ############################################################################## echo -ne "Embedded build" if [ $EMB -eq 1 ]; then beautify message ON check_loop || exiterror "Can't build flash images on this machine." else beautify message OFF fi export EMB ############################################################################## # Checking CPU features # ############################################################################## check_supported_target $TARGET || \ exiterror "Your host doesn't have all needed cpu features for building \"$TARGET\"" ############################################################################## # Building Linux From Scratch system configuration # ############################################################################## # Set umask umask 022 # Set LFS Directory LFS=$BASEDIR/build_${TARGET}/${SNAME} # Check /tools symlink if [ -h $TOOLS_DIR ]; then rm -f $TOOLS_DIR fi if [ ! -a $TOOLS_DIR ]; then ln -s $BASEDIR/build_${TARGET}/$TOOLS_DIR / fi if [ ! -h $TOOLS_DIR ]; then exiterror "Could not create $TOOLS_DIR symbolic link." fi # Setup environment set +h LC_ALL=POSIX export LFS LC_ALL unset CC CXX CPP LD_LIBRARY_PATH LD_PRELOAD # Make some extra directories mkdir -p $BASEDIR/build_${TARGET}/{$TOOLS_DIR,cdrom,$INSTALLER_DIR,$IMAGES_DIR} 2>/dev/null mkdir -p $BASEDIR/{cache,ccache,distcc} 2>/dev/null mkdir -p $BASEDIR/cache/{toolchains,patches,tarballs} 2>/dev/null mkdir -p $LFS/{$TOOLS_DIR,usr/src} 2>/dev/null mkdir -p $LFS/{dev,etc,proc,sys} 2>/dev/null mkdir -p $LFS/dev/pts 2>/dev/null mkdir -p $LFS/usr/src/{cache,config,doc,lfs,log_${TARGET},src,ccache,distcc} 2>/dev/null mkdir -p $LFS/{$INSTALLER_DIR,cdrom,images} 2>/dev/null mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null # Make all sources and proc available under lfs build mount --bind /dev $LFS/dev mount --bind /proc $LFS/proc mount --bind /sys $LFS/sys mount --bind $BASEDIR/cache $LFS/usr/src/cache mount --bind $BASEDIR/ccache $LFS/usr/src/ccache mount --bind $BASEDIR/distcc $LFS/usr/src/distcc mount --bind $BASEDIR/config $LFS/usr/src/config mount --bind $BASEDIR/doc $LFS/usr/src/doc mount --bind $BASEDIR/lfs $LFS/usr/src/lfs mount --bind $BASEDIR/log_${TARGET} $LFS/usr/src/log_${TARGET} mount --bind $BASEDIR/src $LFS/usr/src/src mount --bind $BASEDIR/build_${TARGET}/$TOOLS_DIR $LFS/$TOOLS_DIR mount --bind $BASEDIR/build_${TARGET}/$CDROM_DIR $LFS/$CDROM_DIR mount --bind $BASEDIR/build_${TARGET}/$INSTALLER_DIR $LFS/$INSTALLER_DIR mount --bind $BASEDIR/build_${TARGET}/$IMAGES_DIR $LFS/$IMAGES_DIR # Run LFS static binary creation scripts one by one export CCACHE_DIR=$BASEDIR/ccache export CCACHE_HASHDIR=1 export CCACHE_PREFIX="distcc" export DISTCC_DIR=$BASEDIR/distcc # Remove pre-install list of installed files in case user erase some files before rebuild rm -f $LFS/usr/src/lsalr 2>/dev/null } build() { #a prebuilt toolchain package is only used if found in cache if [ ! -d $BASEDIR/cache ]; then exiterror "Use make.sh source get first!" fi cd $BASEDIR/cache/toolchains PACKAGE=$(ls -v -r $TOOLCHAINNAME.tar.bz2 2>/dev/null | head -n 1) #only restore on a clean disk local BLD_TIME_START=$(date +%s) touch $RUNNING; rm -f $FAILED $BUILD_SPY_FILENAME 2>/dev/null echo -ne "Building for ${BOLD}${TARGET} (${MACHINE}) on ${MACHINE_REAL}${NORMAL}\n" build_spy_send_profile build_spy state compiling build_spy start distcc_get_hosts if [ -f $BASEDIR/log_${TARGET}/02_base/stage2-LFS ]; then prepareenv echo "Using installed toolchain" >> $LOGFILE beautify message DONE "Stage toolchain already built or extracted" else if [ -z "$PACKAGE" ]; then echo "Full toolchain compilation" prepareenv check_toolchain beautify build_stage "Building toolchain" toolchain_build else echo "Restore from $PACKAGE" cd $BASEDIR && tar jxf $BASEDIR/cache/toolchains/$PACKAGE prepareenv fi fi # Run distcc daemon distccd_start beautify build_stage "Building base" base_build beautify build_stage "Building $SNAME" ipfire_build beautify build_stage "Building miscellaneous" misc_build if [ "${EMB}" -eq "0" ]; then beautify build_stage "Building installer" installer_build fi beautify build_stage "Building packages" packages_build echo "" echo "... and all this hard work for this:" ls -sh $BASEDIR/${IMAGENAME}.iso local BLD_TIME_END=$(date +%s) build_spy duration $[ $BLD_TIME_END - $BLD_TIME_START ] build_spy state idle build_spy exit rm -f $RUNNING } gettoolchain() { check_user DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" [ -d "${DIR_TOOLCHAIN}" ] || mkdir -p ${DIR_TOOLCHAIN} if [ ! -f $BASEDIR/cache/toolchains/$TOOLCHAINNAME.tar.bz2 ]; then URL_TOOLCHAIN=$(grep URL_TOOLCHAIN lfs/Config | awk '{ print $3 }') DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" echo "Loading toolchain for $TARGET" scp -2 ${IPFIRE_USER}@${URL_TOOLCHAIN}/$TOOLCHAINNAME.tar.bz2 \ ${DIR_TOOLCHAIN} else echo -n "Toolchain \"$TOOLCHAINNAME\" is already existing" beautify message SKIP fi } puttoolchain() { check_user if [ -f $BASEDIR/cache/toolchains/$TOOLCHAINNAME.tar.bz2 ]; then URL_TOOLCHAIN=$(grep URL_TOOLCHAIN lfs/Config | awk '{ print $3 }') DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" echo "Pushing toolchain for $TARGET" scp -2 ${DIR_TOOLCHAIN}/$TOOLCHAINNAME.tar.bz2 \ ${IPFIRE_USER}@${URL_TOOLCHAIN} else echo -n "Toolchain \"$TOOLCHAINNAME\" is not existing. " echo -n "Run \"./make.sh build\", first" beautify message SKIP fi } getsource() { if [ ! -d $BASEDIR/cache ]; then mkdir -p $BASEDIR/cache/{tarballs,patches} fi mkdir -p $BASEDIR/log_${TARGET} echo -e "${BOLD}Preload all source files${NORMAL}" cd $BASEDIR/lfs for i in *; do if [ -f "$i" -a "$i" != "Config" ]; then make -s -f $i \ LFS_BASEDIR=$BASEDIR \ MESSAGE="$i\t" \ download 2>> $LOGFILE [ $? -ne 0 ] && beautify message FAIL fi done cd $BASEDIR } putsource() { check_user URL_SOURCE=$(grep URL_SOURCE lfs/Config | awk '{ print $3 }') REMOTE_FILES=$(echo "ls -1" | sftp -C ${IPFIRE_USER}@${URL_SOURCE}) cd $BASEDIR/cache/tarballs/ for file in $(ls -1); do grep -q "$file" <<<$REMOTE_FILES && continue NEW_FILES="$NEW_FILES $file" done [ -n "$NEW_FILES" ] && scp -2 $NEW_FILES ${IPFIRE_USER}@${URL_SOURCE} cd $BASEDIR } puttarget() { check_user URL_TARGET=$(grep URL_TARGET lfs/Config | awk '{ print $3 }') DIR="${BASEDIR}/${HOSTNAME}/$(date '+%Y%m%d-%0k')-${TARGET}/" # If there is no iso, we do nothing. ls ${BASEDIR}/${IMAGENAME}.* &>/dev/null || return 0 rm -rf ${BASEDIR}/${HOSTNAME} 2>/dev/null mkdir -p ${DIR} [ -e "${BASEDIR}/packages" ] && cp -al ${BASEDIR}/packages ${DIR} [ -e "${BATCHLOG}" ] && \ python ${BASEDIR}/tools/alog2html < ${BATCHLOG} > ${DIR}/build_log.html pkg_list_packages > ${DIR}/packages-list.txt git_log >/dev/null; cp -l ${BASEDIR}/doc/ChangeLog ${DIR} git_export; cp -l ${BASEDIR}/${SNAME}-${VERSION}.source.tar.gz ${DIR} git_diff >/dev/null && cp -l ${DIFF_NAME} ${DIR} cp -l ${BASEDIR}/${IMAGENAME}.* ${DIR} cd $BASEDIR && \ scp -2 -r ${HOSTNAME} ${IPFIRE_USER}@${URL_TARGET} || : rm -rf ${BASEDIR}/${HOSTNAME} 2>/dev/null }