#!/bin/bash ############################################################################### # # # IPFire.org - A linux based firewall # # Copyright (C) 2008 Michael Tremer & Christian Schmidt # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # # the Free Software Foundation, either version 3 of the License, or # # (at your option) any later version. # # # # This program is distributed in the hope that it will be useful, # # but WITHOUT ANY WARRANTY; without even the implied warranty of # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # # GNU General Public License for more details. # # # # You should have received a copy of the GNU General Public License # # along with this program. If not, see . # # # ############################################################################### ############################################################################### # # System variables # ############################################################################### PWD=$(pwd) BASENAME=$(basename $0) # Debian specific settings if [ ! -e /etc/debian_version ]; then FULLPATH=$(which $0) else if [ -x /usr/bin/realpath ]; then FULLPATH=$(/usr/bin/realpath $0) else echo "ERROR: Need to do apt-get install realpath" exit 1 fi fi BASEDIR=$(echo $FULLPATH | sed "s/\/$BASENAME//g") export BASEDIR . $BASEDIR/tools/make-constants # Load this very early . $BASEDIR/tools/make-beautify mkdir $BASEDIR/log_${MACHINE}/ 2>/dev/null ################################################################################ # # # Necessary shell functions # # # ################################################################################ . $BASEDIR/tools/make-buildspy . $BASEDIR/tools/make-check . $BASEDIR/tools/make-batch . $BASEDIR/tools/make-compilers . $BASEDIR/tools/make-git evaluate() { RETVAL=$? if [ "$RETVAL" -eq "0" ]; then beautify message DONE else beautify message FAIL fi (exit $RETVAL) } stdumount() { sleep 0.3 # Wait one second for finish of processes for fs in $(mount | grep $BASEDIR/build_${MACHINE} | awk '{print $3}'); do umount $fs 2>/dev/null; done } # End of stdumount() exiterror() { stdumount dialogerror $* build_spy state error touch $FAILED rm -f $RUNNING 2>/dev/null exit 1 } # End of exiterror() ################################################################################ # This is the function that sets the environment of a chroot and enters it # ################################################################################ entershell() { PATH=${TOOLS_DIR}/usr/bin:/bin:/usr/bin:/sbin:/usr/sbin:${TOOLS_DIR}/bin:/usr/${MACHINE_REAL}-linux/bin if [ ! -e $LFS/usr/src/lfs/ ]; then exiterror "No such file or directory: $LFS/usr/src/lfs/" fi echo -ne "Entering ${BOLD}$MACHINE${NORMAL} LFS chroot, type exit to return to host environment\n" chroot $LFS $TOOLS_DIR/bin/env -i \ HOME=/root \ TERM=$TERM \ PS1="${BOLD}[chroot-${TARGET}(${MACHINE})]${NORMAL} \u:\w\$ " \ PATH=$PATH \ CONFIG_ROOT=${CONFIG_ROOT} \ VERSION=${VERSION} \ NAME=${NAME} \ SNAME=${SNAME} \ SLOGAN="$SLOGAN" \ CCACHE_DIR=/usr/src/ccache \ CCACHE_PREFIX=${CCACHE_PREFIX} \ CCACHE_HASHDIR=${CCACHE_HASHDIR} \ DISTCC_DIR=/usr/src/distcc \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ INSTALLER_DIR=$INSTALLER_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ CFLAGS="$CFLAGS" \ CXXFLAGS="$CXXFLAGS" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ LOGFILE=$(echo $LOGFILE | sed "s,$BASEDIR,/usr/src,g") \ bash if [ $? -ne 0 ]; then exiterror "chroot error" else stdumount fi } # End of entershell() ################################################################################ # Common checking before entering the chroot and compilling # # Return:0 caller can continue # # :1 skip (nothing to do) # # or fail if no script file found # ################################################################################ lfsmakecommoncheck() { # Script present? if [ ! -f $BASEDIR/lfs/$1 ]; then exiterror "No such file or directory: $BASEDIR/lfs/$1" fi local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) beautify make_pkg "$PKG_VER $*" # Script slipped? local i for i in $SKIP_PACKAGE_LIST; do if [ "$i" == "$1" ]; then beautify result SKIP return 1 fi done # On debug build, we don't strip if [ "$1" == "strip" -a "$BUILD_DEBUG" == "1" ]; then beautify result SKIP return 1 fi # Don't create addons? local EXTRA=$(grep ^EXTRA $BASEDIR/lfs/$1 | awk '{print $3}') if [ "$EXTRA" == "yes" -a "$BUILD_EXTRAS" == "0" ]; then beautify result SKIP return 1 fi # Don't create debugging tools? local DEBUG=$(grep ^DEBUG $BASEDIR/lfs/$1 | awk '{print $3}') if [ "$DEBUG" == "yes" -a "$BUILD_DEBUG" == "1" ]; then beautify result SKIP return 1 fi echo -e "$(date -u '+%b %e %T'): Building $* " >> $LOGFILE cd $BASEDIR/lfs && make -s -f $* MACHINE=$MACHINE LFS_BASEDIR=$BASEDIR MESSAGE="$1\t " download >> $LOGFILE 2>&1 if [ $? -ne 0 ]; then exiterror "Download error in $1" fi build_spy package $1 return 0 # pass all! } # End of lfsmakecommoncheck() ################################################################################ # This is the function that builds every package in stage "toolchain" # ################################################################################ toolchain_make() { lfsmakecommoncheck $* [ $? == 1 ] && return 0 local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) local EXTRA_MAKE=$EXTRA_MAKE local PKG_TIME_START=$(date +%s) cd $BASEDIR/lfs && $EXTRA_MAKE make -f $* \ CONFIG_ROOT=$CONFIG_ROOT \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ LFS_BASEDIR=$BASEDIR \ LFS=$LFS \ INSTALLER_DIR=$INSTALLER_DIR \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ SSP=$SSP \ PIE=$PIE \ PAX=$PAX \ install >> $LOGFILE 2>&1 local COMPILE_SUCCESS=$? local PKG_TIME_END=$(date +%s) if [ $COMPILE_SUCCESS -ne 0 ]; then beautify result FAIL $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE exiterror "Building $*" else beautify result DONE $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE fi return 0 } # End of toolchain_make() ################################################################################ # This is the function that builds every package in stage "base" and "ipfire" # ################################################################################ ipfire_make() { lfsmakecommoncheck $* [ $? == 1 ] && return 0 local PKG_VER=$(get_pkg_ver $BASEDIR/lfs/$1) local EXTRA_MAKE=$EXTRA_MAKE # When cross-compiling, make sure the kernel is compiled for the target [ "$MACHINE" != "$MACHINE_REAL" -a "$1" == "linux" ] && unset EXTRA_MAKE # Also, make sure external kernel modules are compiled 64bit if grep -qEi 'KERNEL_MOD = yes' $1 ; then unset EXTRA_MAKE fi local PKG_TIME_START=$(date +%s) chroot $LFS $TOOLS_DIR/bin/env -i \ HOME=/root \ TERM=$TERM \ PS1='\u:\w\$ ' \ PATH=$PATH \ CONFIG_ROOT=${CONFIG_ROOT} \ VERSION=${VERSION} \ NAME=${NAME} \ SNAME=${SNAME} \ SLOGAN="$SLOGAN" \ CCACHE_DIR=/usr/src/ccache \ CCACHE_PREFIX=${CCACHE_PREFIX} \ CCACHE_HASHDIR=${CCACHE_HASHDIR} \ DISTCC_DIR=/usr/src/distcc \ PARALLELISMFLAGS=$PARALLELISMFLAGS \ LINKER=$LINKER \ TOOLS_DIR=$TOOLS_DIR \ INSTALLER_DIR=$INSTALLER_DIR \ CDROM_DIR=$CDROM_DIR \ IMAGES_DIR=$IMAGES_DIR \ MACHINE="$MACHINE" \ MACHINE_REAL="$MACHINE_REAL" \ CFLAGS="$CFLAGS" \ CXXFLAGS="$CXXFLAGS" \ IFS_HOST="$IFS_HOST" \ IFS_TARGET="$IFS_TARGET" \ IFS_ARCH="$TARGET" \ BUILD_DEBUG=$BUILD_DEBUG \ BUILD_EXTRAS=$BUILD_EXTRAS \ KVER=$KVER \ STAGE=$STAGE \ STAGE_ORDER=$STAGE_ORDER \ SSP=$SSP \ PIE=$PIE \ PAX=$PAX \ EMB=$EMB \ LOGFILE=$(echo $LOGFILE | sed "s,$BASEDIR,/usr/src,g") \ bash -x -c "cd /usr/src/lfs && \ $EXTRA_MAKE make -f $* LFS_BASEDIR=/usr/src install" >>$LOGFILE 2>&1 local COMPILE_SUCCESS=$? local PKG_TIME_END=$(date +%s) if [ $COMPILE_SUCCESS -ne 0 ]; then beautify result FAIL $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE exiterror "Building $*" else beautify result DONE $[ $PKG_TIME_END - $PKG_TIME_START ] $1 $PKG_VER $STAGE_ORDER $STAGE fi return 0 } # End of ipfire_make() ################################################################################ # This prepares the build environment # ################################################################################ prepareenv() { LOGFILE=$BASEDIR/log_${MACHINE}/_build.00-preparation.log export LOGFILE mkdir -p $BASEDIR/log_${MACHINE}/01_toolchain 2>/dev/null mkdir -p $BASEDIR/log_${MACHINE}/02_base 2>/dev/null mkdir -p $BASEDIR/log_${MACHINE}/03_${SNAME} 2>/dev/null mkdir -p $BASEDIR/log_${MACHINE}/04_misc 2>/dev/null mkdir -p $BASEDIR/log_${MACHINE}/05_installer 2>/dev/null mkdir -p $BASEDIR/log_${MACHINE}/06_packages 2>/dev/null ############################################################################# # Are we running the right shell? # ############################################################################# if [ ! "$BASH" ]; then exiterror "BASH environment variable is not set. You're probably running the wrong shell." fi if [ -z "${BASH_VERSION}" ]; then exiterror "Not running BASH shell." fi ############################################################################# # Trap on emergency exit # ############################################################################# trap "exiterror 'Build process interrupted'" SIGINT SIGTERM SIGKILL SIGSTOP SIGQUIT ############################################################################# # Resetting our nice level # ############################################################################# echo -ne "Resetting our nice level to $NICE" renice $NICE $$ > /dev/null if [ $(nice) != "$NICE" ]; then beautify message FAIL exiterror "Failed to set correct nice level" else beautify message DONE fi # Set SCHED_BATCH if [ -x /usr/bin/schedtool ]; then /usr/bin/schedtool -B $$ if [ $? -ne 0 ]; then echo -ne "Setting kernel schedular to SCHED_BATCH" beautify message FAIL fi fi ############################################################################## # Checking if running as root user # ############################################################################## if [ $(id -u) != 0 ]; then echo -ne "Checking if we're running as root user" beautify message FAIL exiterror "Not building as root" fi ############################################################################## # Checking for necessary temporary space # ############################################################################## BASE_DEV=$(df -P -k $BASEDIR | tail -n 1 | awk '{ print $1 }') BASE_ASPACE=$(df -P -k $BASEDIR | tail -n 1 | awk '{ print $4 }') if (( 2048000 > $BASE_ASPACE )); then BASE_USPACE=$(du -skx $BASEDIR | awk '{print $1}') if (( 2048000 - $BASE_USPACE > $BASE_ASPACE )); then echo -ne "Checking for necessary space on disk $BASE_DEV" beautify message FAIL exiterror "Not enough temporary space available, need at least 2GB on $BASE_DEV" fi fi ############################################################################## # Setting security features # ############################################################################## echo -ne "Stack smashing protector (SSP)" if [ $SSP -eq 1 ]; then beautify message ON else beautify message OFF fi echo -ne "Position independent executeables (PIE)" if [ $PIE -eq 1 ]; then beautify message ON else beautify message OFF fi echo -ne "GRSecurity (PAX)" if [ $PAX -eq 1 ]; then beautify message ON else beautify message OFF fi export SSP PIE PAX ############################################################################## # Embedded build # ############################################################################## echo -ne "Embedded build" if [ $EMB -eq 1 ]; then beautify message ON check_loop || exiterror "Can't build flash images on this machine." else beautify message OFF fi export EMB ############################################################################## # Checking CPU features # ############################################################################## CHECK_CPU="" if [ 'via-c7' = $TARGET ]; then CHECK_CPU="sse sse2 pni" elif [ 'via-c3' = $TARGET ]; then CHECK_CPU="3dnow" fi for flag in $CHECK_CPU; do check_cpu $flag || \ exiterror "Your system doesn't support needed cpu feature \"$flag\" to build target $TARGET." done ############################################################################## # Building Linux From Scratch system configuration # ############################################################################## # Set umask umask 022 # Set LFS Directory LFS=$BASEDIR/build_${MACHINE}/${SNAME} # Check /tools symlink if [ -h $TOOLS_DIR ]; then rm -f $TOOLS_DIR fi if [ ! -a $TOOLS_DIR ]; then ln -s $BASEDIR/build_${MACHINE}/$TOOLS_DIR / fi if [ ! -h $TOOLS_DIR ]; then exiterror "Could not create $TOOLS_DIR symbolic link." fi # Setup environment set +h LC_ALL=POSIX export LFS LC_ALL unset CC CXX CPP LD_LIBRARY_PATH LD_PRELOAD # Make some extra directories mkdir -p $BASEDIR/build_${MACHINE}/{$TOOLS_DIR,cdrom,$INSTALLER_DIR,$IMAGES_DIR} 2>/dev/null mkdir -p $BASEDIR/{cache,ccache,distcc} 2>/dev/null mkdir -p $BASEDIR/cache/{toolchains,patches,tarballs} 2>/dev/null mkdir -p $LFS/{$TOOLS_DIR,usr/src} 2>/dev/null mkdir -p $LFS/{dev,etc,proc,sys} 2>/dev/null mkdir -p $LFS/dev/pts 2>/dev/null mkdir -p $LFS/usr/src/{cache,config,doc,lfs,log_${MACHINE},src,ccache,distcc} 2>/dev/null mkdir -p $LFS/{$INSTALLER_DIR,cdrom,images} 2>/dev/null mknod -m 600 $BASEDIR/build/dev/console c 5 1 2>/dev/null mknod -m 666 $BASEDIR/build/dev/null c 1 3 2>/dev/null # Make all sources and proc available under lfs build mount --bind /dev $LFS/dev mount --bind /proc $LFS/proc mount --bind /sys $LFS/sys mount --bind $BASEDIR/cache $LFS/usr/src/cache mount --bind $BASEDIR/ccache $LFS/usr/src/ccache mount --bind $BASEDIR/distcc $LFS/usr/src/distcc mount --bind $BASEDIR/config $LFS/usr/src/config mount --bind $BASEDIR/doc $LFS/usr/src/doc mount --bind $BASEDIR/lfs $LFS/usr/src/lfs mount --bind $BASEDIR/log_${MACHINE} $LFS/usr/src/log_${MACHINE} mount --bind $BASEDIR/src $LFS/usr/src/src mount --bind $BASEDIR/build_${MACHINE}/$TOOLS_DIR $LFS/$TOOLS_DIR mount --bind $BASEDIR/build_${MACHINE}/$CDROM_DIR $LFS/$CDROM_DIR mount --bind $BASEDIR/build_${MACHINE}/$INSTALLER_DIR $LFS/$INSTALLER_DIR mount --bind $BASEDIR/build_${MACHINE}/$IMAGES_DIR $LFS/$IMAGES_DIR # Run LFS static binary creation scripts one by one export CCACHE_DIR=$BASEDIR/ccache export CCACHE_HASHDIR=1 if [ ! -z "$DISTCC_HOSTS" ]; then export CCACHE_PREFIX="distcc" export DISTCC_DIR=$BASEDIR/distcc fi [ -z "$DISTCC_HOSTS" ] || echo "$DISTCC_HOSTS" > $DISTCC_DIR/hosts # Remove pre-install list of installed files in case user erase some files before rebuild rm -f $LFS/usr/src/lsalr 2>/dev/null } build() { clear #a prebuilt toolchain package is only used if found in cache if [ ! -d $BASEDIR/cache ]; then exiterror "Use make.sh source get first!" fi cd $BASEDIR/cache/toolchains PACKAGE=$(ls -v -r $TOOLCHAINNAME.tar.bz2 2>/dev/null | head -n 1) #only restore on a clean disk local BLD_TIME_START=$(date +%s) touch $RUNNING; rm -f $FAILED $BUILD_SPY_FILENAME 2>/dev/null echo -ne "Building for ${BOLD}${TARGET} (${MACHINE}) on ${MACHINE_REAL}${NORMAL}\n" build_spy_send_profile build_spy state compiling BASEDIR=$BASEDIR UUID=$UUID NAME=$NAME VERSION=$VERSION $BASEDIR/tools/make-buildspy & if [ -f $BASEDIR/log_${MACHINE}/02_base/stage2-LFS ]; then prepareenv echo "Using installed toolchain" >> $LOGFILE beautify message DONE "Stage toolchain already built or extracted" else if [ -z "$PACKAGE" ]; then echo "Full toolchain compilation" prepareenv check_toolchain beautify build_stage "Building toolchain" toolchain_build else echo "Restore from $PACKAGE" cd $BASEDIR && tar jxf $BASEDIR/cache/toolchains/$PACKAGE prepareenv fi fi # Run distcc daemon distccd_start beautify build_stage "Building base" base_build beautify build_stage "Building $SNAME" ipfire_build beautify build_stage "Building miscellaneous" misc_build if [ "${EMB}" -eq "0" ]; then beautify build_stage "Building installer" installer_build fi beautify build_stage "Building packages" packages_build echo "" echo "... and all this hard work for this:" ls -sh $BASEDIR/${SNAME}-${VERSION}.${MACHINE}.iso local BLD_TIME_END=$(date +%s) build_spy duration $[ $BLD_TIME_END - $BLD_TIME_START ] build_spy state idle rm -f $RUNNING } gettoolchain() { check_user DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" [ -d "${DIR_TOOLCHAIN}" ] || mkdir -p ${DIR_TOOLCHAIN} if [ ! -f $BASEDIR/cache/toolchains/$TOOLCHAINNAME.tar.bz2 ]; then URL_TOOLCHAIN=$(grep URL_TOOLCHAIN lfs/Config | awk '{ print $3 }') DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" echo "Loading toolchain for $MACHINE" scp -2 ${IPFIRE_USER}@${URL_TOOLCHAIN}/$TOOLCHAINNAME.tar.bz2 \ ${DIR_TOOLCHAIN} else echo -n "Toolchain \"$TOOLCHAINNAME\" is already existing" beautify message SKIP fi } puttoolchain() { check_user if [ -f $BASEDIR/cache/toolchains/$TOOLCHAINNAME.tar.bz2 ]; then URL_TOOLCHAIN=$(grep URL_TOOLCHAIN lfs/Config | awk '{ print $3 }') DIR_TOOLCHAIN="$BASEDIR/cache/toolchains" echo "Pushing toolchain for $MACHINE" scp -2 ${DIR_TOOLCHAIN}/$TOOLCHAINNAME.tar.bz2 \ ${IPFIRE_USER}@${URL_TOOLCHAIN} else echo -n "Toolchain \"$TOOLCHAINNAME\" is not existing. " echo -n "Run \"./make.sh build\", first" beautify message SKIP fi } getsource() { if [ ! -d $BASEDIR/cache ]; then mkdir -p $BASEDIR/cache/{tarballs,patches} fi mkdir -p $BASEDIR/log_${MACHINE} echo -e "${BOLD}Preload all source files${NORMAL}" cd $BASEDIR/lfs for i in *; do if [ -f "$i" -a "$i" != "Config" ]; then make -s -f $i \ LFS_BASEDIR=$BASEDIR \ MESSAGE="$i\t" \ SSP=$SSP \ PIE=$PIE \ PAX=$PAX download 2>> $LOGFILE [ $? -ne 0 ] && beautify message FAIL fi done cd $BASEDIR } putsource() { check_user URL_SOURCE=$(grep URL_SOURCE lfs/Config | awk '{ print $3 }') REMOTE_FILES=$(echo "ls -1" | sftp -C ${IPFIRE_USER}@${URL_SOURCE}) cd $BASEDIR/cache/tarballs/ for file in $(ls -1); do grep -q "$file" <<<$REMOTE_FILES && continue NEW_FILES="$NEW_FILES $file" done [ -n "$NEW_FILES" ] && scp -2 $NEW_FILES ${IPFIRE_USER}@${URL_SOURCE} cd $BASEDIR } puttarget() { check_user URL_TARGET=$(grep URL_TARGET lfs/Config | awk '{ print $3 }') DIR="${BASEDIR}/${HOSTNAME}/$(date '+%Y%m%d-%0k')/" # If there is no iso, we do nothing. [ -e "${BASEDIR}/${SNAME}-${VERSION}.${MACHINE}.iso" ] || return 0 rm -rf ${BASEDIR}/${HOSTNAME} 2>/dev/null mkdir -p ${DIR} [ -e "${BASEDIR}/packages" ] && cp -al ${BASEDIR}/packages ${DIR} [ -e "${BATCHLOG}" ] && \ python ${BASEDIR}/tools/alog2html < ${BATCHLOG} > ${DIR}/build_log.html git_export; cp -l ${BASEDIR}/${SNAME}-${VERSION}.source.tar.gz ${DIR} git_diff >/dev/null && cp -l ${DIFF_NAME} ${DIR} cp -l ${BASEDIR}/${SNAME}-${VERSION}.${MACHINE}.iso ${DIR} cd $BASEDIR && \ scp -2 -r ${HOSTNAME} ${IPFIRE_USER}@${URL_TARGET} || : rm -rf ${BASEDIR}/${HOSTNAME} 2>/dev/null }