build.sh

   1 #!/bin/bash -l
   2
   3 BASEDIR="/build/nightly"
   4 LOCKFILE="/tmp/.nightly-builds.lock"
   5
   6 UPLOAD_DIR="${BASEDIR}/upload"
   7 UPLOAD_TO="pakfire@fs01.haj.ipfire.org:/pub/nightly"
   8 RSYNC_ARGS=( "--delay-updates" )
   9
  10 extract_installer_from_iso() {
  11         local dir="${1}"
  12
  13         local tmpdir="$(mktemp -d)"
  14         mkdir -p "${dir}/images"
  15
  16         local file
  17         for file in ${dir}/*.iso; do
  18                 if mount -o loop,ro "${file}" "${tmpdir}"; then
  19                         local f
  20                         for f in vmlinuz instroot; do
  21                                 cp -f "${tmpdir}/boot/isolinux/${f}" "${dir}/images"
  22                         done
  23                         umount "${tmpdir}"
  24
  25                         local ext
  26                         for ext in "" ".md5"; do
  27                                 ln -s --relative "${file}${ext}" \
  28                                         "${dir}/images/installer.iso${ext}"
  29                         done
  30                         break
  31                 fi
  32         done 2>/dev/null
  33
  34         rm -rf "${tmpdir}"
  35 }
  36
  37 uriencode() {
  38         local path="${1}"
  39
  40         local IFS="/"
  41
  42         local s
  43         local segments=()
  44         for s in ${path}; do
  45                 s="${s//'%'/%25}"
  46                 s="${s//' '/%20}"
  47                 s="${s//'"'/%22}"
  48                 s="${s//'#'/%23}"
  49                 s="${s//'$'/%24}"
  50                 s="${s//'&'/%26}"
  51                 s="${s//'+'/%2B}"
  52                 s="${s//','/%2C}"
  53                 s="${s//'/'/%2F}"
  54                 s="${s//':'/%3A}"
  55                 s="${s//';'/%3B}"
  56                 s="${s//'='/%3D}"
  57                 s="${s//'?'/%3F}"
  58                 s="${s//'@'/%40}"
  59                 s="${s//'['/%5B}"
  60                 s="${s//']'/%5D}"
  61
  62                 segments+=( "${s}" )
  63         done
  64
  65         echo "${segments[*]}"
  66 }
  67
  68 send_email() {
  69         local status="${1}"
  70         local target="${2}"
  71         local branch="${3}"
  72         local commit="${4}"
  73         local build="${5}"
  74
  75         sendmail -ti <<END
  76 From: IPFire Nightly Builder <nightly-builds@ipfire.org>
  77 To: Nightly Builds List <nightly-builds@lists.ipfire.org>
  78 Subject: [${status^^}] Nightly Build of ${branch} (${commit:0:7}) for ${target} on ${HOSTNAME}
  79 Date: $(date --rfc-2822)
  80 MIME-Version: 1.0
  81 Content-Type: text/plain; charset="us-ascii"
  82 Content-Transfer-Encoding: 7bit
  83
  84 https://nightly.ipfire.org$(uriencode "${build:${#UPLOAD_DIR}}")
  85
  86 $(git log -1 "${commit}")
  87
  88 https://git.ipfire.org/?p=ipfire-2.x.git;a=shortlog;h=${commit}
  89
  90 $(<"${build}/build.log")
  91 END
  92 }
  93
  94 build() {
  95         local dir="${1}"
  96         shift
  97
  98         # Check if this actually source of IPFire 2
  99         [ -x "${dir}/make.sh" ] || return 1
 100
 101         pushd "${dir}"
 102
 103         local branch="$(git config build.branch)"
 104         [ -z "${branch}" ] && branch="master"
 105
 106         local commit_old="$(git rev-parse HEAD)"
 107         git fetch
 108
 109         # Search for the latest core branch
 110         if [ "${branch}" = "core" ]; then
 111                 branch=$(git branch -r | awk -F/ '{ print $NF }' | grep ^core | sort --version-sort | tail -n1)
 112         fi
 113
 114         # If the branch was not updated, we won't build
 115         local commit_new="$(git rev-parse origin/${branch})"
 116         [ "${commit_old}" = "${commit_new}" ] && return 2
 117
 118         local current_branch="$(git rev-parse --abbrev-ref HEAD)"
 119         if [ "${current_branch}" != "${branch}" ]; then
 120                 git checkout -b "${branch}" "${commit_new}"
 121         fi
 122
 123         # Checkout the latest commit
 124         git reset --hard "${commit_new}"
 125
 126         local now="$(git log --format="%ci" -1 "${commit_new}")"
 127
 128         local targets="$(git config build.targets)"
 129         [ -z "${targets}" ] && targets="i586"
 130
 131         local target
 132         for target in ${targets}; do
 133                 local build_path="${UPLOAD_DIR}/${branch}/${now}-${commit_new:0:8}"
 134                 local build="${build_path}/${target}"
 135                 local status="failed"
 136
 137                 # Ready for build
 138                 ./make.sh --target="${target}" clean
 139
 140                 # Download the toolchain if required
 141                 ./make.sh --target="${target}" gettoolchain || continue
 142
 143                 # Download all sources
 144                 ./make.sh --target="${target}" downloadsrc || continue
 145
 146                 # Execute the build
 147                 mkdir -p "${build}"
 148                 ./make.sh --target="${target}" build | tee "${build}/build.log"
 149                 local ret=${PIPESTATUS[0]}
 150
 151                 # Save the changelog
 152                 git log -50 > "${build}/changelog.txt"
 153
 154                 # Save the result
 155                 if [ "${ret}" = "0" ]; then
 156                         status="success"
 157                         touch "${build}/.success"
 158
 159                         # Copy images
 160                         mv -v *.iso *.img.gz *.img.xz *.tar.bz2 *.md5 packages/ "${build}"
 161                         extract_installer_from_iso "${build}"
 162                 fi
 163                 mv -v log/ "${build}"
 164
 165                 # Send an email notification
 166                 send_email "${status}" "${target}" "${branch}" "${commit_new}" "${build}"
 167
 168                 # Cleanup the build environment
 169                 ./make.sh --target="${target}" clean
 170
 171                 # Upload the result
 172                 # If that failed, we will keep the result and it will
 173                 # be retried with the next build. If that succeeded, the
 174                 # build will be removed from disk.
 175                 if sync; then
 176                         rm -rf "${UPLOAD_DIR}"
 177                 fi
 178         done
 179
 180         popd
 181 }
 182
 183 sync() {
 184         mkdir -p "${UPLOAD_DIR}"
 185
 186         # Acquire a Kerberos ticket for authentication
 187         kinit -k -t /etc/krb5.keytab "host/${HOSTNAME}"
 188
 189         rsync -avH --progress "${RSYNC_ARGS[@]}" \
 190                 "${UPLOAD_DIR}/" "${UPLOAD_TO}"
 191 }
 192
 193 is_locked() {
 194         [ -e "${LOCKFILE}" ]
 195 }
 196
 197 lock() {
 198         touch "${LOCKFILE}"
 199 }
 200
 201 unlock() {
 202         rm -f "${LOCKFILE}"
 203 }
 204
 205 # Don't start again if the script is already running
 206 # or if an other build script is running
 207 if is_locked || pgrep make.sh >/dev/null; then
 208         exit 0
 209 fi
 210
 211 # Lock
 212 trap unlock EXIT
 213 lock
 214
 215 for repo in $(find ${BASEDIR} -maxdepth 3 -type d -name ".git"); do
 216         [ -d "${repo}" ] || continue
 217
 218         build "$(dirname ${repo})"
 219 done
 220
 221 # Try to sync even nothing was built for retrying failed uploads
 222 sync
 223
 224 exit 0