]> git.ipfire.org Git - thirdparty/asterisk.git/commitdiff
CI: Initial commit for moving CI into source repo
authorGeorge Joseph <gjoseph@digium.com>
Wed, 11 Jul 2018 11:14:49 +0000 (05:14 -0600)
committerGeorge Joseph <gjoseph@digium.com>
Wed, 11 Jul 2018 11:29:29 +0000 (06:29 -0500)
Create tests/CI directory and add files used by Jenkins to
build and test Asterisk.

With this commit, Jenkins will run the Asterisk Unit Tests using
the Jenkinsfile at tests/CI/unittests.jenkinsfile.  Bash scripts
to do the actual building and testing are also in the same directory.
Output is placed in tests/CI/output so that directory has been
added to .gitignore.

Change-Id: I9448065465e6de2b878634510ace8fd1ef378608

.gitignore
tests/CI/buildAsterisk.sh [new file with mode: 0755]
tests/CI/ci.functions [new file with mode: 0644]
tests/CI/installAsterisk.sh [new file with mode: 0755]
tests/CI/runTestsuite.sh [new file with mode: 0755]
tests/CI/runUnittests.sh [new file with mode: 0755]
tests/CI/setupEnvironment.sh [new file with mode: 0755]
tests/CI/unittests.jenkinsfile [new file with mode: 0644]

index 3e09738e0eee45c6facf958713584a9ceecec63c..0f31820627e20f4affc985cb72ea4bd5448d440e 100644 (file)
@@ -36,3 +36,5 @@ latex
 doxygen.log
 out/
 *.orig
+tests/CI/output
+
diff --git a/tests/CI/buildAsterisk.sh b/tests/CI/buildAsterisk.sh
new file mode 100755 (executable)
index 0000000..fce81fc
--- /dev/null
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+
+gen_cats() {
+       set +x
+       action=$1
+       shift
+       cats=$@
+
+       for x in $cats ; do
+               echo " --${action}-category ${x}"
+       done
+}
+
+gen_mods() {
+       set +x
+       action=$1
+       shift
+       mods=$@
+
+       for x in $mods ; do
+               echo " --${action} ${x}"
+       done
+}
+
+sudo mkdir -p /srv/cache/externals /srv/cache/sounds || :
+sudo chown -R jenkins:users /srv/cache
+[ ! -d tests/CI/output ] && mkdir tests/CI/output
+sudo chown -R jenkins:users tests/CI/output
+
+MAKE=`which make`
+printenv | sort
+
+common_config_args="--sysconfdir=/etc --with-pjproject-bundled"
+common_config_args+=" --with-sounds-cache=/srv/cache/sounds --with-externals-cache=/srv/cache/externals"
+common_config_args+=" --enable-dev-mode"
+export WGET_EXTRA_ARGS="--quiet"
+
+runner ./configure ${common_config_args} CCACHE_DISABLE=1 >tests/CI/output/configure.txt
+
+runner ${MAKE} menuselect.makeopts
+
+runner menuselect/menuselect `gen_mods enable DONT_OPTIMIZE BETTER_BACKTRACES MALLOC_DEBUG DO_CRASH TEST_FRAMEWORK` menuselect.makeopts
+runner menuselect/menuselect `gen_mods disable COMPILE_DOUBLE BUILD_NATIVE` menuselect.makeopts
+
+cat_enables="MENUSELECT_BRIDGES MENUSELECT_CEL MENUSELECT_CDR"
+cat_enables+=" MENUSELECT_CHANNELS MENUSELECT_CODECS MENUSELECT_FORMATS MENUSELECT_FUNCS"
+cat_enables+=" MENUSELECT_PBX MENUSELECT_RES MENUSELECT_UTILS MENUSELECT_TESTS"
+runner menuselect/menuselect `gen_cats enable $cat_enables` menuselect.makeopts
+
+mod_disables="res_digium_phone chan_vpb"
+[ "$BRANCH_NAME" == "master" ] && mod_disables+=" codec_opus codec_silk codec_g729a codec_siren7 codec_siren14"
+runner menuselect/menuselect `gen_mods disable $mod_disables` menuselect.makeopts
+
+mod_enables="app_voicemail app_directory FILE_STORAGE"
+mod_enables+=" res_mwi_external res_ari_mailboxes res_mwi_external_ami res_stasis_mailbox"
+mod_enables+=" CORE-SOUNDS-EN-GSM MOH-OPSOUND-GSM EXTRA-SOUNDS-EN-GSM"
+runner menuselect/menuselect `gen_mods enable $mod_enables` menuselect.makeopts
+
+runner ${MAKE} -j8 || runner ${MAKE} -j1 NOISY_BUILD=yes
+
+ALEMBIC=$(which alembic 2>/dev/null || : )
+if [ x"$ALEMBIC" = x ] ; then
+       echo "Alembic not installed"
+       exit 1
+fi
+
+cd contrib/ast-db-manage
+find -name *.pyc -delete
+out=$(alembic -c config.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+       >&2 echo "Alembic branches were found for config"
+       >&2 echo $out
+       exit 1
+else
+       >&2 echo "Alembic for 'config' OK"
+fi
+
+out=$(alembic -c cdr.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+       >&2 echo "Alembic branches were found for cdr"
+       >&2 echo $out
+       exit 1
+else
+       >&2 echo "Alembic for 'cdr' OK"
+fi
+
+out=$(alembic -c voicemail.ini.sample branches)
+if [ "x$out" != "x" ] ; then
+       >&2 echo "Alembic branches were found for voicemail"
+       >&2 echo $out
+       exit 1
+else
+       >&2 echo "Alembic for 'voicemail' OK"
+fi
+
+if [ -f "doc/core-en_US.xml" ] ; then
+       ${MAKE} validate-docs || ${MAKE} NOISY_BUILD=yes validate-docs
+fi
+
+
diff --git a/tests/CI/ci.functions b/tests/CI/ci.functions
new file mode 100644 (file)
index 0000000..f3de16e
--- /dev/null
@@ -0,0 +1,26 @@
+#
+# This file contains useful Bash functions
+# and can be "source"d from the scripts.
+#
+
+for a in "$@" ; do
+       OPTION_COUNT+=1
+       case "$a" in
+               --*=*)
+                       [[ $a =~ --([^=]+)=(.*) ]]
+                       l=${BASH_REMATCH[1]//-/_}
+                       r=${BASH_REMATCH[2]}
+                       eval ${l^^}=\"$r\"
+                       ;;
+               --*)
+                       [[ $a =~ --(.+) ]]
+                       l=${BASH_REMATCH[1]//-/_}
+                       eval ${l^^}=1
+                       ;;
+       esac
+done
+
+runner() {
+       ( set -x ; "$@" )
+}
+
diff --git a/tests/CI/installAsterisk.sh b/tests/CI/installAsterisk.sh
new file mode 100755 (executable)
index 0000000..b75730f
--- /dev/null
@@ -0,0 +1,23 @@
+#!/bin/bash
+MAKE=`which make`
+if [ x"${@}" != x ] ; then
+       mkdir -p "${@}"
+fi
+destdir=${@:+DESTDIR=${@}}
+
+${MAKE} ${destdir} install || ${MAKE} ${destdir} NOISY_BUILD=yes install || exit 1
+${MAKE} ${destdir} samples
+if [ -n "${@}" ] ; then
+       sed -i -r -e "s@\[directories\]\(!\)@[directories]@g" $@/etc/asterisk/asterisk.conf
+       sed -i -r -e "s@ /(var|etc|usr)/@ ${@}/\1/@g" $@/etc/asterisk/asterisk.conf
+fi
+
+set +e
+chown -R jenkins:users ${@}/var/lib/asterisk
+chown -R jenkins:users ${@}/var/spool/asterisk
+chown -R jenkins:users ${@}/var/log/asterisk
+chown -R jenkins:users ${@}/var/run/asterisk
+chown -R jenkins:users ${@}/etc/asterisk
+[ ! -d ${@}/tmp/asterisk-jenkins ] && mkdir ${@}/tmp/asterisk-jenkins
+chown -R jenkins:users ${@}/tmp/asterisk-jenkins
+ldconfig
\ No newline at end of file
diff --git a/tests/CI/runTestsuite.sh b/tests/CI/runTestsuite.sh
new file mode 100755 (executable)
index 0000000..c96b9a4
--- /dev/null
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+pushd $TESTSUITE_DIR
+
+runner sudo PYTHONPATH=./lib/python/ ./runtests.py --cleanup ${TEST_COMMAND} || :
+
+if [ -f asterisk-test-suite-report.xml ]  ; then
+       sudo chown jenkins:users asterisk-test-suite-report.xml
+fi
+
+runner ${CIDIR}/fixTestResults.py asterisk-test-suite-report.xml asterisk-test-suite-report.xml
+
+if [ -f core* ] ; then
+       echo "*** Found a core file after running unit tests ***"
+       sudo /var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+       exit 1
+fi
+
+popd
\ No newline at end of file
diff --git a/tests/CI/runUnittests.sh b/tests/CI/runUnittests.sh
new file mode 100755 (executable)
index 0000000..a463e15
--- /dev/null
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+CIDIR=$(dirname $(readlink -fn $0))
+source $CIDIR/ci.functions
+ASTETCDIR=$DESTDIR/etc/asterisk
+
+echo "full => notice,warning,error,debug,verbose" >    "$ASTETCDIR/logger.conf"
+
+echo "[default]" > "$ASTETCDIR/extensions.conf"
+
+cat <<-EOF > "$ASTETCDIR/manager.conf"
+       [general]
+       enabled=yes
+       bindaddr=127.0.0.1
+       port=5038
+
+       [test]
+       secret=test
+       read = system,call,log,verbose,agent,user,config,dtmf,reporting,cdr,dialplan
+       write = system,call,agent,user,config,command,reporting,originate
+EOF
+
+cat <<-EOF > "$ASTETCDIR/http.conf"
+       [general]
+       enabled=yes
+       bindaddr=127.0.0.1
+       port=8088
+EOF
+
+cat <<-EOF > "$ASTETCDIR/modules.conf"
+       [modules]
+       autoload=yes
+       noload=res_mwi_external.so
+       noload=res_mwi_external_ami.so
+       noload=res_ari_mailboxes.so
+       noload=res_stasis_mailbox.so
+EOF
+
+cat <<-EOF >> "$ASTETCDIR/sorcery.conf"
+       [res_pjsip_pubsub]
+       resource_list=memory
+EOF
+
+ASTERISK="$DESTDIR/usr/sbin/asterisk"
+CONFFILE=$ASTETCDIR/asterisk.conf
+OUTPUTDIR=${OUTPUT_DIR:-tests/CI/output/}
+OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
+
+[ ! -d ${OUTPUTDIR} ] && mkdir -p $OUTPUTDIR
+sudo chown -R jenkins:users $OUTPUTDIR
+
+rm -rf $ASTETCDIR/extensions.{ael,lua} || :
+
+runner sudo $ASTERISK -U jenkins -G users -gn -C $CONFFILE
+sleep 3
+runner $ASTERISK -rx "core waitfullybooted" -C $CONFFILE
+sleep 1
+runner $ASTERISK -rx "${TEST_COMMAND:-test execute all}" -C $CONFFILE
+runner $ASTERISK -rx "test show results failed" -C $CONFFILE
+runner $ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
+runner $ASTERISK -rx "core stop now" -C $CONFFILE
+
+runner rsync -vaH $DESTDIR/var/log/asterisk/. $OUTPUTDIR
+
+sudo chown -R jenkins:users $OUTPUTDIR
+if [ -f core* ] ; then
+       echo "*** Found a core file after running unit tests ***"
+       $DESTDIR/var/lib/asterisk/scripts/ast_coredumper --no-default-search core*
+       exit 1
+fi
diff --git a/tests/CI/setupEnvironment.sh b/tests/CI/setupEnvironment.sh
new file mode 100755 (executable)
index 0000000..eef47a6
--- /dev/null
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+chmod 0750 /etc/sudoers.d
+chmod 0440 /etc/sudoers.d/jenkins
+
+chown root:root -R /root
+chmod -R go-rwx /root/.ssh
+chown -R jenkins:jenkins /home/jenkins
+chown -R jenkins:jenkins /srv/cache
+chown -R jenkins:jenkins /srv/jenkins
+chown -R jenkins:jenkins /srv/git
+chmod -R go-rwx /home/jenkins/.ssh
+chmod -R go-rwx /home/jenkins/.ssh/authorized_keys
diff --git a/tests/CI/unittests.jenkinsfile b/tests/CI/unittests.jenkinsfile
new file mode 100644 (file)
index 0000000..0925910
--- /dev/null
@@ -0,0 +1,170 @@
+/*
+ * This pipeline is the "template" for the Asterisk Unit Tests multi-branch
+ * parent job.  Jenkins will automatically scan the branches in the "asterisk"
+ * or "Security-asterisk" projects in Gerrit and automatically create a branch-
+ * specific job for each branch it finds this file in.
+ *
+ * This file starts as a declarative pipeline because with a declarative
+ * pipeline, you can define the trigger in the pipeline file.  This keeps
+ * everything in one place.  We transition to scripted pipeline later on because
+ * we need to dynamically determine which docker image we're going to use and
+ * you can't do that in a delcarative pipeline.
+ */
+pipeline {
+       triggers {
+               /*
+                * This trigger will match either the "asterisk" or "Security-asterisk"
+                * projects.  The branch is taken from the branch this job was created
+                * for.
+                */
+               gerrit customUrl: '',
+                       commentTextParameterMode: 'PLAIN',
+                       commitMessageParameterMode: 'PLAIN',
+                       gerritBuildStartedVerifiedValue: 0,
+                       gerritBuildNotBuiltVerifiedValue: 0,
+                       gerritBuildSuccessfulVerifiedValue: 1,
+                       gerritBuildFailedVerifiedValue: -1,
+                       gerritBuildUnstableVerifiedValue: -1,
+                       gerritProjects: [
+                               [branches: [[compareType: 'PLAIN', pattern: "${BRANCH_NAME}"]],
+                                       compareType: 'REG_EXP',
+                                       disableStrictForbiddenFileVerification: false,
+                                       pattern: '^(Security-)?asterisk.*'
+                               ]
+                       ],
+                       silentMode: false,
+                       triggerOnEvents: [
+                               commentAddedContains('^recheck$'),
+                               patchsetCreated(excludeDrafts: false,
+                                                               excludeNoCodeChange: true,
+                                                               excludeTrivialRebase: false),
+                               draftPublished()
+                       ],
+                       skipVote: [
+                               onFailed: false,
+                               onNotBuilt: true,
+                               onSuccessful: false,
+                               onUnstable: false
+                       ]
+       }
+       agent {
+               /* All of the stages need to be performed on a docker host */
+               label "swdev-docker"
+       }
+
+       stages {
+               stage ("Unit Tests-->") {
+                       /*
+                        * Jenkins will try to automatically rebuild this job when
+                        * the jenkinsfile changes but since this job is dependent on
+                        * Gerrit, we really don't want to do anything in that case.
+                        */
+                       when {
+                               not { environment name: 'GERRIT_CHANGE_NUMBER', value: '' }
+                       }
+                       steps {
+                               script {
+                                       stage ("Checkout") {
+                                               /*
+                                                * Jenkins has already automatically checked out the base branch
+                                                * for this change but we now need to check out the change itself
+                                                * and rebase it on the current base branch.  If the rebase fails,
+                                                * that's an indication to the user that they'll need to sort their
+                                                * change out.
+                                                *
+                                                * The Gerrit Trigger provides all the URLs and refspecs to
+                                                * check out the change.
+                                                */
+                                               checkout scm: [$class: 'GitSCM',
+                                                       branches: [[name: env.GERRIT_BRANCH ]],
+                                                       extensions: [
+                                                               [$class: 'CleanBeforeCheckout'],
+                                                               [$class: 'PreBuildMerge', options: [
+                                                                       mergeRemote: env.GERRIT_NAME,
+                                                                       mergeTarget: env.GERRIT_BRANCH ]
+                                                               ],
+                                                               [$class: 'CloneOption',
+                                                                       noTags: true,
+                                                                       depth: 10,
+                                                                       honorRefspec: true,
+                                                                       shallow: true
+                                                               ],
+                                                               [$class: 'BuildChooserSetting',
+                                                                       buildChooser: [$class: 'GerritTriggerBuildChooser']
+                                                               ]
+                                                       ],
+                                                       userRemoteConfigs: [
+                                                               [name: env.GERRIT_NAME, refspec: env.GERRIT_REFSPEC, url: env.GIT_URL ]
+                                                       ]
+                                               ]
+                                       }
+
+                                       def images = env.DOCKER_IMAGES.split(' ')
+                                       def r = currentBuild.startTimeInMillis % images.length
+                                       def ri = images[(int)r]
+                                       def randomImage = env.DOCKER_REGISTRY + "/" + ri;
+                                       def dockerOptions =
+                                               "-v /srv/jenkins:/srv/jenkins:rw -v /srv/cache:/srv/cache:rw" +
+                                                " --entrypoint='' --name ${BUILD_TAG}-build"
+
+                                       docker.image(randomImage).inside(dockerOptions) {
+
+                                               stage ('Build') {
+                                                       echo 'Building..'
+                                                       sh './tests/CI/buildAsterisk.sh'
+
+                                                       archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: false,
+                                                               artifacts: "tests/CI/output/*"
+                                               }
+
+                                               stage ('Test') {
+                                                       def outputdir = "tests/CI/output/UnitTests"
+                                                       def outputfile = "${outputdir}/unittests-results.xml"
+                                                       def testcmd = "test execute all"
+
+                                                       sh 'sudo ./tests/CI/installAsterisk.sh'
+                                                       echo "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+                                                       sh "tests/CI/runUnittests.sh --output-dir='${outputdir}' --output-xml='${outputfile}' --test-command='${testcmd}'"
+
+                                                       archiveArtifacts allowEmptyArchive: true, defaultExcludes: false, fingerprint: true,
+                                                               artifacts: "${outputdir}/**"
+
+                                                       junit testResults: outputfile,
+                                                               healthScaleFactor: 1.0,
+                                                               keepLongStdio: true
+                                               }
+                                       }
+
+                                       stage ('Cleanup') {
+                                               sh "sudo make distclean || : "
+                                       }
+                               }
+                       }
+               }
+       }
+       /*
+        * The Gerrit Trigger will automatically post the "Verified" results back
+        * to Gerrit but the verification publisher publishes extra stuff in the
+        * "Code Review" section of the review.
+        */
+       post {
+               success {
+                       gerritverificationpublisher verifyStatusValue: 1, verifyStatusCategory: 'Passed',
+                               verifyStatusURL: '', verifyStatusComment: '',
+                               verifyStatusName: '', verifyStatusReporter: '',
+                               verifyStatusRerun: 'recheck'
+               }
+               failure {
+                       gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+                               verifyStatusURL: '', verifyStatusComment: '',
+                               verifyStatusName: '', verifyStatusReporter: '',
+                               verifyStatusRerun: 'recheck'
+               }
+               unstable {
+                       gerritverificationpublisher verifyStatusValue: -1, verifyStatusCategory: 'Failed',
+                               verifyStatusURL: '', verifyStatusComment: '',
+                               verifyStatusName: '', verifyStatusReporter: '',
+                               verifyStatusRerun: 'recheck'
+               }
+       }
+}