diff options
author | Benjamin Copeland <ben.copeland@linaro.org> | 2019-04-02 08:54:42 +0100 |
---|---|---|
committer | Benjamin Copeland <ben.copeland@linaro.org> | 2019-04-02 08:54:42 +0100 |
commit | c78807dcc1e758e1bad54a9b3a9bbe395b4bd71c (patch) | |
tree | 9bb401ad7335960ced9c4ebff16f2b4815aa0b46 | |
parent | 36edb91885b4dd528947acca608054c5ca201e1e (diff) | |
download | tf-ci-scripts-c78807dcc1e758e1bad54a9b3a9bbe395b4bd71c.tar.gz |
Dockerfiles: Setup inital dockerfile deployment
Change-Id: If11aea463c03eb8032886632b3c94c5f4b50890b
-rw-r--r-- | ci-dockerfiles-deployment.yaml | 71 | ||||
-rw-r--r-- | ci-dockerfiles-deployment/build-image.sh | 30 | ||||
-rw-r--r-- | ci-dockerfiles-deployment/builders.sh | 99 | ||||
-rwxr-xr-x | ci-dockerfiles-deployment/update-images.sh | 42 |
4 files changed, 242 insertions, 0 deletions
diff --git a/ci-dockerfiles-deployment.yaml b/ci-dockerfiles-deployment.yaml new file mode 100644 index 0000000..d1a2f04 --- /dev/null +++ b/ci-dockerfiles-deployment.yaml @@ -0,0 +1,71 @@ +- job: + name: ci-dockerfiles-deployment + project-type: matrix + defaults: global + properties: + - authorization: + anonymous: + - job-discover + linaro: + - job-read + - job-extended-read + - build-discarder: + days-to-keep: 90 + num-to-keep: 200 + disabled: false + node: master + concurrent: false + display-name: 'CI Dockerfiles build/publish deployment' + scm: + - git: + url: https://review.trustedfirmware.org/${GERRIT_PROJECT} + refspec: ${GERRIT_REFSPEC} + branches: + - ${GERRIT_BRANCH} + skip-tag: true + clean: + before: true + choosing-strategy: gerrit + basedir: dockerfiles + triggers: + - gerrit: + server-name: 'review.trustedfirmware.org' + trigger-on: + - change-merged-event + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'ci/dockerfiles' + branches: + - branch-compare-type: 'PLAIN' + branch-pattern: 'master' + silent-start: true + axes: + - axis: + type: slave + name: label + values: + - build-amd64 + execution-strategy: + sequential: false + wrappers: + - timestamps + - copy-to-slave: + includes: + - config.json + relative-to: 'somewhereElse' + - credentials-binding: + - text: + credential-id: DOCKER_AUTH + variable: DOCKER_AUTH + builders: + - shell: | + #!/bin/bash -e + echo "#${BUILD_NUMBER}-${GERRIT_PATCHSET_REVISION:0:8}" > ${WORKSPACE}/version.txt + - build-name-setter: + name: 'version.txt' + file: true + - shell: + !include-raw: ci-dockerfiles-deployment/builders.sh + publishers: + - email: + recipients: ben.copeland@linaro.org diff --git a/ci-dockerfiles-deployment/build-image.sh b/ci-dockerfiles-deployment/build-image.sh new file mode 100644 index 0000000..277096d --- /dev/null +++ b/ci-dockerfiles-deployment/build-image.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -ex + +trap cleanup_exit INT TERM EXIT + +cleanup_exit() +{ + rm -rf ${HOME}/.docker dockerfiles +} + +mkdir -p ${HOME}/.docker +sed -e "s|\${DOCKER_AUTH}|${DOCKER_AUTH}|" < ${WORKSPACE}/config.json > ${HOME}/.docker/config.json +chmod 0600 ${HOME}/.docker/config.json + +rm -rf dockerfiles/ +git clone --depth 1 https://git.trustedfirmware.org/ci/dockerfiles.git/ + +cd dockerfiles/${image}/ +if ! ./build.sh; then + echo "=== FAIL: ${image} ===" + exit 1 +fi + +# now we have image name in .docker-tag +if [ -r .docker-tag ]; then + docker_tag=$(cat .docker-tag) + docker push ${docker_tag} +fi + diff --git a/ci-dockerfiles-deployment/builders.sh b/ci-dockerfiles-deployment/builders.sh new file mode 100644 index 0000000..0cfd7a5 --- /dev/null +++ b/ci-dockerfiles-deployment/builders.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +set -ex + +trap cleanup_exit INT TERM EXIT + +cleanup_exit() +{ + rm -rf ${HOME}/.docker + rm -f ${WORKSPACE}/{log,config.json,version.txt} +} + +mkdir -p ${HOME}/.docker +sed -e "s|\${DOCKER_AUTH}|${DOCKER_AUTH}|" < ${WORKSPACE}/config.json > ${HOME}/.docker/config.json +chmod 0600 ${HOME}/.docker/config.json + +echo "" +echo "########################################################################" +echo " Gerrit Environment" +env |grep '^GERRIT' +echo "########################################################################" + +rm -f ${WORKSPACE}/{log,config.json,version.txt} +cd dockerfiles/ + +git_previous_commit=$(git rev-parse HEAD~1) +git_commit=$(git rev-parse HEAD) +files=$(git diff --name-only ${git_previous_commit} ${git_commit}) +echo Changes in: ${files} +changed_dirs=$(dirname ${files}) + +update_images="" +for dir in ${changed_dirs}; do + # Find the closest directory with build.sh. This is, primarily, + # to handle changes to tcwg-base/tcwg-build/tcwg-builslave/* directories. + while [ ! -e ${dir}/build.sh -a ! -e ${dir}/.git ]; do + dir=$(dirname ${dir}) + done + # Add this and all dependant images in the update. + dir_basename=$(basename ${dir}) + case "${dir_basename}" in + "tcwg-"*) + # ${dir} is one of generic tcwg-base/* directories. Add dependent + # images to the list. + update_images="${update_images} $(dirname $(find . -path "*-${dir_basename}*/build.sh" | sed -e "s#^\./##g"))" + ;; + *) + update_images="${update_images} $(dirname $(find ${dir} -name build.sh))" + ;; + esac +done +update_images="$(echo "${update_images}" | tr " " "\n" | sort -u)" + +host_arch=$(dpkg-architecture -qDEB_HOST_ARCH) + +for image in ${update_images}; do + ( + cd ${image} + image_arch=$(basename ${PWD} | cut -f2 -d '-') + skip="skip" + if [ -f gerrit-branches ]; then + # Build only from branches mentioned in gerrit-branches + if grep -q "^${GERRIT_BRANCH}\$" gerrit-branches; then + skip="no" + fi + elif [ x"${GERRIT_BRANCH}" = x"master" ]; then + # No gerrit-branch file, so build only from "master" branch. + skip="no" + fi + case "${skip}:${host_arch}:${image_arch}" in + "skip:"*) + echo "Skipping: don't need to build ${image} on branch ${GERRIT_BRANCH}" + ;; + "no:amd64:amd64"|"no:amd64:i386"|"no:arm64:arm64"|"no:armhf:armhf") + echo "=== Start build: ${image} ===" + ./build.sh || echo "=== FAIL: ${image} ===" >> ${WORKSPACE}/log + ;; + *) + echo "Skipping: can't build for ${image_arch} on ${host_arch}" + ;; + esac + if [ -r .docker-tag ]; then + docker_tag=$(cat .docker-tag) + if [ x"${GERRIT_BRANCH}" != x"master" ]; then + new_tag=${docker_tag}-${GERRIT_BRANCH} + docker tag ${docker_tag} ${new_tag} + docker_tag=${new_tag} + fi + docker push ${docker_tag} + fi + )||echo $image failed >> ${WORKSPACE}/log +done + +if [ -e ${WORKSPACE}/log ] +then + echo "some images failed:" + cat ${WORKSPACE}/log + exit 1 +fi diff --git a/ci-dockerfiles-deployment/update-images.sh b/ci-dockerfiles-deployment/update-images.sh new file mode 100755 index 0000000..d4b9058 --- /dev/null +++ b/ci-dockerfiles-deployment/update-images.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e + +cd dockerfiles/ + +images_to_update="" + +# find out which images haven't had any commits in the last 30 days + +for dir in ./*/; do + shortdir=$(basename $dir) + # Skip toolchain images + echo $shortdir|grep -q tcwg && continue + # not an image dir + [ -x $shortdir/build.sh ]||continue + changed=$(git log -1 --oneline --since "1 month" ${shortdir}|wc -l) + if [ $changed -eq 1 ]; then + echo "new: $shortdir" + else + echo "nothing new: $shortdir" + images_to_update="$images_to_update $shortdir" + fi +done + +echo $images_to_update + +# trigger builds for every non-updated image over the http api +for image in $images_to_update +do + arch=$(echo ${image} | cut -f2 -d '-') + if [ "$arch" = "aarch64" ]; then + arch=arm64 + fi + if [ "$arch" = "amd64" -o "$arch" = "arm64" -o "$arch" = "armhf" ]; then + cat > ../docker_${image}_build.txt << EOF +nodelabel=build-${arch} +image=${image} +EOF + else + echo "unknown arch: $arch in $image" + fi +done + |