JJB: Add Jenkins job builder script
Signed-off-by: Arthur She <arthur.she@linaro.org>
Change-Id: Ic9a1d58dcd6e9cb72aa91d31e16ec2c42b6b54da
diff --git a/ci/run-jjb.py b/ci/run-jjb.py
new file mode 100755
index 0000000..731ab18
--- /dev/null
+++ b/ci/run-jjb.py
@@ -0,0 +1,183 @@
+#!/usr/bin/python3
+
+import os
+import shutil
+import signal
+import string
+import subprocess
+import sys
+import xml.etree.ElementTree
+from distutils.spawn import find_executable
+
+
+def findparentfiles(fname):
+ filelist = []
+ newlist = []
+ args = ['grep', '-rl', '--exclude-dir=.git', fname]
+ proc = subprocess.run(args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ return filelist
+ for filename in data.splitlines():
+ if filename.endswith('.yaml') and '/' not in filename:
+ filelist.append(filename)
+ else:
+ newlist = findparentfiles(filename)
+ for tempname in newlist:
+ filelist.append(tempname)
+ return filelist
+
+
+jjb_cmd = find_executable('jenkins-jobs') or sys.exit('jenkins-jobs is not found.')
+jjb_args = [jjb_cmd]
+
+jjb_user = os.environ.get('JJB_USER')
+jjb_password = os.environ.get('JJB_PASSWORD')
+if jjb_user is not None and jjb_password is not None:
+ jenkins_jobs_ini = ('[job_builder]\n'
+ 'ignore_cache=True\n'
+ 'keep_descriptions=False\n'
+ '\n'
+ '[jenkins]\n'
+ 'user=%s\n'
+ 'password=%s\n'
+ 'url=https://ci.trustedfirmware.org/\n' % (jjb_user, jjb_password))
+ with open('jenkins_jobs.ini', 'w') as f:
+ f.write(jenkins_jobs_ini)
+ jjb_args.append('--conf=jenkins_jobs.ini')
+
+jjb_test_args = list(jjb_args)
+jjb_delete_args = list(jjb_args)
+
+# !!! "update" below and through out this file is replaced by "test" (using sed)
+# !!! in the sanity-check job.
+main_action = 'update'
+jjb_args.extend([main_action, 'template.yaml'])
+jjb_test_args.extend(['test', '-o', 'out/', 'template.yaml'])
+jjb_delete_args.extend(['delete'])
+
+if main_action == 'test':
+ # Dry-run, don't delete jobs.
+ jjb_delete_args.insert(0, 'echo')
+
+try:
+ git_args = ['git', 'diff', '--raw',
+ os.environ.get('GIT_PREVIOUS_COMMIT'),
+ os.environ.get('GIT_COMMIT')]
+ proc = subprocess.run(git_args, capture_output=True)
+except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+data = proc.stdout.decode()
+if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+filelist = []
+deletelist = []
+files = []
+for line in data.splitlines():
+ # Format of the git-diff; we only need OPERATION and FILE1
+ #
+ # :<OLD MODE> <NEW MODE> <OLD REF> <NEW REF> <OPERATION> <FILE1> <FILE2>
+ elems = line.split()
+ operation = elems[4][0]
+ filename = elems[5]
+
+ if filename.endswith('.yaml') and '/' not in filename:
+ # No point trying to test deleted jobs because they don't exist any
+ # more.
+ if operation == 'D':
+ deletelist.append(filename[:-5])
+ continue
+ # operation R100 is 100% rename, which means sixth element is the renamed file
+ if operation == 'R':
+ filename = elems[6]
+ # delete old job name
+ deletelist.append(elems[5][:-5])
+ filelist.append(filename)
+ else:
+ files = findparentfiles(filename)
+ for tempname in files:
+ filelist.append(tempname)
+
+# Remove duplicate entries in the list
+filelist = list(set(filelist))
+
+for conf_filename in filelist:
+ with open(conf_filename) as f:
+ buffer = f.read()
+ template = string.Template(buffer)
+ buffer = template.safe_substitute(
+ AUTH_TOKEN=os.environ.get('AUTH_TOKEN'),
+ LT_QCOM_KEY=os.environ.get('LT_QCOM_KEY'),
+ LAVA_USER=os.environ.get('LAVA_USER'),
+ LAVA_TOKEN=os.environ.get('LAVA_TOKEN'))
+ with open('template.yaml', 'w') as f:
+ f.write(buffer)
+ try:
+ proc = subprocess.run(jjb_args, capture_output=True)
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ try:
+ shutil.rmtree('out/', ignore_errors=True)
+
+ proc = subprocess.run(jjb_test_args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ proc = subprocess.run(['ls', 'out/'], capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ for filename in data.splitlines():
+ # old job conf might have been removed because the job is now generated through the template
+ # do not delete the job in this case
+ if filename in deletelist:
+ deletelist.remove(filename)
+
+ conf_name=os.path.splitext(conf_filename)[0]
+ conf_name=conf_name[:len(filename)]
+ if not filename.startswith(conf_name):
+ raise ValueError("Job name %s does not match the file it is in: %s" % (filename, conf_name))
+ try:
+ xmlroot = xml.etree.ElementTree.parse('out/' + filename).getroot()
+ disabled = next(xmlroot.iterfind('disabled')).text
+ if disabled != 'true':
+ continue
+ displayName = next(xmlroot.iterfind('displayName')).text
+ if displayName != 'DELETE ME':
+ continue
+ except:
+ continue
+
+ deletelist.append(filename)
+
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+ shutil.rmtree('out/', ignore_errors=True)
+ os.remove('template.yaml')
+
+
+for deletejob in deletelist:
+ delete_args = list(jjb_delete_args)
+ delete_args.extend([deletejob])
+ try:
+ proc = subprocess.run(delete_args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+ print(data)
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+if os.path.exists('jenkins_jobs.ini'):
+ os.remove('jenkins_jobs.ini')
+