lava-submit.jpl: Add generic retries with backoff
Don't retry twice with adhoc code. Retry using a normal for loop, so number
of retries can be easily adjusted. Use 5 retries for now. Also add backoff
(multiplicative though).
Triggered by seeing a 4+ hour nightly build which failed only because two
back-to-back retries failed.
Signed-off-by: Paul Sokolovsky <paul.sokolovsky@linaro.org>
Change-Id: Ie9618e9c679479279c672acb32b54f3a69f44f9e
diff --git a/jenkins/lava-submit.jpl b/jenkins/lava-submit.jpl
index 88d1af3..f6898a6 100644
--- a/jenkins/lava-submit.jpl
+++ b/jenkins/lava-submit.jpl
@@ -75,14 +75,21 @@
def device_type = filterTestDevice()
withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) {
- print("Generating LAVA jobs...")
- try {
- submitJobs(device_type)
- } catch (Exception ex) {
- print("LAVA-Submit failed! Exception: ${ex}")
- print("Try to submit again...")
- submitJobs(device_type)
- currentBuild.setDescription(currentBuild.getDescription() + " Submitted twice!")
+ print("Submitting LAVA jobs...")
+ def success = false
+ for (i = 0; i < 5; i++) {
+ try {
+ submitJobs(device_type)
+ success = true
+ break
+ } catch (Exception ex) {
+ print("LAVA-Submit failed! Exception: ${ex}")
+ print("Try to submit again...")
+ sleep(i * 3000)
+ }
+ }
+ if (!success) {
+ throw new Exception("LAVA submission failed")
}
}
}