use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class FailureRedeployerTest method testRetryingFailedJobsDuringDeployment.
@Test
public void testRetryingFailedJobsDuringDeployment() {
DeploymentTester tester = new DeploymentTester();
ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").environment(Environment.prod).region("us-east-3").build();
Version version = Version.fromString("5.0");
tester.updateVersionStatus(version);
Application app = tester.createApplication("app1", "tenant1", 1, 11L);
tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit();
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
// New version is released
version = Version.fromString("5.1");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
// Test environments pass
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
// Production job fails and is retried
// Advance time so that we can detect jobs in progress
tester.clock().advance(Duration.ofSeconds(1));
tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
assertEquals("Production job is retried", 1, tester.deploymentQueue().jobs().size());
assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
// Another version is released, which cancels any pending upgrades to lower versions
version = Version.fromString("5.2");
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Application starts upgrading to new version", 1, tester.deploymentQueue().jobs().size());
assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
// Failure re-deployer does not retry failing job for prod.us-east-3, since it no longer has an available change
tester.clock().advance(Duration.ofMinutes(1));
tester.jobCompletion(DeploymentJobs.JobType.productionUsEast3).application(app).unsuccessful().submit();
assertFalse("Job is not retried", tester.deploymentQueue().jobs().stream().anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName())));
// Test environments pass
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
// Production job fails again, and is retried
tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
tester.readyJobTrigger().maintain();
assertEquals("Job is retried", Collections.singletonList(new BuildService.BuildJob(app.deploymentJobs().projectId().get(), productionUsEast3.jobName())), tester.deploymentQueue().jobs());
// Production job finally succeeds
tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures());
}
Aggregations