Search in sources :

Example 6 with DeploymentJobs

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.

the class FailureRedeployerTest method testRetryingFailedJobsDuringDeployment.

@Test
public void testRetryingFailedJobsDuringDeployment() {
    DeploymentTester tester = new DeploymentTester();
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").environment(Environment.prod).region("us-east-3").build();
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    Application app = tester.createApplication("app1", "tenant1", 1, 11L);
    tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit();
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Test environments pass
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    // Production job fails and is retried
    // Advance time so that we can detect jobs in progress
    tester.clock().advance(Duration.ofSeconds(1));
    tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
    assertEquals("Production job is retried", 1, tester.deploymentQueue().jobs().size());
    assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
    // Another version is released, which cancels any pending upgrades to lower versions
    version = Version.fromString("5.2");
    tester.updateVersionStatus(version);
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Application starts upgrading to new version", 1, tester.deploymentQueue().jobs().size());
    assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
    // Failure re-deployer does not retry failing job for prod.us-east-3, since it no longer has an available change
    tester.clock().advance(Duration.ofMinutes(1));
    tester.jobCompletion(DeploymentJobs.JobType.productionUsEast3).application(app).unsuccessful().submit();
    assertFalse("Job is not retried", tester.deploymentQueue().jobs().stream().anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.jobName())));
    // Test environments pass
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    // Production job fails again, and is retried
    tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
    tester.readyJobTrigger().maintain();
    assertEquals("Job is retried", Collections.singletonList(new BuildService.BuildJob(app.deploymentJobs().projectId().get(), productionUsEast3.jobName())), tester.deploymentQueue().jobs());
    // Production job finally succeeds
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
    assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures());
}
Also used : Collection(com.yahoo.log.event.Collection) Version(com.yahoo.component.Version) Files(java.nio.file.Files) Environment(com.yahoo.config.provision.Environment) Slime(com.yahoo.slime.Slime) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) DeploymentJobs(com.yahoo.vespa.hosted.controller.application.DeploymentJobs) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) ZoneId(com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId) SlimeUtils(com.yahoo.vespa.config.SlimeUtils) Paths(java.nio.file.Paths) Assert.assertFalse(org.junit.Assert.assertFalse) Duration(java.time.Duration) SystemName(com.yahoo.config.provision.SystemName) BuildService(com.yahoo.vespa.hosted.controller.api.integration.BuildService) JobType.productionUsEast3(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3) Collections(java.util.Collections) JobType.component(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component) Application(com.yahoo.vespa.hosted.controller.Application) Assert.assertEquals(org.junit.Assert.assertEquals) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) Version(com.yahoo.component.Version) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Application(com.yahoo.vespa.hosted.controller.Application) Test(org.junit.Test) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest)

Aggregations

DeploymentJobs (com.yahoo.vespa.hosted.controller.application.DeploymentJobs)6 Application (com.yahoo.vespa.hosted.controller.Application)5 Version (com.yahoo.component.Version)4 ApplicationId (com.yahoo.config.provision.ApplicationId)4 Environment (com.yahoo.config.provision.Environment)4 ApplicationPackage (com.yahoo.vespa.hosted.controller.application.ApplicationPackage)4 Change (com.yahoo.vespa.hosted.controller.application.Change)4 Deployment (com.yahoo.vespa.hosted.controller.application.Deployment)4 Duration (java.time.Duration)4 DeploymentSpec (com.yahoo.config.application.api.DeploymentSpec)3 SlimeUtils (com.yahoo.vespa.config.SlimeUtils)3 TenantId (com.yahoo.vespa.hosted.controller.api.identifiers.TenantId)3 BuildService (com.yahoo.vespa.hosted.controller.api.integration.BuildService)3 ZoneId (com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId)3 ApplicationVersion (com.yahoo.vespa.hosted.controller.application.ApplicationVersion)3 JobStatus (com.yahoo.vespa.hosted.controller.application.JobStatus)3 SourceRevision (com.yahoo.vespa.hosted.controller.application.SourceRevision)3 Collections (java.util.Collections)3 List (java.util.List)3 Optional (java.util.Optional)3