Examples with DeploymentTester com.yahoo.vespa.hosted.controller.deployment.DeploymentTester used on opensource projects

Example 31 with DeploymentTester

use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.

the class UpgraderTest method testBlockVersionChange.

@Test
public void testBlockVersionChange() {
    // Tuesday, 18:00
    ManualClock clock = new ManualClock(Instant.parse("2017-09-26T18:00:00.00Z"));
    DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").blockChange(false, true, "tue", "18-19", "UTC").region("us-west-1").build();
    Application app = tester.createAndDeploy("app1", 1, applicationPackage);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    // Application is not upgraded at this time
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertTrue("No jobs scheduled", tester.deploymentQueue().jobs().isEmpty());
    // One hour passes, time is 19:00, still no upgrade
    tester.clock().advance(Duration.ofHours(1));
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertTrue("No jobs scheduled", tester.deploymentQueue().jobs().isEmpty());
    // Two hours pass in total, time is 20:00 and application upgrades
    tester.clock().advance(Duration.ofHours(1));
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertFalse("Job is scheduled", tester.deploymentQueue().jobs().isEmpty());
    tester.completeUpgrade(app, version, applicationPackage);
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
}

Example 32 with DeploymentTester

use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.

the class UpgraderTest method testUpgradeCancelledWithDeploymentInProgress.

@Test
public void testUpgradeCancelledWithDeploymentInProgress() {
    DeploymentTester tester = new DeploymentTester();
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    // Setup applications
    Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
    Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
    Application default0 = tester.createAndDeploy("default0", 3, "default");
    Application default1 = tester.createAndDeploy("default1", 4, "default");
    Application default2 = tester.createAndDeploy("default2", 5, "default");
    Application default3 = tester.createAndDeploy("default3", 6, "default");
    Application default4 = tester.createAndDeploy("default4", 7, "default");
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Canaries upgrade and raise confidence
    tester.completeUpgrade(canary0, version, "canary");
    tester.completeUpgrade(canary1, version, "canary");
    tester.updateVersionStatus(version);
    assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
    // Applications with default policy start upgrading
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Upgrade scheduled for remaining apps", 5, tester.deploymentQueue().jobs().size());
    // 4/5 applications fail and lowers confidence
    tester.completeUpgradeWithError(default0, version, "default", systemTest);
    tester.completeUpgradeWithError(default1, version, "default", systemTest);
    tester.completeUpgradeWithError(default2, version, "default", systemTest);
    tester.completeUpgradeWithError(default3, version, "default", systemTest);
    tester.updateVersionStatus(version);
    assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // 5th app passes system-test, but does not trigger next job as upgrade is cancelled
    assertFalse("No change present", tester.applications().require(default4.id()).change().isPresent());
    tester.jobCompletion(systemTest).application(default4).submit();
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
}

Example 33 with DeploymentTester

use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.

the class UpgraderTest method testVersionIsBrokenAfterAZoneIsLive.

/**
 * Scenario:
 *   An application A is on version V0
 *   Version V2 is released.
 *   A upgrades one production zone to V2.
 *   V2 is marked as broken and upgrade of A to V2 is cancelled.
 *   Upgrade of A to V1 is scheduled: Should skip the zone on V2 but upgrade the next zone to V1
 */
@Test
public void testVersionIsBrokenAfterAZoneIsLive() {
    DeploymentTester tester = new DeploymentTester();
    Version v0 = Version.fromString("5.0");
    tester.updateVersionStatus(v0);
    // Setup applications on V0
    Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
    Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
    Application default0 = tester.createAndDeploy("default0", 3, "default");
    Application default1 = tester.createAndDeploy("default1", 4, "default");
    Application default2 = tester.createAndDeploy("default2", 5, "default");
    Application default3 = tester.createAndDeploy("default3", 6, "default");
    Application default4 = tester.createAndDeploy("default4", 7, "default");
    // V1 is released
    Version v1 = Version.fromString("5.1");
    tester.updateVersionStatus(v1);
    assertEquals(v1, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Canaries upgrade and raise confidence of V+1 (other apps are not upgraded)
    tester.completeUpgrade(canary0, v1, "canary");
    tester.completeUpgrade(canary1, v1, "canary");
    tester.updateVersionStatus(v1);
    assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
    // V2 is released
    Version v2 = Version.fromString("5.2");
    tester.updateVersionStatus(v2);
    assertEquals(v2, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // We "manually" cancel upgrades to V1 so that we can use the applications to make V2 fail instead
    // But we keep one (default4) to avoid V1 being garbage collected
    tester.deploymentTrigger().cancelChange(default0.id(), false);
    tester.deploymentTrigger().cancelChange(default1.id(), false);
    tester.deploymentTrigger().cancelChange(default2.id(), false);
    tester.deploymentTrigger().cancelChange(default3.id(), false);
    // Currently we don't cancel running jobs, so this is necessary to allow a new triggering below
    tester.clock().advance(Duration.ofHours(13));
    // Canaries upgrade and raise confidence of V2
    tester.completeUpgrade(canary0, v2, "canary");
    tester.completeUpgrade(canary1, v2, "canary");
    tester.updateVersionStatus(v2);
    assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
    // Applications with default policy start upgrading to V2
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Upgrade scheduled for remaining apps", 5, tester.deploymentQueue().jobs().size());
    // 4/5 applications fail (in the last prod zone) and lowers confidence
    tester.completeUpgradeWithError(default0, v2, "default", DeploymentJobs.JobType.productionUsEast3);
    tester.completeUpgradeWithError(default1, v2, "default", DeploymentJobs.JobType.productionUsEast3);
    tester.completeUpgradeWithError(default2, v2, "default", DeploymentJobs.JobType.productionUsEast3);
    tester.completeUpgradeWithError(default3, v2, "default", DeploymentJobs.JobType.productionUsEast3);
    tester.updateVersionStatus(v2);
    assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
    assertEquals(v2, tester.application("default0").deployments().get(ZoneId.from("prod.us-west-1")).version());
    assertEquals(v0, tester.application("default0").deployments().get(ZoneId.from("prod.us-east-3")).version());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Upgrade to 5.1 scheduled for apps not completely on 5.1 or 5.2", 5, tester.deploymentQueue().jobs().size());
    tester.deploymentTrigger().triggerReadyJobs();
    assertEquals("Testing of 5.1 for 5 applications is triggered", 5, tester.deploymentQueue().jobs().size());
    assertEquals(systemTest.jobName(), tester.deploymentQueue().jobs().get(0).jobName());
    assertEquals(systemTest.jobName(), tester.deploymentQueue().jobs().get(1).jobName());
    assertEquals(systemTest.jobName(), tester.deploymentQueue().jobs().get(2).jobName());
    assertEquals(systemTest.jobName(), tester.deploymentQueue().jobs().get(3).jobName());
    assertEquals(systemTest.jobName(), tester.deploymentQueue().jobs().get(4).jobName());
    // The tester code for completing upgrades does not handle this scenario, so we trigger each step manually (for one app)
    tester.deployAndNotify(tester.application("default0"), "default", true, systemTest);
    tester.deployAndNotify(tester.application("default0"), "default", true, DeploymentJobs.JobType.stagingTest);
    // prod zone on 5.2 (usWest1) is skipped, but we still trigger the next zone from triggerReadyJobs:
    // Currently we don't cancel running jobs, so this is necessary to allow a new triggering below
    tester.clock().advance(Duration.ofHours(13));
    tester.deploymentTrigger().triggerReadyJobs();
    tester.deployAndNotify(tester.application("default0"), "default", true, DeploymentJobs.JobType.productionUsEast3);
    // Resulting state:
    assertEquals(v2, tester.application("default0").deployments().get(ZoneId.from("prod.us-west-1")).version());
    assertEquals("Last zone is upgraded to v1", v1, tester.application("default0").deployments().get(ZoneId.from("prod.us-east-3")).version());
    assertFalse(tester.application("default0").change().isPresent());
}

Example 34 with DeploymentTester

use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.

the class UpgraderTest method testBlockVersionChangeHalfwayThough.

@Test
public void testBlockVersionChangeHalfwayThough() {
    // Tuesday, 17:00
    ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z"));
    DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
    ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), Duration.ofHours(1), new JobControl(tester.controllerTester().curator()));
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").blockChange(false, true, "tue", "18-19", "UTC").region("us-west-1").region("us-central-1").region("us-east-3").build();
    Application app = tester.createAndDeploy("app1", 1, applicationPackage);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    // Application upgrade starts
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    tester.deployAndNotify(app, applicationPackage, true, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    // Entering block window after prod job is triggered
    clock.advance(Duration.ofHours(1));
    tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
    // Next job not triggered due to being in the block window
    assertTrue(tester.deploymentQueue().jobs().isEmpty());
    // One hour passes, time is 19:00, still no upgrade
    tester.clock().advance(Duration.ofHours(1));
    readyJobsTrigger.maintain();
    assertTrue("No jobs scheduled", tester.deploymentQueue().jobs().isEmpty());
    // Another hour pass, time is 20:00 and application upgrades
    tester.clock().advance(Duration.ofHours(1));
    readyJobsTrigger.maintain();
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
}

Example 35 with DeploymentTester

use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.

the class FailureRedeployerTest method testRetriesJobsFailingForCurrentChange.

@Test
public void testRetriesJobsFailingForCurrentChange() {
    DeploymentTester tester = new DeploymentTester();
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").environment(Environment.prod).region("us-east-3").build();
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    Application app = tester.createApplication("app1", "tenant1", 1, 11L);
    tester.jobCompletion(component).application(app).uploadArtifact(applicationPackage).submit();
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
    // system-test fails and is left with a retry
    tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest);
    // Another version is released
    version = Version.fromString("5.2");
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    // Job is left "running", so needs to time out before it can be retried.
    tester.clock().advance(Duration.ofHours(13));
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Application has pending upgrade to " + version, version, tester.application(app.id()).change().platform().get());
    // Cancellation of outdated version and triggering on a new version is done by the upgrader.
    assertEquals(version, tester.application(app.id()).deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().version());
}