use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.
the class DnsMaintainerTest method removes_record_for_unassigned_rotation.
@Test
public void removes_record_for_unassigned_rotation() {
DeploymentTester tester = new DeploymentTester();
Application application = tester.createApplication("app1", "tenant1", 1, 1L);
DnsMaintainer dnsMaintainer = new DnsMaintainer(tester.controller(), Duration.ofHours(12), new JobControl(new MockCuratorDb()), tester.controllerTester().nameService());
ApplicationPackage applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod).globalServiceId("foo").region("us-west-1").region("us-central-1").build();
// Deploy application
tester.deployCompletely(application, applicationPackage);
assertEquals(2, tester.controllerTester().nameService().records().size());
Optional<Record> record = tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com"));
assertTrue(record.isPresent());
assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString());
assertEquals("rotation-fqdn-01.", record.get().data().asString());
record = tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com"));
assertTrue(record.isPresent());
assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString());
assertEquals("rotation-fqdn-01.", record.get().data().asString());
// DnsMaintainer does nothing
dnsMaintainer.maintain();
assertTrue("DNS record is not removed", tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com")).isPresent());
assertTrue("DNS record is not removed", tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com")).isPresent());
// Remove application
applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod).allow(ValidationId.deploymentRemoval).build();
tester.jobCompletion(component).application(application).nextBuildNumber().uploadArtifact(applicationPackage).submit();
tester.deployAndNotify(application, applicationPackage, true, systemTest);
tester.applications().deactivate(application, ZoneId.from(Environment.test, RegionName.from("us-east-1")));
tester.applications().deactivate(application, ZoneId.from(Environment.staging, RegionName.from("us-east-3")));
tester.applications().deleteApplication(application.id(), Optional.of(new NToken("ntoken")));
// DnsMaintainer removes records
dnsMaintainer.maintain();
assertFalse("DNS record removed", tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1--tenant1.global.vespa.yahooapis.com")).isPresent());
dnsMaintainer.maintain();
assertFalse("DNS record removed", tester.controllerTester().nameService().findRecord(Record.Type.CNAME, RecordName.from("app1.tenant1.global.vespa.yahooapis.com")).isPresent());
}
use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.
the class MetricsReporterTest method test_deployment_average_duration.
@Test
public void test_deployment_average_duration() {
DeploymentTester tester = new DeploymentTester();
ApplicationPackage applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod).region("us-west-1").build();
MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.cd);
Application app = tester.createApplication("app1", "tenant1", 1, 11L);
tester.deployCompletely(app, applicationPackage);
reporter.maintain();
// An exceptionally fast deployment :-)
assertEquals(Duration.ZERO, getAverageDeploymentDuration(app));
// App spends 3 hours deploying
tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit();
tester.clock().advance(Duration.ofHours(1));
tester.deployAndNotify(app, applicationPackage, true, systemTest);
tester.clock().advance(Duration.ofMinutes(30));
tester.deployAndNotify(app, applicationPackage, true, stagingTest);
tester.clock().advance(Duration.ofMinutes(90));
tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
reporter.maintain();
// Average time is 1 hour
assertEquals(Duration.ofHours(1), getAverageDeploymentDuration(app));
// Another deployment starts and stalls for 12 hours
tester.jobCompletion(component).application(app).nextBuildNumber(2).uploadArtifact(applicationPackage).submit();
tester.clock().advance(Duration.ofHours(12));
reporter.maintain();
assertEquals(// hanging system-test
Duration.ofHours(12).plus(// previous staging-test
Duration.ofMinutes(30)).plus(// previous production job
Duration.ofMinutes(90)).dividedBy(// Total number of orchestrated jobs
3), getAverageDeploymentDuration(app));
}
use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.
the class UpgraderTest method testUpgrading.
@Test
public void testUpgrading() {
// --- Setup
DeploymentTester tester = new DeploymentTester();
Version version = Version.fromString("5.0");
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("No applications: Nothing to do", 0, tester.deploymentQueue().jobs().size());
// Setup applications
Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
Application default0 = tester.createAndDeploy("default0", 3, "default");
Application default1 = tester.createAndDeploy("default1", 4, "default");
Application default2 = tester.createAndDeploy("default2", 5, "default");
Application conservative0 = tester.createAndDeploy("conservative0", 6, "conservative");
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("All already on the right version: Nothing to do", 0, tester.deploymentQueue().jobs().size());
// --- 5.1 is released - everything goes smoothly
version = Version.fromString("5.1");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("New system version: Should upgrade Canaries", 2, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary0, version, "canary");
assertEquals(version, tester.configServer().lastPrepareVersion().get());
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("One canary pending; nothing else", 1, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary1, version, "canary");
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Canaries done: Should upgrade defaults", 3, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(default0, version, "default");
tester.completeUpgrade(default1, version, "default");
tester.completeUpgrade(default2, version, "default");
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Normals done: Should upgrade conservatives", 1, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(conservative0, version, "conservative");
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Nothing to do", 0, tester.deploymentQueue().jobs().size());
// --- 5.2 is released - which fails a Canary
version = Version.fromString("5.2");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("New system version: Should upgrade Canaries", 2, tester.deploymentQueue().jobs().size());
tester.completeUpgradeWithError(canary0, version, "canary", DeploymentJobs.JobType.stagingTest);
assertEquals("Other Canary was cancelled", 2, tester.deploymentQueue().jobs().size());
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Version broken, but Canaries should keep trying", 2, tester.deploymentQueue().jobs().size());
// --- A new version is released - which repairs the Canary app and fails a default
tester.clock().advance(Duration.ofHours(13));
version = Version.fromString("5.3");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("New system version: Should upgrade Canaries", 2, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary0, version, "canary");
assertEquals(version, tester.configServer().lastPrepareVersion().get());
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("One canary pending; nothing else", 1, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary1, version, "canary");
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Canaries done: Should upgrade defaults", 3, tester.deploymentQueue().jobs().size());
tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest);
tester.completeUpgrade(default1, version, "default");
tester.completeUpgrade(default2, version, "default");
tester.updateVersionStatus(version);
assertEquals("Not enough evidence to mark this as neither broken nor high", VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
assertEquals("Upgrade with error should retry", 1, tester.deploymentQueue().jobs().size());
// --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold
// Deploy application change
tester.deploymentQueue().takeJobsToRun();
tester.jobCompletion(component).application(default0).nextBuildNumber().uploadArtifact(DeploymentTester.applicationPackage("default")).submit();
tester.jobCompletion(stagingTest).application(default0).unsuccessful().submit();
tester.deployAndNotify(default0, "default", true, systemTest);
tester.deployAndNotify(default0, "default", true, stagingTest);
tester.deployAndNotify(default0, "default", true, productionUsWest1);
tester.deployAndNotify(default0, "default", true, productionUsEast3);
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.high, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Normals done: Should upgrade conservatives", 1, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(conservative0, version, "conservative");
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Applications are on 5.3 - nothing to do", 0, tester.deploymentQueue().jobs().size());
// --- Starting upgrading to a new version which breaks, causing upgrades to commence on the previous version
Version version54 = Version.fromString("5.4");
// need 4 to break a version
Application default3 = tester.createAndDeploy("default3", 5, "default");
Application default4 = tester.createAndDeploy("default4", 5, "default");
tester.updateVersionStatus(version54);
// cause canary upgrades to 5.4
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
tester.completeUpgrade(canary0, version54, "canary");
tester.completeUpgrade(canary1, version54, "canary");
tester.updateVersionStatus(version54);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Upgrade of defaults are scheduled", 5, tester.deploymentQueue().jobs().size());
assertEquals(version54, tester.application(default0.id()).change().platform().get());
assertEquals(version54, tester.application(default1.id()).change().platform().get());
assertEquals(version54, tester.application(default2.id()).change().platform().get());
assertEquals(version54, tester.application(default3.id()).change().platform().get());
assertEquals(version54, tester.application(default4.id()).change().platform().get());
tester.completeUpgrade(default0, version54, "default");
// State: Default applications started upgrading to 5.4 (and one completed)
Version version55 = Version.fromString("5.5");
tester.updateVersionStatus(version55);
// cause canary upgrades to 5.5
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
tester.completeUpgrade(canary0, version55, "canary");
tester.completeUpgrade(canary1, version55, "canary");
tester.updateVersionStatus(version55);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Upgrade of defaults are scheduled", 5, tester.deploymentQueue().jobs().size());
assertEquals(version55, tester.application(default0.id()).change().platform().get());
assertEquals(version54, tester.application(default1.id()).change().platform().get());
assertEquals(version54, tester.application(default2.id()).change().platform().get());
assertEquals(version54, tester.application(default3.id()).change().platform().get());
assertEquals(version54, tester.application(default4.id()).change().platform().get());
tester.completeUpgrade(default1, version54, "default");
tester.completeUpgrade(default2, version54, "default");
tester.completeUpgradeWithError(default3, version54, "default", DeploymentJobs.JobType.stagingTest);
tester.completeUpgradeWithError(default4, version54, "default", DeploymentJobs.JobType.productionUsWest1);
// State: Default applications started upgrading to 5.5
tester.clock().advance(Duration.ofHours(13));
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
tester.completeUpgradeWithError(default0, version55, "default", DeploymentJobs.JobType.stagingTest);
tester.completeUpgradeWithError(default1, version55, "default", DeploymentJobs.JobType.stagingTest);
tester.completeUpgradeWithError(default2, version55, "default", DeploymentJobs.JobType.stagingTest);
tester.completeUpgradeWithError(default3, version55, "default", DeploymentJobs.JobType.productionUsWest1);
tester.updateVersionStatus(version55);
assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
// Finish running job, without retry.
tester.clock().advance(Duration.ofHours(1));
tester.jobCompletion(DeploymentJobs.JobType.productionUsWest1).application(default3).unsuccessful().submit();
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken: " + "This is default3 since it failed upgrade on both 5.4 and 5.5", 1, tester.deploymentQueue().jobs().size());
assertEquals("5.4", tester.application(default3.id()).change().platform().get().toString());
}
use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.
the class UpgraderTest method testReschedulesUpgradeAfterTimeout.
@Test
public void testReschedulesUpgradeAfterTimeout() {
DeploymentTester tester = new DeploymentTester();
Version version = Version.fromString("5.0");
tester.updateVersionStatus(version);
ApplicationPackage canaryApplicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").environment(Environment.prod).region("us-west-1").build();
ApplicationPackage defaultApplicationPackage = new ApplicationPackageBuilder().upgradePolicy("default").environment(Environment.prod).region("us-west-1").build();
// Setup applications
Application canary0 = tester.createAndDeploy("canary0", 1, canaryApplicationPackage);
Application canary1 = tester.createAndDeploy("canary1", 2, canaryApplicationPackage);
Application default0 = tester.createAndDeploy("default0", 3, defaultApplicationPackage);
Application default1 = tester.createAndDeploy("default1", 4, defaultApplicationPackage);
Application default2 = tester.createAndDeploy("default2", 5, defaultApplicationPackage);
Application default3 = tester.createAndDeploy("default3", 6, defaultApplicationPackage);
Application default4 = tester.createAndDeploy("default4", 7, defaultApplicationPackage);
assertEquals(version, default0.oldestDeployedVersion().get());
// New version is released
version = Version.fromString("5.1");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
// Canaries upgrade and raise confidence
tester.completeUpgrade(canary0, version, canaryApplicationPackage);
tester.completeUpgrade(canary1, version, canaryApplicationPackage);
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
// Applications with default policy start upgrading
tester.clock().advance(Duration.ofMinutes(1));
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Upgrade scheduled for remaining apps", 5, tester.deploymentQueue().jobs().size());
// 4/5 applications fail, confidence is lowered and upgrade is cancelled
tester.completeUpgradeWithError(default0, version, defaultApplicationPackage, systemTest);
tester.completeUpgradeWithError(default1, version, defaultApplicationPackage, systemTest);
tester.completeUpgradeWithError(default2, version, defaultApplicationPackage, systemTest);
tester.completeUpgradeWithError(default3, version, defaultApplicationPackage, systemTest);
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
// Finish runs
tester.jobCompletion(systemTest).application(default0).unsuccessful().submit();
tester.jobCompletion(systemTest).application(default1).unsuccessful().submit();
tester.jobCompletion(systemTest).application(default2).unsuccessful().submit();
tester.jobCompletion(systemTest).application(default3).unsuccessful().submit();
// 5th app never reports back and has a dead job, but no ongoing change
Application deadLocked = tester.applications().require(default4.id());
assertTrue("Jobs in progress", deadLocked.deploymentJobs().isRunning(tester.controller().applications().deploymentTrigger().jobTimeoutLimit()));
assertFalse("No change present", deadLocked.change().isPresent());
// 4 out of 5 applications are repaired and confidence is restored
ApplicationPackage defaultApplicationPackageV2 = new ApplicationPackageBuilder().searchDefinition("search test { field test type string {} }").upgradePolicy("default").environment(Environment.prod).region("us-west-1").build();
tester.deployCompletely(default0, defaultApplicationPackageV2, 43);
tester.deployCompletely(default1, defaultApplicationPackageV2, 43);
tester.deployCompletely(default2, defaultApplicationPackageV2, 43);
tester.deployCompletely(default3, defaultApplicationPackageV2, 43);
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Upgrade scheduled for previously failing apps", 4, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(default0, version, defaultApplicationPackageV2);
tester.completeUpgrade(default1, version, defaultApplicationPackageV2);
tester.completeUpgrade(default2, version, defaultApplicationPackageV2);
tester.completeUpgrade(default3, version, defaultApplicationPackageV2);
assertEquals(version, tester.application(default0.id()).oldestDeployedVersion().get());
assertEquals(version, tester.application(default1.id()).oldestDeployedVersion().get());
assertEquals(version, tester.application(default2.id()).oldestDeployedVersion().get());
assertEquals(version, tester.application(default3.id()).oldestDeployedVersion().get());
}
use of com.yahoo.vespa.hosted.controller.deployment.DeploymentTester in project vespa by vespa-engine.
the class UpgraderTest method testUpgradingToVersionWhichBreaksSomeNonCanaries.
@Test
public void testUpgradingToVersionWhichBreaksSomeNonCanaries() {
// --- Setup
DeploymentTester tester = new DeploymentTester();
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("No system version: Nothing to do", 0, tester.deploymentQueue().jobs().size());
// (lower than the hardcoded version in the config server client)
Version version = Version.fromString("5.0");
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("No applications: Nothing to do", 0, tester.deploymentQueue().jobs().size());
// Setup applications
Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
Application default0 = tester.createAndDeploy("default0", 3, "default");
Application default1 = tester.createAndDeploy("default1", 4, "default");
Application default2 = tester.createAndDeploy("default2", 5, "default");
Application default3 = tester.createAndDeploy("default3", 6, "default");
Application default4 = tester.createAndDeploy("default4", 7, "default");
Application default5 = tester.createAndDeploy("default5", 8, "default");
Application default6 = tester.createAndDeploy("default6", 9, "default");
Application default7 = tester.createAndDeploy("default7", 10, "default");
Application default8 = tester.createAndDeploy("default8", 11, "default");
Application default9 = tester.createAndDeploy("default9", 12, "default");
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("All already on the right version: Nothing to do", 0, tester.deploymentQueue().jobs().size());
// --- A new version is released
version = Version.fromString("5.1");
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("New system version: Should upgrade Canaries", 2, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary0, version, "canary");
assertEquals(version, tester.configServer().lastPrepareVersion().get());
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("One canary pending; nothing else", 1, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(canary1, version, "canary");
tester.updateVersionStatus(version);
assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals("Canaries done: Should upgrade defaults", 10, tester.deploymentQueue().jobs().size());
tester.completeUpgrade(default0, version, "default");
tester.completeUpgradeWithError(default1, version, "default", systemTest);
tester.completeUpgradeWithError(default2, version, "default", systemTest);
tester.completeUpgradeWithError(default3, version, "default", systemTest);
tester.completeUpgradeWithError(default4, version, "default", systemTest);
// > 40% and at least 4 failed - version is broken
tester.updateVersionStatus(version);
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
assertEquals("Upgrades are cancelled", 0, tester.deploymentQueue().jobs().size());
}
Aggregations