Search in sources :

Example 1 with Deployment

use of com.yahoo.vespa.hosted.controller.application.Deployment in project vespa by vespa-engine.

the class VersionStatus method computeDeploymentStatistics.

private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, List<Application> applications) {
    Map<Version, DeploymentStatistics> versionMap = new HashMap<>();
    for (Version infrastructureVersion : infrastructureVersions) {
        versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion));
    }
    ApplicationList applicationList = ApplicationList.from(applications).notPullRequest().hasProductionDeployment();
    for (Application application : applicationList.asList()) {
        // (ignore non-production versions)
        for (Deployment deployment : application.productionDeployments().values()) {
            versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty);
        }
        // List versions which have failing jobs, versions which are in production, and versions for which there are running deployment jobs
        // Failing versions
        JobList.from(application).failing().not().failingApplicationChange().not().failingBecause(outOfCapacity).mapToList(job -> job.lastCompleted().get().version()).forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withFailing(application.id())));
        // Succeeding versions
        JobList.from(application).lastSuccess().present().production().mapToList(job -> job.lastSuccess().get().version()).forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withProduction(application.id())));
        // Deploying versions
        JobList.from(application).upgrading().mapToList(job -> job.lastTriggered().get().version()).forEach(version -> versionMap.put(version, versionMap.getOrDefault(version, DeploymentStatistics.empty(version)).withDeploying(application.id())));
    }
    return versionMap.values();
}
Also used : JobError.outOfCapacity(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity) Version(com.yahoo.component.Version) Vtag(com.yahoo.component.Vtag) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ApplicationList(com.yahoo.vespa.hosted.controller.application.ApplicationList) HashSet(java.util.HashSet) ZoneId(com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId) ImmutableList(com.google.common.collect.ImmutableList) JobList(com.yahoo.vespa.hosted.controller.application.JobList) Map(java.util.Map) URI(java.net.URI) Application(com.yahoo.vespa.hosted.controller.Application) Collection(java.util.Collection) Set(java.util.Set) Instant(java.time.Instant) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) GitSha(com.yahoo.vespa.hosted.controller.api.integration.github.GitSha) List(java.util.List) Optional(java.util.Optional) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) Collections(java.util.Collections) ListMap(com.yahoo.collections.ListMap) Controller(com.yahoo.vespa.hosted.controller.Controller) Version(com.yahoo.component.Version) HashMap(java.util.HashMap) ApplicationList(com.yahoo.vespa.hosted.controller.application.ApplicationList) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) Application(com.yahoo.vespa.hosted.controller.Application)

Example 2 with Deployment

use of com.yahoo.vespa.hosted.controller.application.Deployment in project vespa by vespa-engine.

the class ControllerTest method runDeployment.

private void runDeployment(DeploymentTester tester, Application app, ApplicationVersion version, Optional<Version> upgrade, Optional<ApplicationPackage> applicationPackage) {
    Version vespaVersion = upgrade.orElseGet(tester::defaultVespaVersion);
    // Deploy in test
    tester.deployAndNotify(app, applicationPackage, true, true, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, true, stagingTest);
    assertStatus(JobStatus.initial(stagingTest).withTriggering(vespaVersion, version, "", tester.clock().instant().minus(Duration.ofMillis(1))).withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app.id(), tester.controller());
    // Deploy in production
    tester.deployAndNotify(app, applicationPackage, true, true, productionCorpUsEast1);
    assertStatus(JobStatus.initial(productionCorpUsEast1).withTriggering(vespaVersion, version, "", tester.clock().instant().minus(Duration.ofMillis(1))).withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app.id(), tester.controller());
    tester.deployAndNotify(app, applicationPackage, true, true, productionUsEast3);
    assertStatus(JobStatus.initial(productionUsEast3).withTriggering(vespaVersion, version, "", tester.clock().instant().minus(Duration.ofMillis(1))).withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app.id(), tester.controller());
    // Verify deployed version
    app = tester.controller().applications().require(app.id());
    for (Deployment deployment : app.productionDeployments().values()) {
        assertEquals(version, deployment.applicationVersion());
        upgrade.ifPresent(v -> assertEquals(v, deployment.version()));
    }
}
Also used : ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) Version(com.yahoo.component.Version) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment)

Example 3 with Deployment

use of com.yahoo.vespa.hosted.controller.application.Deployment in project vespa by vespa-engine.

the class UpgraderTest method testAllowApplicationChangeDuringFailingUpgrade.

@Test
public void testAllowApplicationChangeDuringFailingUpgrade() {
    DeploymentTester tester = new DeploymentTester();
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod).region("us-west-1").build();
    Application app = tester.createAndDeploy("app1", 1, applicationPackage);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    tester.deployAndNotify(app, applicationPackage, true, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, stagingTest);
    tester.deployAndNotify(app, applicationPackage, false, productionUsWest1);
    // New application change
    tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit();
    String applicationVersion = "1.0.43-commit1";
    // Application change recorded together with ongoing upgrade
    app = tester.application(app.id());
    assertTrue("Change contains both upgrade and application change", app.change().platform().get().equals(version) && app.change().application().get().id().equals(applicationVersion));
    // Deployment completes
    tester.deployAndNotify(app, applicationPackage, true, false, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, false, stagingTest);
    tester.jobCompletion(productionUsWest1).application(app).unsuccessful().submit();
    tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
    app = tester.application(app.id());
    for (Deployment deployment : app.deployments().values()) {
        assertEquals(version, deployment.version());
        assertEquals(applicationVersion, deployment.applicationVersion().id());
    }
}
Also used : Version(com.yahoo.component.Version) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Application(com.yahoo.vespa.hosted.controller.Application) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) Test(org.junit.Test) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest)

Example 4 with Deployment

use of com.yahoo.vespa.hosted.controller.application.Deployment in project vespa by vespa-engine.

the class UpgraderTest method testBlockVersionChangeHalfwayThoughThenNewVersion.

/**
 * Tests the scenario where a release is deployed to 2 of 3 production zones, then blocked,
 * followed by timeout of the upgrade and a new release.
 * In this case, the blocked production zone should not progress with upgrading to the previous version,
 * and should not upgrade to the new version until the other production zones have it
 * (expected behavior; both requirements are debatable).
 */
@Test
public void testBlockVersionChangeHalfwayThoughThenNewVersion() {
    // Friday, 16:00
    ManualClock clock = new ManualClock(Instant.parse("2017-09-29T16:00:00.00Z"));
    DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").blockChange(false, true, "mon-fri", "00-09,17-23", "UTC").blockChange(false, true, "sat-sun", "00-23", "UTC").region("us-west-1").region("us-central-1").region("us-east-3").build();
    Application app = tester.createAndDeploy("app1", 1, applicationPackage);
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    // Application upgrade starts
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    tester.deployAndNotify(app, applicationPackage, true, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
    // Entering block window after prod job is triggered
    clock.advance(Duration.ofHours(1));
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1);
    // Next job not triggered due to being in the block window
    assertTrue(tester.deploymentQueue().jobs().isEmpty());
    // A day passes and we get a new version
    tester.clock().advance(Duration.ofDays(1));
    version = Version.fromString("5.2");
    tester.updateVersionStatus(version);
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertTrue("Nothing is scheduled", tester.deploymentQueue().jobs().isEmpty());
    // Monday morning: We are not blocked
    // Sunday, 17:00
    tester.clock().advance(Duration.ofDays(1));
    // Monday, 10:00
    tester.clock().advance(Duration.ofHours(17));
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // We proceed with the new version in the expected order, not starting with the previously blocked version:
    // Test jobs are run with the new version, but not production as we are in the block window
    tester.deployAndNotify(app, applicationPackage, true, systemTest);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
    tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsCentral1);
    tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
    assertTrue("All jobs consumed", tester.deploymentQueue().jobs().isEmpty());
    // App is completely upgraded to the latest version
    for (Deployment deployment : tester.applications().require(app.id()).deployments().values()) assertEquals(version, deployment.version());
}
Also used : ManualClock(com.yahoo.test.ManualClock) Version(com.yahoo.component.Version) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Application(com.yahoo.vespa.hosted.controller.Application) ControllerTester(com.yahoo.vespa.hosted.controller.ControllerTester) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) Test(org.junit.Test) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest)

Example 5 with Deployment

use of com.yahoo.vespa.hosted.controller.application.Deployment in project vespa by vespa-engine.

the class ClusterInfoMaintainerTest method maintain.

@Test
public void maintain() {
    ControllerTester tester = new ControllerTester();
    ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id();
    // Precondition: no cluster info attached to the deployments
    Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
    Assert.assertEquals(0, deployment.clusterInfo().size());
    ClusterInfoMaintainer mainainer = new ClusterInfoMaintainer(tester.controller(), Duration.ofHours(1), new JobControl(new MockCuratorDb()));
    mainainer.maintain();
    deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get();
    Assert.assertEquals(2, deployment.clusterInfo().size());
    Assert.assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost());
}
Also used : MockCuratorDb(com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) ApplicationId(com.yahoo.config.provision.ApplicationId) ControllerTester(com.yahoo.vespa.hosted.controller.ControllerTester) Test(org.junit.Test)

Aggregations

Deployment (com.yahoo.vespa.hosted.controller.application.Deployment)21 Application (com.yahoo.vespa.hosted.controller.Application)15 ApplicationId (com.yahoo.config.provision.ApplicationId)12 Version (com.yahoo.component.Version)9 ZoneId (com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId)9 ApplicationVersion (com.yahoo.vespa.hosted.controller.application.ApplicationVersion)7 DeploymentSpec (com.yahoo.config.application.api.DeploymentSpec)6 DeploymentId (com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId)6 ApplicationPackage (com.yahoo.vespa.hosted.controller.application.ApplicationPackage)6 DeploymentJobs (com.yahoo.vespa.hosted.controller.application.DeploymentJobs)6 Collections (java.util.Collections)6 List (java.util.List)6 Optional (java.util.Optional)6 ControllerTester (com.yahoo.vespa.hosted.controller.ControllerTester)5 DeploymentMetrics (com.yahoo.vespa.hosted.controller.application.DeploymentMetrics)5 URI (java.net.URI)5 ArrayList (java.util.ArrayList)5 Environment (com.yahoo.config.provision.Environment)4 TenantName (com.yahoo.config.provision.TenantName)4 JobStatus (com.yahoo.vespa.hosted.controller.application.JobStatus)4