Search in sources :

Example 36 with ApplicationPackage

use of com.yahoo.vespa.hosted.controller.application.ApplicationPackage in project vespa by vespa-engine.

the class ApplicationPackageBuilder method build.

public ApplicationPackage build() {
    ByteArrayOutputStream zip = new ByteArrayOutputStream();
    ZipOutputStream out = new ZipOutputStream(zip);
    try {
        out.putNextEntry(new ZipEntry("deployment.xml"));
        out.write(deploymentSpec());
        out.closeEntry();
        out.putNextEntry(new ZipEntry("validation-overrides.xml"));
        out.write(validationOverrides());
        out.closeEntry();
        out.putNextEntry(new ZipEntry("search-definitions/test.sd"));
        out.write(searchDefinition());
        out.closeEntry();
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            out.close();
        } catch (IOException ignored) {
        }
    }
    return new ApplicationPackage(zip.toByteArray());
}
Also used : ZipOutputStream(java.util.zip.ZipOutputStream) ZipEntry(java.util.zip.ZipEntry) UncheckedIOException(java.io.UncheckedIOException) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage)

Example 37 with ApplicationPackage

use of com.yahoo.vespa.hosted.controller.application.ApplicationPackage in project vespa by vespa-engine.

the class DeploymentTester method completeDeployment.

private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob, boolean includingProductionZones) {
    DeploymentOrder order = new DeploymentOrder(controller());
    List<JobType> jobs = order.jobsFrom(applicationPackage.deploymentSpec());
    if (!includingProductionZones)
        jobs = jobs.stream().filter(job -> !job.isProduction()).collect(Collectors.toList());
    for (JobType job : jobs) {
        boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
        deployAndNotify(application, applicationPackage, !failJob, false, job);
        if (failJob) {
            break;
        }
    }
    if (failOnJob.isPresent()) {
        assertTrue(applications().require(application.id()).change().isPresent());
        assertTrue(applications().require(application.id()).deploymentJobs().hasFailures());
    } else if (includingProductionZones) {
        assertFalse(applications().require(application.id()).change().isPresent());
    } else {
        assertTrue(applications().require(application.id()).change().isPresent());
    }
}
Also used : Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) DeploymentJobs(com.yahoo.vespa.hosted.controller.application.DeploymentJobs) JobControl(com.yahoo.vespa.hosted.controller.maintenance.JobControl) ArtifactRepositoryMock(com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock) Change(com.yahoo.vespa.hosted.controller.application.Change) TenantId(com.yahoo.vespa.hosted.controller.api.identifiers.TenantId) Duration(java.time.Duration) Application(com.yahoo.vespa.hosted.controller.Application) Upgrader(com.yahoo.vespa.hosted.controller.maintenance.Upgrader) Environment(com.yahoo.config.provision.Environment) ApplicationController(com.yahoo.vespa.hosted.controller.ApplicationController) ConfigServerClientMock(com.yahoo.vespa.hosted.controller.ConfigServerClientMock) ManualClock(com.yahoo.test.ManualClock) Assert.assertTrue(org.junit.Assert.assertTrue) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) List(java.util.List) ControllerTester(com.yahoo.vespa.hosted.controller.ControllerTester) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) BuildService(com.yahoo.vespa.hosted.controller.api.integration.BuildService) ReadyJobsTrigger(com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger) VersionStatus(com.yahoo.vespa.hosted.controller.versions.VersionStatus) Controller(com.yahoo.vespa.hosted.controller.Controller) Assert.assertEquals(org.junit.Assert.assertEquals) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType)

Example 38 with ApplicationPackage

use of com.yahoo.vespa.hosted.controller.application.ApplicationPackage in project vespa by vespa-engine.

the class ApplicationController method deployApplication.

/**
 * Deploys an application. If the application does not exist it is created.
 */
// TODO: Get rid of the options arg
public ActivateResult deployApplication(ApplicationId applicationId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, DeployOptions options) {
    try (Lock lock = lock(applicationId)) {
        LockedApplication application = get(applicationId).map(app -> new LockedApplication(app, lock)).orElseGet(() -> new LockedApplication(createApplication(applicationId, Optional.empty()), lock));
        final boolean canDeployDirectly = canDeployDirectlyTo(zone, options);
        // Determine Vespa version to use
        Version version;
        if (options.deployCurrentVersion) {
            version = application.versionIn(zone, controller);
        } else if (canDeployDirectlyTo(zone, options)) {
            version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion());
        } else if (!application.change().isPresent() && !zone.environment().isManuallyDeployed()) {
            return unexpectedDeployment(applicationId, zone, applicationPackageFromDeployer);
        } else {
            version = application.deployVersionIn(zone, controller);
        }
        // Determine application package to use
        Pair<ApplicationPackage, ApplicationVersion> artifact = artifactFor(zone, application, applicationPackageFromDeployer, canDeployDirectly, options.deployCurrentVersion);
        ApplicationPackage applicationPackage = artifact.getFirst();
        ApplicationVersion applicationVersion = artifact.getSecond();
        validate(applicationPackage.deploymentSpec());
        // Update application with information from application package
        if (!options.deployCurrentVersion) {
            // Store information about application package
            application = application.with(applicationPackage.deploymentSpec());
            application = application.with(applicationPackage.validationOverrides());
            // Delete zones not listed in DeploymentSpec, if allowed
            // We do this at deployment time to be able to return a validation failure message when necessary
            application = deleteRemovedDeployments(application);
            // Clean up deployment jobs that are no longer referenced by deployment spec
            application = deleteUnreferencedDeploymentJobs(application);
            // TODO jvenstad: Store triggering information here, including versions, when job status is read from the build service.
            // store missing information even if we fail deployment below
            store(application);
        }
        // Validate the change being deployed
        if (!canDeployDirectly) {
            validateChange(application, zone, version);
        }
        // Assign global rotation
        application = withRotation(application, zone);
        Set<String> rotationNames = new HashSet<>();
        Set<String> cnames = new HashSet<>();
        application.rotation().ifPresent(applicationRotation -> {
            rotationNames.add(applicationRotation.id().asString());
            cnames.add(applicationRotation.dnsName());
            cnames.add(applicationRotation.secureDnsName());
        });
        // Carry out deployment
        options = withVersion(version, options);
        ConfigServerClient.PreparedApplication preparedApplication = configServer.prepare(new DeploymentId(applicationId, zone), options, cnames, rotationNames, applicationPackage.zippedContent());
        preparedApplication.activate();
        application = application.withNewDeployment(zone, applicationVersion, version, clock.instant());
        store(application);
        return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length);
    }
}
Also used : ArtifactRepository(com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository) ZmsClient(com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient) EndpointStatus(com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus) DeploymentTrigger(com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger) URISyntaxException(java.net.URISyntaxException) DeploymentJobs(com.yahoo.vespa.hosted.controller.application.DeploymentJobs) ValidationId(com.yahoo.config.application.api.ValidationId) JobReport(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport) TenantName(com.yahoo.config.provision.TenantName) Tenant(com.yahoo.vespa.hosted.controller.api.Tenant) ZoneId(com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId) DeploymentExpirer(com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer) RevisionId(com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId) Duration(java.time.Duration) Map(java.util.Map) DeployOptions(com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions) URI(java.net.URI) Rotation(com.yahoo.vespa.hosted.controller.rotation.Rotation) Exceptions(com.yahoo.yolean.Exceptions) RotationRepository(com.yahoo.vespa.hosted.controller.rotation.RotationRepository) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) Set(java.util.Set) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) List(java.util.List) Optional(java.util.Optional) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) RotationsConfig(com.yahoo.vespa.hosted.rotation.config.RotationsConfig) Log(com.yahoo.vespa.hosted.controller.api.integration.configserver.Log) AthenzClientFactory(com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory) Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) DeploymentId(com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId) RecordId(com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId) ConfigServerClient(com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient) HashMap(java.util.HashMap) NToken(com.yahoo.vespa.athenz.api.NToken) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) HashSet(java.util.HashSet) ConfigChangeActions(com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions) ImmutableList(com.google.common.collect.ImmutableList) TenantId(com.yahoo.vespa.hosted.controller.api.identifiers.TenantId) RecordData(com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData) RoutingEndpoint(com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint) RoutingGenerator(com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator) ActivateResult(com.yahoo.vespa.hosted.controller.api.ActivateResult) NoInstanceException(com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException) Lock(com.yahoo.vespa.curator.Lock) Hostname(com.yahoo.vespa.hosted.controller.api.identifiers.Hostname) Environment(com.yahoo.config.provision.Environment) ControllerDb(com.yahoo.vespa.hosted.controller.persistence.ControllerDb) CuratorDb(com.yahoo.vespa.hosted.controller.persistence.CuratorDb) IOException(java.io.IOException) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Consumer(java.util.function.Consumer) Pair(com.yahoo.collections.Pair) DeploymentSpec(com.yahoo.config.application.api.DeploymentSpec) RecordName(com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName) Clock(java.time.Clock) NameService(com.yahoo.vespa.hosted.controller.api.integration.dns.NameService) PrepareResponse(com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse) Collections(java.util.Collections) Record(com.yahoo.vespa.hosted.controller.api.integration.dns.Record) RotationLock(com.yahoo.vespa.hosted.controller.rotation.RotationLock) DeploymentId(com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) RevisionId(com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId) Lock(com.yahoo.vespa.curator.Lock) RotationLock(com.yahoo.vespa.hosted.controller.rotation.RotationLock) ActivateResult(com.yahoo.vespa.hosted.controller.api.ActivateResult) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) Version(com.yahoo.component.Version) ConfigServerClient(com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient) HashSet(java.util.HashSet)

Example 39 with ApplicationPackage

use of com.yahoo.vespa.hosted.controller.application.ApplicationPackage in project vespa by vespa-engine.

the class MetricsReporterTest method test_deployment_average_duration.

@Test
public void test_deployment_average_duration() {
    DeploymentTester tester = new DeploymentTester();
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod).region("us-west-1").build();
    MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.cd);
    Application app = tester.createApplication("app1", "tenant1", 1, 11L);
    tester.deployCompletely(app, applicationPackage);
    reporter.maintain();
    // An exceptionally fast deployment :-)
    assertEquals(Duration.ZERO, getAverageDeploymentDuration(app));
    // App spends 3 hours deploying
    tester.jobCompletion(component).application(app).nextBuildNumber().uploadArtifact(applicationPackage).submit();
    tester.clock().advance(Duration.ofHours(1));
    tester.deployAndNotify(app, applicationPackage, true, systemTest);
    tester.clock().advance(Duration.ofMinutes(30));
    tester.deployAndNotify(app, applicationPackage, true, stagingTest);
    tester.clock().advance(Duration.ofMinutes(90));
    tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
    reporter.maintain();
    // Average time is 1 hour
    assertEquals(Duration.ofHours(1), getAverageDeploymentDuration(app));
    // Another deployment starts and stalls for 12 hours
    tester.jobCompletion(component).application(app).nextBuildNumber(2).uploadArtifact(applicationPackage).submit();
    tester.clock().advance(Duration.ofHours(12));
    reporter.maintain();
    assertEquals(// hanging system-test
    Duration.ofHours(12).plus(// previous staging-test
    Duration.ofMinutes(30)).plus(// previous production job
    Duration.ofMinutes(90)).dividedBy(// Total number of orchestrated jobs
    3), getAverageDeploymentDuration(app));
}
Also used : DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Application(com.yahoo.vespa.hosted.controller.Application) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) Test(org.junit.Test) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest)

Example 40 with ApplicationPackage

use of com.yahoo.vespa.hosted.controller.application.ApplicationPackage in project vespa by vespa-engine.

the class UpgraderTest method testReschedulesUpgradeAfterTimeout.

@Test
public void testReschedulesUpgradeAfterTimeout() {
    DeploymentTester tester = new DeploymentTester();
    Version version = Version.fromString("5.0");
    tester.updateVersionStatus(version);
    ApplicationPackage canaryApplicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").environment(Environment.prod).region("us-west-1").build();
    ApplicationPackage defaultApplicationPackage = new ApplicationPackageBuilder().upgradePolicy("default").environment(Environment.prod).region("us-west-1").build();
    // Setup applications
    Application canary0 = tester.createAndDeploy("canary0", 1, canaryApplicationPackage);
    Application canary1 = tester.createAndDeploy("canary1", 2, canaryApplicationPackage);
    Application default0 = tester.createAndDeploy("default0", 3, defaultApplicationPackage);
    Application default1 = tester.createAndDeploy("default1", 4, defaultApplicationPackage);
    Application default2 = tester.createAndDeploy("default2", 5, defaultApplicationPackage);
    Application default3 = tester.createAndDeploy("default3", 6, defaultApplicationPackage);
    Application default4 = tester.createAndDeploy("default4", 7, defaultApplicationPackage);
    assertEquals(version, default0.oldestDeployedVersion().get());
    // New version is released
    version = Version.fromString("5.1");
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Canaries upgrade and raise confidence
    tester.completeUpgrade(canary0, version, canaryApplicationPackage);
    tester.completeUpgrade(canary1, version, canaryApplicationPackage);
    tester.updateVersionStatus(version);
    assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
    // Applications with default policy start upgrading
    tester.clock().advance(Duration.ofMinutes(1));
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Upgrade scheduled for remaining apps", 5, tester.deploymentQueue().jobs().size());
    // 4/5 applications fail, confidence is lowered and upgrade is cancelled
    tester.completeUpgradeWithError(default0, version, defaultApplicationPackage, systemTest);
    tester.completeUpgradeWithError(default1, version, defaultApplicationPackage, systemTest);
    tester.completeUpgradeWithError(default2, version, defaultApplicationPackage, systemTest);
    tester.completeUpgradeWithError(default3, version, defaultApplicationPackage, systemTest);
    tester.updateVersionStatus(version);
    assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Finish runs
    tester.jobCompletion(systemTest).application(default0).unsuccessful().submit();
    tester.jobCompletion(systemTest).application(default1).unsuccessful().submit();
    tester.jobCompletion(systemTest).application(default2).unsuccessful().submit();
    tester.jobCompletion(systemTest).application(default3).unsuccessful().submit();
    // 5th app never reports back and has a dead job, but no ongoing change
    Application deadLocked = tester.applications().require(default4.id());
    assertTrue("Jobs in progress", deadLocked.deploymentJobs().isRunning(tester.controller().applications().deploymentTrigger().jobTimeoutLimit()));
    assertFalse("No change present", deadLocked.change().isPresent());
    // 4 out of 5 applications are repaired and confidence is restored
    ApplicationPackage defaultApplicationPackageV2 = new ApplicationPackageBuilder().searchDefinition("search test { field test type string {} }").upgradePolicy("default").environment(Environment.prod).region("us-west-1").build();
    tester.deployCompletely(default0, defaultApplicationPackageV2, 43);
    tester.deployCompletely(default1, defaultApplicationPackageV2, 43);
    tester.deployCompletely(default2, defaultApplicationPackageV2, 43);
    tester.deployCompletely(default3, defaultApplicationPackageV2, 43);
    tester.updateVersionStatus(version);
    assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    assertEquals("Upgrade scheduled for previously failing apps", 4, tester.deploymentQueue().jobs().size());
    tester.completeUpgrade(default0, version, defaultApplicationPackageV2);
    tester.completeUpgrade(default1, version, defaultApplicationPackageV2);
    tester.completeUpgrade(default2, version, defaultApplicationPackageV2);
    tester.completeUpgrade(default3, version, defaultApplicationPackageV2);
    assertEquals(version, tester.application(default0.id()).oldestDeployedVersion().get());
    assertEquals(version, tester.application(default1.id()).oldestDeployedVersion().get());
    assertEquals(version, tester.application(default2.id()).oldestDeployedVersion().get());
    assertEquals(version, tester.application(default3.id()).oldestDeployedVersion().get());
}
Also used : Version(com.yahoo.component.Version) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) Application(com.yahoo.vespa.hosted.controller.Application) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) Test(org.junit.Test) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest)

Aggregations

ApplicationPackage (com.yahoo.vespa.hosted.controller.application.ApplicationPackage)50 Test (org.junit.Test)44 Application (com.yahoo.vespa.hosted.controller.Application)37 ApplicationPackageBuilder (com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder)33 JobType.systemTest (com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest)31 JobType.stagingTest (com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest)26 Version (com.yahoo.component.Version)23 DeploymentTester (com.yahoo.vespa.hosted.controller.deployment.DeploymentTester)22 VespaVersion (com.yahoo.vespa.hosted.controller.versions.VespaVersion)11 LockedApplication (com.yahoo.vespa.hosted.controller.LockedApplication)10 ApplicationVersion (com.yahoo.vespa.hosted.controller.application.ApplicationVersion)10 TenantId (com.yahoo.vespa.hosted.controller.api.identifiers.TenantId)8 ApplicationId (com.yahoo.config.provision.ApplicationId)5 ManualClock (com.yahoo.test.ManualClock)5 ControllerTester (com.yahoo.vespa.hosted.controller.ControllerTester)5 ZoneId (com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId)5 Environment (com.yahoo.config.provision.Environment)4 DeployOptions (com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions)4 BuildService (com.yahoo.vespa.hosted.controller.api.integration.BuildService)4 Record (com.yahoo.vespa.hosted.controller.api.integration.dns.Record)4