Search in sources :

Example 1 with JobType

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType in project vespa by vespa-engine.

the class ControllerTest method testCleanupOfStaleDeploymentData.

@Test
public void testCleanupOfStaleDeploymentData() throws IOException {
    DeploymentTester tester = new DeploymentTester();
    tester.controllerTester().zoneRegistry().setSystem(SystemName.cd);
    tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1"));
    Supplier<Map<JobType, JobStatus>> statuses = () -> tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus();
    // Current system version, matches version in test data
    Version version = Version.fromString("6.141.117");
    tester.configServer().setDefaultVersion(version);
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    // Load test data data
    byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json"));
    Application application = tester.controllerTester().createApplication(SlimeUtils.jsonToSlime(json));
    ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").region("cd-us-central-1").build();
    tester.jobCompletion(component).application(application).uploadArtifact(applicationPackage).submit();
    long cdJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.cd).isPresent()).count();
    long mainJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.main).isPresent() && !type.zone(SystemName.cd).isPresent()).count();
    assertEquals("Irrelevant (main) data is present.", 8, mainJobsCount);
    // New version is released
    version = Version.fromString("6.142.1");
    tester.configServer().setDefaultVersion(version);
    tester.updateVersionStatus(version);
    assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
    tester.upgrader().maintain();
    tester.readyJobTrigger().maintain();
    // Test environments pass
    tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage);
    tester.deploymentQueue().takeJobsToRun();
    tester.clock().advance(Duration.ofMinutes(10));
    tester.jobCompletion(systemTest).application(application).submit();
    long newCdJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.cd).isPresent()).count();
    long newMainJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.main).isPresent() && !type.zone(SystemName.cd).isPresent()).count();
    assertEquals("Irrelevant (main) job data is removed.", 0, newMainJobsCount);
    assertEquals("Relevant (cd) data is not removed.", cdJobsCount, newCdJobsCount);
}
Also used : InstanceName(com.yahoo.config.provision.InstanceName) EndpointStatus(com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus) JobType.productionCorpUsEast1(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1) DeploymentJobs(com.yahoo.vespa.hosted.controller.application.DeploymentJobs) ValidationId(com.yahoo.config.application.api.ValidationId) RegionName(com.yahoo.config.provision.RegionName) TenantName(com.yahoo.config.provision.TenantName) ZoneId(com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId) Duration(java.time.Duration) Map(java.util.Map) JobStatus(com.yahoo.vespa.hosted.controller.application.JobStatus) ApplicationSerializer(com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer) Assert.fail(org.junit.Assert.fail) DeployOptions(com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions) ScrewdriverBuildJob(com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) Instant(java.time.Instant) JobError(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) List(java.util.List) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest) Assert.assertFalse(org.junit.Assert.assertFalse) SystemName(com.yahoo.config.provision.SystemName) Optional(java.util.Optional) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) DeploymentStatistics(com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics) ApplicationName(com.yahoo.config.provision.ApplicationName) Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) DeploymentId(com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId) Supplier(java.util.function.Supplier) NToken(com.yahoo.vespa.athenz.api.NToken) ArrayList(java.util.ArrayList) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) SlimeUtils(com.yahoo.vespa.config.SlimeUtils) Change(com.yahoo.vespa.hosted.controller.application.Change) DeploymentQueue(com.yahoo.vespa.hosted.controller.deployment.DeploymentQueue) TenantId(com.yahoo.vespa.hosted.controller.api.identifiers.TenantId) BuildJob(com.yahoo.vespa.hosted.controller.deployment.BuildJob) Files(java.nio.file.Files) Environment(com.yahoo.config.provision.Environment) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) SourceRevision(com.yahoo.vespa.hosted.controller.application.SourceRevision) Test(org.junit.Test) IOException(java.io.IOException) RotationId(com.yahoo.vespa.hosted.controller.rotation.RotationId) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) JobType.productionUsWest1(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsWest1) Assert.assertNull(org.junit.Assert.assertNull) Paths(java.nio.file.Paths) RecordName(com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName) BuildService(com.yahoo.vespa.hosted.controller.api.integration.BuildService) JobType.productionUsEast3(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionUsEast3) ScrewdriverId(com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId) VersionStatus(com.yahoo.vespa.hosted.controller.versions.VersionStatus) Collections(java.util.Collections) JobType.component(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component) Record(com.yahoo.vespa.hosted.controller.api.integration.dns.Record) RotationLock(com.yahoo.vespa.hosted.controller.rotation.RotationLock) Assert.assertEquals(org.junit.Assert.assertEquals) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) VespaVersion(com.yahoo.vespa.hosted.controller.versions.VespaVersion) Version(com.yahoo.component.Version) DeploymentTester(com.yahoo.vespa.hosted.controller.deployment.DeploymentTester) ApplicationPackageBuilder(com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder) Map(java.util.Map) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) JobType.stagingTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest) JobType.systemTest(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest) Test(org.junit.Test)

Example 2 with JobType

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType in project vespa by vespa-engine.

the class ScrewdriverApiHandler method trigger.

private HttpResponse trigger(HttpRequest request, String tenantName, String applicationName) {
    JobType jobType = Optional.of(asString(request.getData())).filter(s -> !s.isEmpty()).map(JobType::fromJobName).orElse(JobType.component);
    ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default");
    controller.applications().lockOrThrow(applicationId, application -> {
        // Since this is a manual operation we likely want it to trigger as soon as possible so we add it at to the
        // front of the queue
        application = controller.applications().deploymentTrigger().trigger(new DeploymentTrigger.Triggering(application, jobType, true, "Triggered from screwdriver/v1"), Collections.emptySet(), true);
        controller.applications().store(application);
    });
    Slime slime = new Slime();
    Cursor cursor = slime.setObject();
    cursor.setString("message", "Triggered " + jobType.jobName() + " for " + applicationId);
    return new SlimeJsonResponse(slime);
}
Also used : JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) SlimeJsonResponse(com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse) ApplicationId(com.yahoo.config.provision.ApplicationId) Slime(com.yahoo.slime.Slime) Cursor(com.yahoo.slime.Cursor)

Example 3 with JobType

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType in project vespa by vespa-engine.

the class DeploymentTester method consumeJobs.

/**
 * Assert that the sceduled jobs of this application are exactly those given, and take them
 */
private void consumeJobs(Application application, boolean expectOnlyTheseJobs, JobType... jobs) {
    for (JobType job : jobs) {
        BuildService.BuildJob buildJob = findJob(application, job);
        assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.projectId());
        assertEquals(job.jobName(), buildJob.jobName());
    }
    if (expectOnlyTheseJobs)
        assertEquals("Unexpected job queue: " + jobsOf(application), jobs.length, jobsOf(application).size());
    deploymentQueue().removeJobs(application.id());
}
Also used : JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) BuildService(com.yahoo.vespa.hosted.controller.api.integration.BuildService)

Example 4 with JobType

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType in project vespa by vespa-engine.

the class DeploymentTester method completeDeployment.

private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob, boolean includingProductionZones) {
    DeploymentOrder order = new DeploymentOrder(controller());
    List<JobType> jobs = order.jobsFrom(applicationPackage.deploymentSpec());
    if (!includingProductionZones)
        jobs = jobs.stream().filter(job -> !job.isProduction()).collect(Collectors.toList());
    for (JobType job : jobs) {
        boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
        deployAndNotify(application, applicationPackage, !failJob, false, job);
        if (failJob) {
            break;
        }
    }
    if (failOnJob.isPresent()) {
        assertTrue(applications().require(application.id()).change().isPresent());
        assertTrue(applications().require(application.id()).deploymentJobs().hasFailures());
    } else if (includingProductionZones) {
        assertFalse(applications().require(application.id()).change().isPresent());
    } else {
        assertTrue(applications().require(application.id()).change().isPresent());
    }
}
Also used : Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) DeploymentJobs(com.yahoo.vespa.hosted.controller.application.DeploymentJobs) JobControl(com.yahoo.vespa.hosted.controller.maintenance.JobControl) ArtifactRepositoryMock(com.yahoo.vespa.hosted.controller.ArtifactRepositoryMock) Change(com.yahoo.vespa.hosted.controller.application.Change) TenantId(com.yahoo.vespa.hosted.controller.api.identifiers.TenantId) Duration(java.time.Duration) Application(com.yahoo.vespa.hosted.controller.Application) Upgrader(com.yahoo.vespa.hosted.controller.maintenance.Upgrader) Environment(com.yahoo.config.provision.Environment) ApplicationController(com.yahoo.vespa.hosted.controller.ApplicationController) ConfigServerClientMock(com.yahoo.vespa.hosted.controller.ConfigServerClientMock) ManualClock(com.yahoo.test.ManualClock) Assert.assertTrue(org.junit.Assert.assertTrue) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) ApplicationPackage(com.yahoo.vespa.hosted.controller.application.ApplicationPackage) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) List(java.util.List) ControllerTester(com.yahoo.vespa.hosted.controller.ControllerTester) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) BuildService(com.yahoo.vespa.hosted.controller.api.integration.BuildService) ReadyJobsTrigger(com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger) VersionStatus(com.yahoo.vespa.hosted.controller.versions.VersionStatus) Controller(com.yahoo.vespa.hosted.controller.Controller) Assert.assertEquals(org.junit.Assert.assertEquals) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType)

Example 5 with JobType

use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType in project vespa by vespa-engine.

the class DeploymentTrigger method changeDeployed.

/**
 * Returns whether the given application should skip deployment of its current change to the given production job zone.
 *
 * If the currently deployed application has a newer platform or application version than the application's
 * current change, the method returns {@code true}, to avoid a downgrade.
 * Otherwise, it returns whether the current change is redundant, i.e., all its components are already deployed.
 */
private boolean changeDeployed(Application application, JobType job) {
    if (!job.isProduction())
        throw new IllegalArgumentException(job + " is not a production job!");
    Deployment deployment = application.deployments().get(job.zone(controller.system()).get());
    if (deployment == null)
        return false;
    int applicationComparison = application.change().application().map(version -> version.compareTo(deployment.applicationVersion())).orElse(0);
    int platformComparion = application.change().platform().map(version -> version.compareTo(deployment.version())).orElse(0);
    if (applicationComparison == -1 || platformComparion == -1)
        // Avoid downgrades!
        return true;
    return applicationComparison == 0 && platformComparion == 0;
}
Also used : Version(com.yahoo.component.Version) ApplicationId(com.yahoo.config.provision.ApplicationId) JobReport(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport) ApplicationList(com.yahoo.vespa.hosted.controller.application.ApplicationList) JobList(com.yahoo.vespa.hosted.controller.application.JobList) Change(com.yahoo.vespa.hosted.controller.application.Change) Duration(java.time.Duration) JobStatus(com.yahoo.vespa.hosted.controller.application.JobStatus) Application(com.yahoo.vespa.hosted.controller.Application) LockedApplication(com.yahoo.vespa.hosted.controller.LockedApplication) ApplicationController(com.yahoo.vespa.hosted.controller.ApplicationController) Collection(java.util.Collection) ApplicationVersion(com.yahoo.vespa.hosted.controller.application.ApplicationVersion) CuratorDb(com.yahoo.vespa.hosted.controller.persistence.CuratorDb) Instant(java.time.Instant) Logger(java.util.logging.Logger) JobError(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError) Objects(java.util.Objects) JobType(com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType) List(java.util.List) Stream(java.util.stream.Stream) SystemName(com.yahoo.config.provision.SystemName) Clock(java.time.Clock) Optional(java.util.Optional) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment) Collections(java.util.Collections) Controller(com.yahoo.vespa.hosted.controller.Controller) Deployment(com.yahoo.vespa.hosted.controller.application.Deployment)

Aggregations

JobType (com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType)6 Version (com.yahoo.component.Version)4 ApplicationId (com.yahoo.config.provision.ApplicationId)4 BuildService (com.yahoo.vespa.hosted.controller.api.integration.BuildService)3 ApplicationVersion (com.yahoo.vespa.hosted.controller.application.ApplicationVersion)3 Change (com.yahoo.vespa.hosted.controller.application.Change)3 JobStatus (com.yahoo.vespa.hosted.controller.application.JobStatus)3 Duration (java.time.Duration)3 List (java.util.List)3 Optional (java.util.Optional)3 Environment (com.yahoo.config.provision.Environment)2 SystemName (com.yahoo.config.provision.SystemName)2 Application (com.yahoo.vespa.hosted.controller.Application)2 ApplicationController (com.yahoo.vespa.hosted.controller.ApplicationController)2 Controller (com.yahoo.vespa.hosted.controller.Controller)2 TenantId (com.yahoo.vespa.hosted.controller.api.identifiers.TenantId)2 ApplicationPackage (com.yahoo.vespa.hosted.controller.application.ApplicationPackage)2 Deployment (com.yahoo.vespa.hosted.controller.application.Deployment)2 DeploymentJobs (com.yahoo.vespa.hosted.controller.application.DeploymentJobs)2 JobError (com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError)2