use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class ApplicationSerializer method fromSlime.
// ------------------ Deserialization
public Application fromSlime(Slime slime) {
Inspector root = slime.get();
ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString());
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false);
ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString());
List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField));
DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField));
Change deploying = changeFromSlime(root.field(deployingField));
Change outstandingChange = changeFromSlime(root.field(outstandingChangeField));
Optional<IssueId> ownershipIssueId = optionalString(root.field(ownershipIssueIdField)).map(IssueId::from);
ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble());
Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField));
return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics, rotation);
}
use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class ControllerTest method testCleanupOfStaleDeploymentData.
@Test
public void testCleanupOfStaleDeploymentData() throws IOException {
DeploymentTester tester = new DeploymentTester();
tester.controllerTester().zoneRegistry().setSystem(SystemName.cd);
tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1"));
Supplier<Map<JobType, JobStatus>> statuses = () -> tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus();
// Current system version, matches version in test data
Version version = Version.fromString("6.141.117");
tester.configServer().setDefaultVersion(version);
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
// Load test data data
byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json"));
Application application = tester.controllerTester().createApplication(SlimeUtils.jsonToSlime(json));
ApplicationPackage applicationPackage = new ApplicationPackageBuilder().upgradePolicy("canary").region("cd-us-central-1").build();
tester.jobCompletion(component).application(application).uploadArtifact(applicationPackage).submit();
long cdJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.cd).isPresent()).count();
long mainJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.main).isPresent() && !type.zone(SystemName.cd).isPresent()).count();
assertEquals("Irrelevant (main) data is present.", 8, mainJobsCount);
// New version is released
version = Version.fromString("6.142.1");
tester.configServer().setDefaultVersion(version);
tester.updateVersionStatus(version);
assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
tester.upgrader().maintain();
tester.readyJobTrigger().maintain();
// Test environments pass
tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage);
tester.deploymentQueue().takeJobsToRun();
tester.clock().advance(Duration.ofMinutes(10));
tester.jobCompletion(systemTest).application(application).submit();
long newCdJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.cd).isPresent()).count();
long newMainJobsCount = statuses.get().keySet().stream().filter(type -> type.zone(SystemName.main).isPresent() && !type.zone(SystemName.cd).isPresent()).count();
assertEquals("Irrelevant (main) job data is removed.", 0, newMainJobsCount);
assertEquals("Relevant (cd) data is not removed.", cdJobsCount, newCdJobsCount);
}
use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class ApplicationApiHandler method toSlime.
private void toSlime(Cursor object, Application application, HttpRequest request) {
object.setString("application", application.id().application().value());
object.setString("instance", application.id().instance().value());
// Currently deploying change
if (application.change().isPresent()) {
Cursor deployingObject = object.setObject("deploying");
application.change().platform().ifPresent(v -> deployingObject.setString("version", v.toString()));
application.change().application().filter(v -> v != ApplicationVersion.unknown).ifPresent(v -> toSlime(v, deployingObject.setObject("revision")));
}
// Jobs sorted according to deployment spec
List<JobStatus> jobStatus = controller.applications().deploymentTrigger().deploymentOrder().sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus().values());
Cursor deploymentsArray = object.setArray("deploymentJobs");
for (JobStatus job : jobStatus) {
Cursor jobObject = deploymentsArray.addObject();
jobObject.setString("type", job.type().jobName());
jobObject.setBool("success", job.isSuccess());
job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered")));
job.lastCompleted().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastCompleted")));
job.firstFailing().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("firstFailing")));
job.lastSuccess().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastSuccess")));
}
// Change blockers
Cursor changeBlockers = object.setArray("changeBlockers");
application.deploymentSpec().changeBlocker().forEach(changeBlocker -> {
Cursor changeBlockerObject = changeBlockers.addObject();
changeBlockerObject.setBool("versions", changeBlocker.blocksVersions());
changeBlockerObject.setBool("revisions", changeBlocker.blocksRevisions());
changeBlockerObject.setString("timeZone", changeBlocker.window().zone().getId());
Cursor days = changeBlockerObject.setArray("days");
changeBlocker.window().days().stream().map(DayOfWeek::getValue).forEach(days::addLong);
Cursor hours = changeBlockerObject.setArray("hours");
changeBlocker.window().hours().forEach(hours::addLong);
});
// Compile version. The version that should be used when building an application
object.setString("compileVersion", application.oldestDeployedVersion().orElse(controller.systemVersion()).toFullString());
// Rotation
Cursor globalRotationsArray = object.setArray("globalRotations");
application.rotation().ifPresent(rotation -> {
globalRotationsArray.addString(rotation.url().toString());
globalRotationsArray.addString(rotation.secureUrl().toString());
object.setString("rotationId", rotation.id().asString());
});
// Deployments sorted according to deployment spec
List<Deployment> deployments = controller.applications().deploymentTrigger().deploymentOrder().sortBy(application.deploymentSpec().zones(), application.deployments().values());
Cursor instancesArray = object.setArray("instances");
for (Deployment deployment : deployments) {
Cursor deploymentObject = instancesArray.addObject();
deploymentObject.setString("environment", deployment.zone().environment().value());
deploymentObject.setString("region", deployment.zone().region().value());
// pointless
deploymentObject.setString("instance", application.id().instance().value());
if (application.rotation().isPresent()) {
Map<String, RotationStatus> rotationHealthStatus = application.rotation().map(rotation -> controller.getHealthStatus(rotation.dnsName())).orElse(Collections.emptyMap());
setRotationStatus(deployment, rotationHealthStatus, deploymentObject);
}
if (// List full deployment information when recursive.
recurseOverDeployments(request))
toSlime(deploymentObject, new DeploymentId(application.id(), deployment.zone()), deployment, request);
else
deploymentObject.setString("url", withPath(request.getUri().getPath() + "/environment/" + deployment.zone().environment().value() + "/region/" + deployment.zone().region().value() + "/instance/" + application.id().instance().value(), request.getUri()).toString());
}
// Metrics
Cursor metricsObject = object.setObject("metrics");
metricsObject.setDouble("queryServiceQuality", application.metrics().queryServiceQuality());
metricsObject.setDouble("writeServiceQuality", application.metrics().writeServiceQuality());
application.ownershipIssueId().ifPresent(issueId -> object.setString("ownershipIssueId", issueId.value()));
application.deploymentJobs().issueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value()));
}
use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class DeploymentTester method completeDeployment.
private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob, boolean includingProductionZones) {
DeploymentOrder order = new DeploymentOrder(controller());
List<JobType> jobs = order.jobsFrom(applicationPackage.deploymentSpec());
if (!includingProductionZones)
jobs = jobs.stream().filter(job -> !job.isProduction()).collect(Collectors.toList());
for (JobType job : jobs) {
boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
deployAndNotify(application, applicationPackage, !failJob, false, job);
if (failJob) {
break;
}
}
if (failOnJob.isPresent()) {
assertTrue(applications().require(application.id()).change().isPresent());
assertTrue(applications().require(application.id()).deploymentJobs().hasFailures());
} else if (includingProductionZones) {
assertFalse(applications().require(application.id()).change().isPresent());
} else {
assertTrue(applications().require(application.id()).change().isPresent());
}
}
use of com.yahoo.vespa.hosted.controller.application.DeploymentJobs in project vespa by vespa-engine.
the class ApplicationSerializerTest method testSerialization.
@Test
public void testSerialization() {
ControllerTester tester = new ControllerTester();
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("<deployment version='1.0'>" + " <staging/>" + "</deployment>");
ValidationOverrides validationOverrides = ValidationOverrides.fromXml("<validation-overrides version='1.0'>" + " <allow until='2017-06-15'>deployment-removal</allow>" + "</validation-overrides>");
List<Deployment> deployments = new ArrayList<>();
ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31);
ApplicationVersion applicationVersion2 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 32);
// One deployment without cluster info and utils
deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3)));
deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), createClusterUtils(3, 0.2), createClusterInfo(3, 4), new DeploymentMetrics(2, 3, 4, 5, 6)));
Optional<Long> projectId = Optional.of(123L);
List<JobStatus> statusList = new ArrayList<>();
statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest).withTriggering(Version.fromString("5.6.7"), ApplicationVersion.unknown, "Test", Instant.ofEpochMilli(7)).withCompletion(30, Optional.empty(), Instant.ofEpochMilli(8), tester.controller()));
statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest).withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, "Test 2", Instant.ofEpochMilli(5)).withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller()));
DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty());
Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, validationOverrides, deployments, deploymentJobs, Change.of(Version.fromString("6.7")), Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)), Optional.of(IssueId.from("1234")), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of(new RotationId("my-rotation")));
Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original));
assertEquals(original.id(), serialized.id());
assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
assertEquals(2, serialized.deployments().size());
assertEquals(original.deployments().get(zone1).applicationVersion(), serialized.deployments().get(zone1).applicationVersion());
assertEquals(original.deployments().get(zone2).applicationVersion(), serialized.deployments().get(zone2).applicationVersion());
assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version());
assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version());
assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at());
assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at());
assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId());
assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size());
assertEquals(original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest));
assertEquals(original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest));
assertEquals(original.outstandingChange(), serialized.outstandingChange());
assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId());
assertEquals(original.change(), serialized.change());
assertEquals(original.rotation().get().id(), serialized.rotation().get().id());
// Test cluster utilization
assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size());
assertEquals(3, serialized.deployments().get(zone2).clusterUtils().size());
assertEquals(0.4, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01);
assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01);
assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01);
// Test cluster info
assertEquals(3, serialized.deployments().get(zone2).clusterInfo().size());
assertEquals(10, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost());
assertEquals(ClusterSpec.Type.content, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType());
assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor());
assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size());
assertEquals(2, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE);
assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE);
assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE);
// Test metrics
assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
{
// test more deployment serialization cases
Application original2 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 42)));
Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2));
assertEquals(original2.change(), serialized2.change());
assertEquals(serialized2.change().application().get().source(), original2.change().application().get().source());
Application original3 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3));
assertEquals(original3.change(), serialized3.change());
assertEquals(serialized3.change().application().get().source(), original3.change().application().get().source());
Application original4 = writable(original).withChange(Change.empty());
Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4));
assertEquals(original4.change(), serialized4.change());
Application original5 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized5 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original5));
assertEquals(original5.change(), serialized5.change());
Application original6 = writable(original).withOutstandingChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized6 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original6));
assertEquals(original6.outstandingChange(), serialized6.outstandingChange());
}
}
Aggregations