use of com.yahoo.config.application.api.DeploymentSpec in project vespa by vespa-engine.
the class DeploymentSpecXmlReader method read.
/**
* Reads a deployment spec from XML
*/
public DeploymentSpec read(String xmlForm) {
List<Step> steps = new ArrayList<>();
Optional<String> globalServiceId = Optional.empty();
Element root = XML.getDocument(xmlForm).getDocumentElement();
if (validate)
validateTagOrder(root);
for (Element environmentTag : XML.getChildren(root)) {
if (!isEnvironmentName(environmentTag.getTagName()))
continue;
Environment environment = Environment.from(environmentTag.getTagName());
if (environment == Environment.prod) {
for (Element stepTag : XML.getChildren(environmentTag)) {
Optional<AthenzService> athenzService = stringAttribute("athenz-service", environmentTag).map(AthenzService::from);
if (stepTag.getTagName().equals("delay")) {
steps.add(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + longAttribute("minutes", stepTag) * 60 + longAttribute("seconds", stepTag))));
} else if (stepTag.getTagName().equals("parallel")) {
List<DeclaredZone> zones = new ArrayList<>();
for (Element regionTag : XML.getChildren(stepTag)) {
zones.add(readDeclaredZone(environment, athenzService, regionTag));
}
steps.add(new ParallelZones(zones));
} else {
// a region: deploy step
steps.add(readDeclaredZone(environment, athenzService, stepTag));
}
}
} else {
steps.add(new DeclaredZone(environment));
}
if (environment == Environment.prod)
globalServiceId = readGlobalServiceId(environmentTag);
else if (readGlobalServiceId(environmentTag).isPresent())
throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag.");
}
Optional<AthenzDomain> athenzDomain = stringAttribute("athenz-domain", root).map(AthenzDomain::from);
Optional<AthenzService> athenzService = stringAttribute("athenz-service", root).map(AthenzService::from);
return new DeploymentSpec(globalServiceId, readUpgradePolicy(root), readChangeBlockers(root), steps, xmlForm, athenzDomain, athenzService);
}
use of com.yahoo.config.application.api.DeploymentSpec in project vespa by vespa-engine.
the class ApplicationSerializer method fromSlime.
// ------------------ Deserialization
public Application fromSlime(Slime slime) {
Inspector root = slime.get();
ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString());
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false);
ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString());
List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField));
DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField));
Change deploying = changeFromSlime(root.field(deployingField));
Change outstandingChange = changeFromSlime(root.field(outstandingChangeField));
Optional<IssueId> ownershipIssueId = optionalString(root.field(ownershipIssueIdField)).map(IssueId::from);
ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble());
Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField));
return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics, rotation);
}
use of com.yahoo.config.application.api.DeploymentSpec in project vespa by vespa-engine.
the class DeploymentFileValidator method validate.
@Override
public void validate(VespaModel model, DeployState deployState) {
Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment();
if (deployment.isPresent()) {
Reader deploymentReader = deployment.get();
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader);
final Optional<String> globalServiceId = deploymentSpec.globalServiceId();
if (globalServiceId.isPresent()) {
Set<ContainerCluster> containerClusters = model.getRoot().configModelRepo().getModels(ContainerModel.class).stream().map(ContainerModel::getCluster).filter(cc -> cc.getName().equals(globalServiceId.get())).collect(Collectors.toSet());
if (containerClusters.size() != 1) {
throw new IllegalArgumentException("global-service-id '" + globalServiceId.get() + "' specified in deployment.xml does not match any container cluster id");
}
}
}
}
use of com.yahoo.config.application.api.DeploymentSpec in project vespa by vespa-engine.
the class ApplicationController method deployApplication.
/**
* Deploys an application. If the application does not exist it is created.
*/
// TODO: Get rid of the options arg
public ActivateResult deployApplication(ApplicationId applicationId, ZoneId zone, Optional<ApplicationPackage> applicationPackageFromDeployer, DeployOptions options) {
try (Lock lock = lock(applicationId)) {
LockedApplication application = get(applicationId).map(app -> new LockedApplication(app, lock)).orElseGet(() -> new LockedApplication(createApplication(applicationId, Optional.empty()), lock));
final boolean canDeployDirectly = canDeployDirectlyTo(zone, options);
// Determine Vespa version to use
Version version;
if (options.deployCurrentVersion) {
version = application.versionIn(zone, controller);
} else if (canDeployDirectlyTo(zone, options)) {
version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion());
} else if (!application.change().isPresent() && !zone.environment().isManuallyDeployed()) {
return unexpectedDeployment(applicationId, zone, applicationPackageFromDeployer);
} else {
version = application.deployVersionIn(zone, controller);
}
// Determine application package to use
Pair<ApplicationPackage, ApplicationVersion> artifact = artifactFor(zone, application, applicationPackageFromDeployer, canDeployDirectly, options.deployCurrentVersion);
ApplicationPackage applicationPackage = artifact.getFirst();
ApplicationVersion applicationVersion = artifact.getSecond();
validate(applicationPackage.deploymentSpec());
// Update application with information from application package
if (!options.deployCurrentVersion) {
// Store information about application package
application = application.with(applicationPackage.deploymentSpec());
application = application.with(applicationPackage.validationOverrides());
// Delete zones not listed in DeploymentSpec, if allowed
// We do this at deployment time to be able to return a validation failure message when necessary
application = deleteRemovedDeployments(application);
// Clean up deployment jobs that are no longer referenced by deployment spec
application = deleteUnreferencedDeploymentJobs(application);
// TODO jvenstad: Store triggering information here, including versions, when job status is read from the build service.
// store missing information even if we fail deployment below
store(application);
}
// Validate the change being deployed
if (!canDeployDirectly) {
validateChange(application, zone, version);
}
// Assign global rotation
application = withRotation(application, zone);
Set<String> rotationNames = new HashSet<>();
Set<String> cnames = new HashSet<>();
application.rotation().ifPresent(applicationRotation -> {
rotationNames.add(applicationRotation.id().asString());
cnames.add(applicationRotation.dnsName());
cnames.add(applicationRotation.secureDnsName());
});
// Carry out deployment
options = withVersion(version, options);
ConfigServerClient.PreparedApplication preparedApplication = configServer.prepare(new DeploymentId(applicationId, zone), options, cnames, rotationNames, applicationPackage.zippedContent());
preparedApplication.activate();
application = application.withNewDeployment(zone, applicationVersion, version, clock.instant());
store(application);
return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length);
}
}
use of com.yahoo.config.application.api.DeploymentSpec in project vespa by vespa-engine.
the class ApplicationSerializerTest method testSerialization.
@Test
public void testSerialization() {
ControllerTester tester = new ControllerTester();
DeploymentSpec deploymentSpec = DeploymentSpec.fromXml("<deployment version='1.0'>" + " <staging/>" + "</deployment>");
ValidationOverrides validationOverrides = ValidationOverrides.fromXml("<validation-overrides version='1.0'>" + " <allow until='2017-06-15'>deployment-removal</allow>" + "</validation-overrides>");
List<Deployment> deployments = new ArrayList<>();
ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31);
ApplicationVersion applicationVersion2 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 32);
// One deployment without cluster info and utils
deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3)));
deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), createClusterUtils(3, 0.2), createClusterInfo(3, 4), new DeploymentMetrics(2, 3, 4, 5, 6)));
Optional<Long> projectId = Optional.of(123L);
List<JobStatus> statusList = new ArrayList<>();
statusList.add(JobStatus.initial(DeploymentJobs.JobType.systemTest).withTriggering(Version.fromString("5.6.7"), ApplicationVersion.unknown, "Test", Instant.ofEpochMilli(7)).withCompletion(30, Optional.empty(), Instant.ofEpochMilli(8), tester.controller()));
statusList.add(JobStatus.initial(DeploymentJobs.JobType.stagingTest).withTriggering(Version.fromString("5.6.6"), ApplicationVersion.unknown, "Test 2", Instant.ofEpochMilli(5)).withCompletion(11, Optional.of(JobError.unknown), Instant.ofEpochMilli(6), tester.controller()));
DeploymentJobs deploymentJobs = new DeploymentJobs(projectId, statusList, Optional.empty());
Application original = new Application(ApplicationId.from("t1", "a1", "i1"), deploymentSpec, validationOverrides, deployments, deploymentJobs, Change.of(Version.fromString("6.7")), Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)), Optional.of(IssueId.from("1234")), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of(new RotationId("my-rotation")));
Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original));
assertEquals(original.id(), serialized.id());
assertEquals(original.deploymentSpec().xmlForm(), serialized.deploymentSpec().xmlForm());
assertEquals(original.validationOverrides().xmlForm(), serialized.validationOverrides().xmlForm());
assertEquals(2, serialized.deployments().size());
assertEquals(original.deployments().get(zone1).applicationVersion(), serialized.deployments().get(zone1).applicationVersion());
assertEquals(original.deployments().get(zone2).applicationVersion(), serialized.deployments().get(zone2).applicationVersion());
assertEquals(original.deployments().get(zone1).version(), serialized.deployments().get(zone1).version());
assertEquals(original.deployments().get(zone2).version(), serialized.deployments().get(zone2).version());
assertEquals(original.deployments().get(zone1).at(), serialized.deployments().get(zone1).at());
assertEquals(original.deployments().get(zone2).at(), serialized.deployments().get(zone2).at());
assertEquals(original.deploymentJobs().projectId(), serialized.deploymentJobs().projectId());
assertEquals(original.deploymentJobs().jobStatus().size(), serialized.deploymentJobs().jobStatus().size());
assertEquals(original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest));
assertEquals(original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest));
assertEquals(original.outstandingChange(), serialized.outstandingChange());
assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId());
assertEquals(original.change(), serialized.change());
assertEquals(original.rotation().get().id(), serialized.rotation().get().id());
// Test cluster utilization
assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size());
assertEquals(3, serialized.deployments().get(zone2).clusterUtils().size());
assertEquals(0.4, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id2")).getCpu(), 0.01);
assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getCpu(), 0.01);
assertEquals(0.2, serialized.deployments().get(zone2).clusterUtils().get(ClusterSpec.Id.from("id1")).getMemory(), 0.01);
// Test cluster info
assertEquals(3, serialized.deployments().get(zone2).clusterInfo().size());
assertEquals(10, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost());
assertEquals(ClusterSpec.Type.content, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType());
assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor());
assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size());
assertEquals(2, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE);
assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE);
assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE);
// Test metrics
assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE);
assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE);
assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE);
assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE);
assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE);
{
// test more deployment serialization cases
Application original2 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 42)));
Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2));
assertEquals(original2.change(), serialized2.change());
assertEquals(serialized2.change().application().get().source(), original2.change().application().get().source());
Application original3 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3));
assertEquals(original3.change(), serialized3.change());
assertEquals(serialized3.change().application().get().source(), original3.change().application().get().source());
Application original4 = writable(original).withChange(Change.empty());
Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4));
assertEquals(original4.change(), serialized4.change());
Application original5 = writable(original).withChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized5 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original5));
assertEquals(original5.change(), serialized5.change());
Application original6 = writable(original).withOutstandingChange(Change.of(ApplicationVersion.from(new SourceRevision("a", "b", "c"), 42)));
Application serialized6 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original6));
assertEquals(original6.outstandingChange(), serialized6.outstandingChange());
}
}
Aggregations