Search in sources :

Example 1 with TwillRunnableSpecification

use of org.apache.twill.api.TwillRunnableSpecification in project cdap by caskdata.

the class KubeTwillPreparer method saveSpecification.

/**
 * Creates and saves a {@link TwillRunnableSpecification} to a given path.
 */
private void saveSpecification(TwillSpecification spec, Path targetFile) throws IOException {
    Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().map(e -> new AbstractMap.SimpleImmutableEntry<>(e.getKey(), new DefaultRuntimeSpecification(e.getValue().getName(), e.getValue().getRunnableSpecification(), e.getValue().getResourceSpecification(), runnableLocalFiles.get(e.getKey())))).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    LOG.debug("Saving twill specification for {} to {}", spec.getName(), targetFile);
    TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), new LogOnlyEventHandler().configure());
    TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, appLocation.getLocationFactory().getHomeLocation().getName(), appLocation.toURI(), "", twillRunId, twillSpec.getName(), "", Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
    }
}
Also used : HttpURLConnection(java.net.HttpURLConnection) Arrays(java.util.Arrays) TypeToken(com.google.gson.reflect.TypeToken) BatchV1Api(io.kubernetes.client.openapi.apis.BatchV1Api) V1ResourceRequirements(io.kubernetes.client.openapi.models.V1ResourceRequirements) SecureStore(org.apache.twill.api.SecureStore) DirectoryStream(java.nio.file.DirectoryStream) V1EnvVar(io.kubernetes.client.openapi.models.V1EnvVar) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) Configs(org.apache.twill.api.Configs) Map(java.util.Map) V1VolumeMount(io.kubernetes.client.openapi.models.V1VolumeMount) Path(java.nio.file.Path) V1PersistentVolumeClaimBuilder(io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) V1Volume(io.kubernetes.client.openapi.models.V1Volume) TwillController(org.apache.twill.api.TwillController) LocalFile(org.apache.twill.api.LocalFile) SecretDisk(io.cdap.cdap.master.spi.twill.SecretDisk) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) Set(java.util.Set) StandardCharsets(java.nio.charset.StandardCharsets) Stream(java.util.stream.Stream) V1SecurityContextBuilder(io.kubernetes.client.openapi.models.V1SecurityContextBuilder) MasterEnvironmentContext(io.cdap.cdap.master.spi.environment.MasterEnvironmentContext) V1PersistentVolumeClaim(io.kubernetes.client.openapi.models.V1PersistentVolumeClaim) LogHandler(org.apache.twill.api.logging.LogHandler) V1Deployment(io.kubernetes.client.openapi.models.V1Deployment) StatefulDisk(io.cdap.cdap.master.spi.twill.StatefulDisk) Location(org.apache.twill.filesystem.Location) V1Job(io.kubernetes.client.openapi.models.V1Job) Paths(org.apache.twill.internal.utils.Paths) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) AppsV1Api(io.kubernetes.client.openapi.apis.AppsV1Api) ApiClient(io.kubernetes.client.openapi.ApiClient) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) V1EmptyDirVolumeSource(io.kubernetes.client.openapi.models.V1EmptyDirVolumeSource) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) V1ObjectMeta(io.kubernetes.client.openapi.models.V1ObjectMeta) TwillRuntimeSpecificationAdapter(org.apache.twill.internal.json.TwillRuntimeSpecificationAdapter) ClassAcceptor(org.apache.twill.api.ClassAcceptor) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) Resources(com.google.common.io.Resources) Files(java.nio.file.Files) V1PodSpecBuilder(io.kubernetes.client.openapi.models.V1PodSpecBuilder) IOException(java.io.IOException) V1ResourceRequirementsBuilder(io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillRunnable(org.apache.twill.api.TwillRunnable) V1ObjectMetaBuilder(io.kubernetes.client.openapi.models.V1ObjectMetaBuilder) TwillRunnableSpecification(org.apache.twill.api.TwillRunnableSpecification) JsonObject(com.google.gson.JsonObject) PodInfo(io.cdap.cdap.master.environment.k8s.PodInfo) URL(java.net.URL) TwillPreparer(org.apache.twill.api.TwillPreparer) LoggerFactory(org.slf4j.LoggerFactory) V1DeploymentBuilder(io.kubernetes.client.openapi.models.V1DeploymentBuilder) V1DownwardAPIVolumeSource(io.kubernetes.client.openapi.models.V1DownwardAPIVolumeSource) V1SecurityContext(io.kubernetes.client.openapi.models.V1SecurityContext) Gson(com.google.gson.Gson) V1StatefulSet(io.kubernetes.client.openapi.models.V1StatefulSet) RunId(org.apache.twill.api.RunId) Quantity(io.kubernetes.client.custom.Quantity) URI(java.net.URI) V1ObjectFieldSelector(io.kubernetes.client.openapi.models.V1ObjectFieldSelector) V1StatefulSetBuilder(io.kubernetes.client.openapi.models.V1StatefulSetBuilder) Collection(java.util.Collection) ResourceSpecification(org.apache.twill.api.ResourceSpecification) V1LabelSelector(io.kubernetes.client.openapi.models.V1LabelSelector) Collectors(java.util.stream.Collectors) List(java.util.List) Type(java.lang.reflect.Type) Writer(java.io.Writer) Annotation(java.lang.annotation.Annotation) SecurityContext(io.cdap.cdap.master.spi.twill.SecurityContext) Optional(java.util.Optional) V1JobBuilder(io.kubernetes.client.openapi.models.V1JobBuilder) LogEntry(org.apache.twill.api.logging.LogEntry) Completable(io.cdap.cdap.master.spi.twill.Completable) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) HashSet(java.util.HashSet) ApiException(io.kubernetes.client.openapi.ApiException) JarEntry(java.util.jar.JarEntry) Constants(org.apache.twill.internal.Constants) V1Container(io.kubernetes.client.openapi.models.V1Container) V1DownwardAPIVolumeFile(io.kubernetes.client.openapi.models.V1DownwardAPIVolumeFile) JarOutputStream(java.util.jar.JarOutputStream) OutputStream(java.io.OutputStream) V1ContainerBuilder(io.kubernetes.client.openapi.models.V1ContainerBuilder) Logger(org.slf4j.Logger) KubeMasterEnvironment(io.cdap.cdap.master.environment.k8s.KubeMasterEnvironment) V1SecretVolumeSource(io.kubernetes.client.openapi.models.V1SecretVolumeSource) TimeUnit(java.util.concurrent.TimeUnit) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer) V1PodSpec(io.kubernetes.client.openapi.models.V1PodSpec) AbstractMap(java.util.AbstractMap) TwillSpecification(org.apache.twill.api.TwillSpecification) MasterEnvironmentRunnable(io.cdap.cdap.master.spi.environment.MasterEnvironmentRunnable) Collections(java.util.Collections) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillSpecification(org.apache.twill.api.TwillSpecification) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) AbstractMap(java.util.AbstractMap) Writer(java.io.Writer)

Aggregations

Hashing (com.google.common.hash.Hashing)1 Resources (com.google.common.io.Resources)1 Gson (com.google.gson.Gson)1 JsonObject (com.google.gson.JsonObject)1 TypeToken (com.google.gson.reflect.TypeToken)1 KubeMasterEnvironment (io.cdap.cdap.master.environment.k8s.KubeMasterEnvironment)1 PodInfo (io.cdap.cdap.master.environment.k8s.PodInfo)1 MasterEnvironmentContext (io.cdap.cdap.master.spi.environment.MasterEnvironmentContext)1 MasterEnvironmentRunnable (io.cdap.cdap.master.spi.environment.MasterEnvironmentRunnable)1 Completable (io.cdap.cdap.master.spi.twill.Completable)1 DependentTwillPreparer (io.cdap.cdap.master.spi.twill.DependentTwillPreparer)1 SecretDisk (io.cdap.cdap.master.spi.twill.SecretDisk)1 SecureTwillPreparer (io.cdap.cdap.master.spi.twill.SecureTwillPreparer)1 SecurityContext (io.cdap.cdap.master.spi.twill.SecurityContext)1 StatefulDisk (io.cdap.cdap.master.spi.twill.StatefulDisk)1 StatefulTwillPreparer (io.cdap.cdap.master.spi.twill.StatefulTwillPreparer)1 Quantity (io.kubernetes.client.custom.Quantity)1 ApiClient (io.kubernetes.client.openapi.ApiClient)1 ApiException (io.kubernetes.client.openapi.ApiException)1 AppsV1Api (io.kubernetes.client.openapi.apis.AppsV1Api)1