Search in sources :

Example 1 with DefaultRuntimeSpecification

use of org.apache.twill.internal.DefaultRuntimeSpecification in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method saveSpecification.

private TwillRuntimeSpecification saveSpecification(TwillSpecification spec, Path targetFile, Path stagingDir) throws IOException {
    final Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec, stagingDir);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> {
        RuntimeSpecification value = e.getValue();
        return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), value.getResourceSpecification(), runnableLocalFiles.getOrDefault(e.getKey(), Collections.emptyList()));
    }));
    // Serialize into a local temp file.
    LOG.debug("Creating {}", targetFile);
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        EventHandlerSpecification eventHandler = spec.getEventHandler();
        if (eventHandler == null) {
            eventHandler = new LogOnlyEventHandler().configure();
        }
        TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), eventHandler);
        Map<String, String> configMap = Maps.newHashMap();
        for (Map.Entry<String, String> entry : hConf) {
            if (entry.getKey().startsWith("twill.")) {
                configMap.put(entry.getKey(), entry.getValue());
            }
        }
        TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, "", URI.create("."), "", RunIds.fromString(programRunId.getRun()), twillSpec.getName(), null, logLevels, maxRetries, configMap, runnableConfigs);
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
        LOG.debug("Done {}", targetFile);
        return twillRuntimeSpec;
    }
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) Arrays(java.util.Arrays) ListMultimap(com.google.common.collect.ListMultimap) URL(java.net.URL) TwillPreparer(org.apache.twill.api.TwillPreparer) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) SecureStore(org.apache.twill.api.SecureStore) DirectoryStream(java.nio.file.DirectoryStream) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Cancellable(org.apache.twill.common.Cancellable) URI(java.net.URI) Path(java.nio.file.Path) EventHandlerSpecification(org.apache.twill.api.EventHandlerSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) TwillController(org.apache.twill.api.TwillController) LocalFile(org.apache.twill.api.LocalFile) Collection(java.util.Collection) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) Set(java.util.Set) LoggingContextAccessor(io.cdap.cdap.common.logging.LoggingContextAccessor) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) LocationCache(org.apache.twill.internal.io.LocationCache) StandardCharsets(java.nio.charset.StandardCharsets) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) Objects(java.util.Objects) List(java.util.List) ArgumentsCodec(org.apache.twill.internal.json.ArgumentsCodec) JvmOptions(org.apache.twill.internal.JvmOptions) Writer(java.io.Writer) DirUtils(io.cdap.cdap.common.utils.DirUtils) LogHandler(org.apache.twill.api.logging.LogHandler) Joiner(com.google.common.base.Joiner) LogEntry(org.apache.twill.api.logging.LogEntry) Iterables(com.google.common.collect.Iterables) Location(org.apache.twill.filesystem.Location) Paths(org.apache.twill.internal.utils.Paths) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) JarEntry(java.util.jar.JarEntry) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) TwillRuntimeSpecificationAdapter(org.apache.twill.internal.json.TwillRuntimeSpecificationAdapter) Locations(io.cdap.cdap.common.io.Locations) Constants(org.apache.twill.internal.Constants) Hasher(com.google.common.hash.Hasher) ApplicationBundler(org.apache.twill.internal.ApplicationBundler) JarOutputStream(java.util.jar.JarOutputStream) LoggingContextHelper(io.cdap.cdap.logging.context.LoggingContextHelper) ClassAcceptor(org.apache.twill.api.ClassAcceptor) Logger(org.slf4j.Logger) Files(java.nio.file.Files) RunIds(io.cdap.cdap.common.app.RunIds) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) Maps(com.google.common.collect.Maps) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) Preconditions(com.google.common.base.Preconditions) TwillSpecification(org.apache.twill.api.TwillSpecification) Arguments(org.apache.twill.internal.Arguments) Collections(java.util.Collections) InputStream(java.io.InputStream) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) EventHandlerSpecification(org.apache.twill.api.EventHandlerSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillSpecification(org.apache.twill.api.TwillSpecification) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Writer(java.io.Writer)

Example 2 with DefaultRuntimeSpecification

use of org.apache.twill.internal.DefaultRuntimeSpecification in project cdap by caskdata.

the class KubeTwillPreparer method saveSpecification.

/**
 * Creates and saves a {@link TwillRunnableSpecification} to a given path.
 */
private void saveSpecification(TwillSpecification spec, Path targetFile) throws IOException {
    Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().map(e -> new AbstractMap.SimpleImmutableEntry<>(e.getKey(), new DefaultRuntimeSpecification(e.getValue().getName(), e.getValue().getRunnableSpecification(), e.getValue().getResourceSpecification(), runnableLocalFiles.get(e.getKey())))).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    LOG.debug("Saving twill specification for {} to {}", spec.getName(), targetFile);
    TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), new LogOnlyEventHandler().configure());
    TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, appLocation.getLocationFactory().getHomeLocation().getName(), appLocation.toURI(), "", twillRunId, twillSpec.getName(), "", Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
    }
}
Also used : HttpURLConnection(java.net.HttpURLConnection) Arrays(java.util.Arrays) TypeToken(com.google.gson.reflect.TypeToken) BatchV1Api(io.kubernetes.client.openapi.apis.BatchV1Api) V1ResourceRequirements(io.kubernetes.client.openapi.models.V1ResourceRequirements) SecureStore(org.apache.twill.api.SecureStore) DirectoryStream(java.nio.file.DirectoryStream) V1EnvVar(io.kubernetes.client.openapi.models.V1EnvVar) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) Configs(org.apache.twill.api.Configs) Map(java.util.Map) V1VolumeMount(io.kubernetes.client.openapi.models.V1VolumeMount) Path(java.nio.file.Path) V1PersistentVolumeClaimBuilder(io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) V1Volume(io.kubernetes.client.openapi.models.V1Volume) TwillController(org.apache.twill.api.TwillController) LocalFile(org.apache.twill.api.LocalFile) SecretDisk(io.cdap.cdap.master.spi.twill.SecretDisk) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) Set(java.util.Set) StandardCharsets(java.nio.charset.StandardCharsets) Stream(java.util.stream.Stream) V1SecurityContextBuilder(io.kubernetes.client.openapi.models.V1SecurityContextBuilder) MasterEnvironmentContext(io.cdap.cdap.master.spi.environment.MasterEnvironmentContext) V1PersistentVolumeClaim(io.kubernetes.client.openapi.models.V1PersistentVolumeClaim) LogHandler(org.apache.twill.api.logging.LogHandler) V1Deployment(io.kubernetes.client.openapi.models.V1Deployment) StatefulDisk(io.cdap.cdap.master.spi.twill.StatefulDisk) Location(org.apache.twill.filesystem.Location) V1Job(io.kubernetes.client.openapi.models.V1Job) Paths(org.apache.twill.internal.utils.Paths) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) AppsV1Api(io.kubernetes.client.openapi.apis.AppsV1Api) ApiClient(io.kubernetes.client.openapi.ApiClient) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) V1EmptyDirVolumeSource(io.kubernetes.client.openapi.models.V1EmptyDirVolumeSource) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) V1ObjectMeta(io.kubernetes.client.openapi.models.V1ObjectMeta) TwillRuntimeSpecificationAdapter(org.apache.twill.internal.json.TwillRuntimeSpecificationAdapter) ClassAcceptor(org.apache.twill.api.ClassAcceptor) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) Resources(com.google.common.io.Resources) Files(java.nio.file.Files) V1PodSpecBuilder(io.kubernetes.client.openapi.models.V1PodSpecBuilder) IOException(java.io.IOException) V1ResourceRequirementsBuilder(io.kubernetes.client.openapi.models.V1ResourceRequirementsBuilder) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillRunnable(org.apache.twill.api.TwillRunnable) V1ObjectMetaBuilder(io.kubernetes.client.openapi.models.V1ObjectMetaBuilder) TwillRunnableSpecification(org.apache.twill.api.TwillRunnableSpecification) JsonObject(com.google.gson.JsonObject) PodInfo(io.cdap.cdap.master.environment.k8s.PodInfo) URL(java.net.URL) TwillPreparer(org.apache.twill.api.TwillPreparer) LoggerFactory(org.slf4j.LoggerFactory) V1DeploymentBuilder(io.kubernetes.client.openapi.models.V1DeploymentBuilder) V1DownwardAPIVolumeSource(io.kubernetes.client.openapi.models.V1DownwardAPIVolumeSource) V1SecurityContext(io.kubernetes.client.openapi.models.V1SecurityContext) Gson(com.google.gson.Gson) V1StatefulSet(io.kubernetes.client.openapi.models.V1StatefulSet) RunId(org.apache.twill.api.RunId) Quantity(io.kubernetes.client.custom.Quantity) URI(java.net.URI) V1ObjectFieldSelector(io.kubernetes.client.openapi.models.V1ObjectFieldSelector) V1StatefulSetBuilder(io.kubernetes.client.openapi.models.V1StatefulSetBuilder) Collection(java.util.Collection) ResourceSpecification(org.apache.twill.api.ResourceSpecification) V1LabelSelector(io.kubernetes.client.openapi.models.V1LabelSelector) Collectors(java.util.stream.Collectors) List(java.util.List) Type(java.lang.reflect.Type) Writer(java.io.Writer) Annotation(java.lang.annotation.Annotation) SecurityContext(io.cdap.cdap.master.spi.twill.SecurityContext) Optional(java.util.Optional) V1JobBuilder(io.kubernetes.client.openapi.models.V1JobBuilder) LogEntry(org.apache.twill.api.logging.LogEntry) Completable(io.cdap.cdap.master.spi.twill.Completable) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) HashSet(java.util.HashSet) ApiException(io.kubernetes.client.openapi.ApiException) JarEntry(java.util.jar.JarEntry) Constants(org.apache.twill.internal.Constants) V1Container(io.kubernetes.client.openapi.models.V1Container) V1DownwardAPIVolumeFile(io.kubernetes.client.openapi.models.V1DownwardAPIVolumeFile) JarOutputStream(java.util.jar.JarOutputStream) OutputStream(java.io.OutputStream) V1ContainerBuilder(io.kubernetes.client.openapi.models.V1ContainerBuilder) Logger(org.slf4j.Logger) KubeMasterEnvironment(io.cdap.cdap.master.environment.k8s.KubeMasterEnvironment) V1SecretVolumeSource(io.kubernetes.client.openapi.models.V1SecretVolumeSource) TimeUnit(java.util.concurrent.TimeUnit) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer) V1PodSpec(io.kubernetes.client.openapi.models.V1PodSpec) AbstractMap(java.util.AbstractMap) TwillSpecification(org.apache.twill.api.TwillSpecification) MasterEnvironmentRunnable(io.cdap.cdap.master.spi.environment.MasterEnvironmentRunnable) Collections(java.util.Collections) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillSpecification(org.apache.twill.api.TwillSpecification) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) AbstractMap(java.util.AbstractMap) Writer(java.io.Writer)

Aggregations

Hashing (com.google.common.hash.Hashing)2 IOException (java.io.IOException)2 Writer (java.io.Writer)2 URI (java.net.URI)2 URL (java.net.URL)2 StandardCharsets (java.nio.charset.StandardCharsets)2 DirectoryStream (java.nio.file.DirectoryStream)2 Files (java.nio.file.Files)2 Path (java.nio.file.Path)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 TimeUnit (java.util.concurrent.TimeUnit)2 JarEntry (java.util.jar.JarEntry)2 JarOutputStream (java.util.jar.JarOutputStream)2