Search in sources :

Example 1 with EventHandlerSpecification

use of org.apache.twill.api.EventHandlerSpecification in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method saveSpecification.

private TwillRuntimeSpecification saveSpecification(TwillSpecification spec, Path targetFile, Path stagingDir) throws IOException {
    final Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec, stagingDir);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> {
        RuntimeSpecification value = e.getValue();
        return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), value.getResourceSpecification(), runnableLocalFiles.getOrDefault(e.getKey(), Collections.emptyList()));
    }));
    // Serialize into a local temp file.
    LOG.debug("Creating {}", targetFile);
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        EventHandlerSpecification eventHandler = spec.getEventHandler();
        if (eventHandler == null) {
            eventHandler = new LogOnlyEventHandler().configure();
        }
        TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), eventHandler);
        Map<String, String> configMap = Maps.newHashMap();
        for (Map.Entry<String, String> entry : hConf) {
            if (entry.getKey().startsWith("twill.")) {
                configMap.put(entry.getKey(), entry.getValue());
            }
        }
        TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, "", URI.create("."), "", RunIds.fromString(programRunId.getRun()), twillSpec.getName(), null, logLevels, maxRetries, configMap, runnableConfigs);
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
        LOG.debug("Done {}", targetFile);
        return twillRuntimeSpec;
    }
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) Arrays(java.util.Arrays) ListMultimap(com.google.common.collect.ListMultimap) URL(java.net.URL) TwillPreparer(org.apache.twill.api.TwillPreparer) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) SecureStore(org.apache.twill.api.SecureStore) DirectoryStream(java.nio.file.DirectoryStream) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) Configuration(org.apache.hadoop.conf.Configuration) Map(java.util.Map) Cancellable(org.apache.twill.common.Cancellable) URI(java.net.URI) Path(java.nio.file.Path) EventHandlerSpecification(org.apache.twill.api.EventHandlerSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) TwillController(org.apache.twill.api.TwillController) LocalFile(org.apache.twill.api.LocalFile) Collection(java.util.Collection) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) Set(java.util.Set) LoggingContextAccessor(io.cdap.cdap.common.logging.LoggingContextAccessor) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) LocationCache(org.apache.twill.internal.io.LocationCache) StandardCharsets(java.nio.charset.StandardCharsets) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) Objects(java.util.Objects) List(java.util.List) ArgumentsCodec(org.apache.twill.internal.json.ArgumentsCodec) JvmOptions(org.apache.twill.internal.JvmOptions) Writer(java.io.Writer) DirUtils(io.cdap.cdap.common.utils.DirUtils) LogHandler(org.apache.twill.api.logging.LogHandler) Joiner(com.google.common.base.Joiner) LogEntry(org.apache.twill.api.logging.LogEntry) Iterables(com.google.common.collect.Iterables) Location(org.apache.twill.filesystem.Location) Paths(org.apache.twill.internal.utils.Paths) Hashing(com.google.common.hash.Hashing) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) JarEntry(java.util.jar.JarEntry) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) TwillRuntimeSpecificationAdapter(org.apache.twill.internal.json.TwillRuntimeSpecificationAdapter) Locations(io.cdap.cdap.common.io.Locations) Constants(org.apache.twill.internal.Constants) Hasher(com.google.common.hash.Hasher) ApplicationBundler(org.apache.twill.internal.ApplicationBundler) JarOutputStream(java.util.jar.JarOutputStream) LoggingContextHelper(io.cdap.cdap.logging.context.LoggingContextHelper) ClassAcceptor(org.apache.twill.api.ClassAcceptor) Logger(org.slf4j.Logger) Files(java.nio.file.Files) RunIds(io.cdap.cdap.common.app.RunIds) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) Maps(com.google.common.collect.Maps) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) Preconditions(com.google.common.base.Preconditions) TwillSpecification(org.apache.twill.api.TwillSpecification) Arguments(org.apache.twill.internal.Arguments) Collections(java.util.Collections) InputStream(java.io.InputStream) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) EventHandlerSpecification(org.apache.twill.api.EventHandlerSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) DefaultRuntimeSpecification(org.apache.twill.internal.DefaultRuntimeSpecification) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) DefaultTwillSpecification(org.apache.twill.internal.DefaultTwillSpecification) TwillSpecification(org.apache.twill.api.TwillSpecification) LogOnlyEventHandler(org.apache.twill.internal.LogOnlyEventHandler) TwillRuntimeSpecification(org.apache.twill.internal.TwillRuntimeSpecification) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Writer(java.io.Writer)

Aggregations

Joiner (com.google.common.base.Joiner)1 Preconditions (com.google.common.base.Preconditions)1 ArrayListMultimap (com.google.common.collect.ArrayListMultimap)1 Iterables (com.google.common.collect.Iterables)1 ListMultimap (com.google.common.collect.ListMultimap)1 Maps (com.google.common.collect.Maps)1 Sets (com.google.common.collect.Sets)1 Hasher (com.google.common.hash.Hasher)1 Hashing (com.google.common.hash.Hashing)1 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)1 RunIds (io.cdap.cdap.common.app.RunIds)1 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)1 Locations (io.cdap.cdap.common.io.Locations)1 LoggingContext (io.cdap.cdap.common.logging.LoggingContext)1 LoggingContextAccessor (io.cdap.cdap.common.logging.LoggingContextAccessor)1 DirUtils (io.cdap.cdap.common.utils.DirUtils)1 LoggingContextHelper (io.cdap.cdap.logging.context.LoggingContextHelper)1 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)1 File (java.io.File)1 IOException (java.io.IOException)1