Examples with TwillRuntimeSpecification org.apache.twill.internal.TwillRuntimeSpecification used on opensource projects

Example 1 with TwillRuntimeSpecification

use of org.apache.twill.internal.TwillRuntimeSpecification in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method saveSpecification.

private TwillRuntimeSpecification saveSpecification(TwillSpecification spec, Path targetFile, Path stagingDir) throws IOException {
    final Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec, stagingDir);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> {
        RuntimeSpecification value = e.getValue();
        return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), value.getResourceSpecification(), runnableLocalFiles.getOrDefault(e.getKey(), Collections.emptyList()));
    }));
    // Serialize into a local temp file.
    LOG.debug("Creating {}", targetFile);
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        EventHandlerSpecification eventHandler = spec.getEventHandler();
        if (eventHandler == null) {
            eventHandler = new LogOnlyEventHandler().configure();
        }
        TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), eventHandler);
        Map<String, String> configMap = Maps.newHashMap();
        for (Map.Entry<String, String> entry : hConf) {
            if (entry.getKey().startsWith("twill.")) {
                configMap.put(entry.getKey(), entry.getValue());
            }
        }
        TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, "", URI.create("."), "", RunIds.fromString(programRunId.getRun()), twillSpec.getName(), null, logLevels, maxRetries, configMap, runnableConfigs);
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
        LOG.debug("Done {}", targetFile);
        return twillRuntimeSpec;
    }
}

Example 2 with TwillRuntimeSpecification

use of org.apache.twill.internal.TwillRuntimeSpecification in project cdap by caskdata.

the class KubeTwillPreparer method saveSpecification.

/**
 * Creates and saves a {@link TwillRunnableSpecification} to a given path.
 */
private void saveSpecification(TwillSpecification spec, Path targetFile) throws IOException {
    Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().map(e -> new AbstractMap.SimpleImmutableEntry<>(e.getKey(), new DefaultRuntimeSpecification(e.getValue().getName(), e.getValue().getRunnableSpecification(), e.getValue().getResourceSpecification(), runnableLocalFiles.get(e.getKey())))).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    LOG.debug("Saving twill specification for {} to {}", spec.getName(), targetFile);
    TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), new LogOnlyEventHandler().configure());
    TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, appLocation.getLocationFactory().getHomeLocation().getName(), appLocation.toURI(), "", twillRunId, twillSpec.getName(), "", Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
    }
}

Example 3 with TwillRuntimeSpecification

use of org.apache.twill.internal.TwillRuntimeSpecification in project cdap by caskdata.

the class KubeTwillLauncher method run.

@Override
public void run(String[] args) throws Exception {
    if (args.length < 1) {
        throw new IllegalArgumentException("Requires runnable name in the argument");
    }
    String runnableName = args[0];
    Path runtimeConfigDir = Paths.get(Constants.Files.RUNTIME_CONFIG_JAR);
    Path argumentsPath = runtimeConfigDir.resolve(Constants.Files.ARGUMENTS);
    // Deserialize the arguments
    List<String> appArgs;
    List<String> runnableArgs;
    try (Reader reader = Files.newBufferedReader(argumentsPath, StandardCharsets.UTF_8)) {
        JsonObject jsonObj = GSON.fromJson(reader, JsonObject.class);
        appArgs = GSON.fromJson(jsonObj.get("arguments"), new TypeToken<List<String>>() {
        }.getType());
        Map<String, List<String>> map = GSON.fromJson(jsonObj.get("runnableArguments"), new TypeToken<Map<String, List<String>>>() {
        }.getType());
        runnableArgs = map.getOrDefault(runnableName, Collections.emptyList());
    }
    PodInfo podInfo = masterEnv.getPodInfo();
    try {
        TwillRuntimeSpecification twillRuntimeSpec = TwillRuntimeSpecificationAdapter.create().fromJson(runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC).toFile());
        RuntimeSpecification runtimeSpec = twillRuntimeSpec.getTwillSpecification().getRunnables().get(runnableName);
        RunId runId = twillRuntimeSpec.getTwillAppRunId();
        String runnableClassName = runtimeSpec.getRunnableSpecification().getClassName();
        Class<?> runnableClass = context.getClass().getClassLoader().loadClass(runnableClassName);
        if (!TwillRunnable.class.isAssignableFrom(runnableClass)) {
            throw new IllegalArgumentException("Class " + runnableClass + " is not an instance of " + TwillRunnable.class);
        }
        twillRunnable = (TwillRunnable) Instances.newInstance(runnableClass);
        try (KubeTwillContext twillContext = new KubeTwillContext(runtimeSpec, runId, RunIds.fromString(runId.getId() + "-0"), appArgs.toArray(new String[0]), runnableArgs.toArray(new String[0]), masterEnv)) {
            twillRunnable.initialize(twillContext);
            if (!stopped) {
                twillRunnable.run();
            }
        }
    } finally {
        try {
            TwillRunnable runnable = twillRunnable;
            if (runnable != null) {
                runnable.destroy();
            }
        } finally {
            if (Arrays.stream(args).noneMatch(str -> str.equalsIgnoreCase(KubeMasterEnvironment.DISABLE_POD_DELETION))) {
                // Delete the pod itself to avoid pod goes into CrashLoopBackoff. This is added for preview pods.
                // When pod is exited, exponential backoff happens. So pod restart time keep increasing.
                // Deleting pod does not trigger exponential backoff.
                // See https://github.com/kubernetes/kubernetes/issues/57291
                deletePod(podInfo);
            }
        }
    }
}

Example 4 with TwillRuntimeSpecification

use of org.apache.twill.internal.TwillRuntimeSpecification in project cdap by caskdata.

the class FileLocalizer method run.

@Override
public void run(String[] args) throws Exception {
    if (args.length < 2) {
        // This should never happen
        throw new IllegalArgumentException("Expected to have two arguments: runtime config uri and the runnable name.");
    }
    LocalLocationFactory localLocationFactory = new LocalLocationFactory();
    // Localize the runtime config jar
    URI uri = URI.create(args[0]);
    Path runtimeConfigDir;
    if (localLocationFactory.getHomeLocation().toURI().getScheme().equals(uri.getScheme())) {
        try (FileInputStream is = new FileInputStream(new File(uri))) {
            runtimeConfigDir = expand(uri, is, Paths.get(Constants.Files.RUNTIME_CONFIG_JAR));
        }
    } else {
        try (InputStream is = getHttpURLConnectionInputStream(fileDownloadURLPath(uri))) {
            runtimeConfigDir = expand(uri, is, Paths.get(Constants.Files.RUNTIME_CONFIG_JAR));
        }
    }
    try (Reader reader = Files.newBufferedReader(runtimeConfigDir.resolve(Constants.Files.TWILL_SPEC), StandardCharsets.UTF_8)) {
        TwillRuntimeSpecification twillRuntimeSpec = TwillRuntimeSpecificationAdapter.create().fromJson(reader);
        Path targetDir = Paths.get(System.getProperty("user.dir"));
        Files.createDirectories(targetDir);
        for (LocalFile localFile : twillRuntimeSpec.getTwillSpecification().getRunnables().get(args[1]).getLocalFiles()) {
            if (stopped) {
                LOG.info("Stop localization on request");
                break;
            }
            Path targetPath = targetDir.resolve(localFile.getName());
            try (InputStream is = getHttpURLConnectionInputStream(fileDownloadURLPath(localFile.getURI()))) {
                if (localFile.isArchive()) {
                    expand(localFile.getURI(), is, targetPath);
                } else {
                    copy(localFile.getURI(), is, targetPath);
                }
            }
        }
    }
}