Examples with LocalFile org.apache.twill.api.LocalFile used on opensource projects

Example 1 with LocalFile

use of org.apache.twill.api.LocalFile in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method populateRunnableLocalFiles.

/**
 * Based on the given {@link TwillSpecification}, copy file to local filesystem.
 * @param spec The {@link TwillSpecification} for populating resource.
 */
private Map<String, Collection<LocalFile>> populateRunnableLocalFiles(TwillSpecification spec, Path stagingDir) throws IOException {
    Map<String, Collection<LocalFile>> localFiles = new HashMap<>();
    LOG.debug("Populating Runnable LocalFiles");
    for (Map.Entry<String, RuntimeSpecification> entry : spec.getRunnables().entrySet()) {
        String runnableName = entry.getKey();
        for (LocalFile localFile : entry.getValue().getLocalFiles()) {
            LocalFile resolvedLocalFile = resolveLocalFile(localFile, stagingDir);
            localFiles.computeIfAbsent(runnableName, s -> new ArrayList<>()).add(resolvedLocalFile);
            LOG.debug("Added file {}", resolvedLocalFile.getURI());
        }
    }
    LOG.debug("Done Runnable LocalFiles");
    return localFiles;
}

Example 2 with LocalFile

use of org.apache.twill.api.LocalFile in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method saveSpecification.

private TwillRuntimeSpecification saveSpecification(TwillSpecification spec, Path targetFile, Path stagingDir) throws IOException {
    final Map<String, Collection<LocalFile>> runnableLocalFiles = populateRunnableLocalFiles(spec, stagingDir);
    // Rewrite LocalFiles inside twillSpec
    Map<String, RuntimeSpecification> runtimeSpec = spec.getRunnables().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> {
        RuntimeSpecification value = e.getValue();
        return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(), value.getResourceSpecification(), runnableLocalFiles.getOrDefault(e.getKey(), Collections.emptyList()));
    }));
    // Serialize into a local temp file.
    LOG.debug("Creating {}", targetFile);
    try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
        EventHandlerSpecification eventHandler = spec.getEventHandler();
        if (eventHandler == null) {
            eventHandler = new LogOnlyEventHandler().configure();
        }
        TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(), eventHandler);
        Map<String, String> configMap = Maps.newHashMap();
        for (Map.Entry<String, String> entry : hConf) {
            if (entry.getKey().startsWith("twill.")) {
                configMap.put(entry.getKey(), entry.getValue());
            }
        }
        TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(newTwillSpec, "", URI.create("."), "", RunIds.fromString(programRunId.getRun()), twillSpec.getName(), null, logLevels, maxRetries, configMap, runnableConfigs);
        TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
        LOG.debug("Done {}", targetFile);
        return twillRuntimeSpec;
    }
}

Example 3 with LocalFile

use of org.apache.twill.api.LocalFile in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method resolveLocalFile.

private LocalFile resolveLocalFile(LocalFile localFile, Path stagingDir) throws IOException {
    URI uri = localFile.getURI();
    String scheme = uri.getScheme();
    // If local file, resolve the last modified time and the file size
    if (scheme == null || "file".equals(scheme)) {
        File file = new File(uri.getPath());
        return new DefaultLocalFile(localFile.getName(), uri, file.lastModified(), file.length(), localFile.isArchive(), localFile.getPattern());
    }
    // If have the same scheme as the location factory, resolve time and size using Location
    if (Objects.equals(locationFactory.getHomeLocation().toURI().getScheme(), scheme)) {
        Location location = locationFactory.create(uri);
        return new DefaultLocalFile(localFile.getName(), uri, location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern());
    }
    // For other cases, attempt to save the URI content to local file, using support URLSteamHandler
    try (InputStream input = uri.toURL().openStream()) {
        Path tempFile = Files.createTempFile(stagingDir, localFile.getName(), Paths.getExtension(localFile.getName()));
        Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING);
        BasicFileAttributes attrs = Files.readAttributes(tempFile, BasicFileAttributes.class);
        return new DefaultLocalFile(localFile.getName(), tempFile.toUri(), attrs.lastModifiedTime().toMillis(), attrs.size(), localFile.isArchive(), localFile.getPattern());
    }
}

Example 4 with LocalFile

use of org.apache.twill.api.LocalFile in project cdap by caskdata.

the class RuntimeJobTwillPreparer method launch.

@Override
protected void launch(TwillRuntimeSpecification twillRuntimeSpec, RuntimeSpecification runtimeSpec, JvmOptions jvmOptions, Map<String, String> environments, Map<String, LocalFile> localFiles, TimeoutChecker timeoutChecker) throws Exception {
    try (RuntimeJobManager jobManager = jobManagerSupplier.get()) {
        timeoutChecker.throwIfTimeout();
        Map<String, LocalFile> localizeFiles = new HashMap<>(localFiles);
        for (Map.Entry<String, Location> secretFile : secretFiles.entrySet()) {
            Location secretFileLocation = secretFile.getValue();
            localizeFiles.put(secretFile.getKey(), new DefaultLocalFile(secretFile.getKey(), secretFileLocation.toURI(), secretFileLocation.lastModified(), secretFileLocation.length(), false, null));
        }
        RuntimeJobInfo runtimeJobInfo = createRuntimeJobInfo(runtimeSpec, localizeFiles, jvmOptions.getRunnableExtraOptions(runtimeSpec.getName()));
        LOG.info("Starting runnable {} for runId {} with job manager.", runtimeSpec.getName(), getProgramRunId());
        // launch job using job manager
        jobManager.launch(runtimeJobInfo);
    }
}

Example 5 with LocalFile

use of org.apache.twill.api.LocalFile in project cdap by caskdata.

the class TetheringRuntimeJobManager method getLocalFileAsCompressedBytes.

/**
 * Use GZIPOutputStream to compress the LocalFile
 */
@VisibleForTesting
byte[] getLocalFileAsCompressedBytes(LocalFile localFile) throws IOException {
    File file = new File(localFile.getURI());
    // use 500kb buffer
    byte[] buffer = new byte[1024 * 500];
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (GZIPOutputStream os = new GZIPOutputStream(baos);
        FileInputStream fis = new FileInputStream(file)) {
        int length;
        while ((length = fis.read(buffer)) > 0) {
            os.write(buffer, 0, length);
        }
    }
    return baos.toByteArray();
}