Examples with DefaultLocalFile org.apache.twill.internal.DefaultLocalFile used on opensource projects

Example 1 with DefaultLocalFile

use of org.apache.twill.internal.DefaultLocalFile in project cdap by caskdata.

the class AbstractRuntimeTwillPreparer method resolveLocalFile.

private LocalFile resolveLocalFile(LocalFile localFile, Path stagingDir) throws IOException {
    URI uri = localFile.getURI();
    String scheme = uri.getScheme();
    // If local file, resolve the last modified time and the file size
    if (scheme == null || "file".equals(scheme)) {
        File file = new File(uri.getPath());
        return new DefaultLocalFile(localFile.getName(), uri, file.lastModified(), file.length(), localFile.isArchive(), localFile.getPattern());
    }
    // If have the same scheme as the location factory, resolve time and size using Location
    if (Objects.equals(locationFactory.getHomeLocation().toURI().getScheme(), scheme)) {
        Location location = locationFactory.create(uri);
        return new DefaultLocalFile(localFile.getName(), uri, location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern());
    }
    // For other cases, attempt to save the URI content to local file, using support URLSteamHandler
    try (InputStream input = uri.toURL().openStream()) {
        Path tempFile = Files.createTempFile(stagingDir, localFile.getName(), Paths.getExtension(localFile.getName()));
        Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING);
        BasicFileAttributes attrs = Files.readAttributes(tempFile, BasicFileAttributes.class);
        return new DefaultLocalFile(localFile.getName(), tempFile.toUri(), attrs.lastModifiedTime().toMillis(), attrs.size(), localFile.isArchive(), localFile.getPattern());
    }
}

Example 2 with DefaultLocalFile

use of org.apache.twill.internal.DefaultLocalFile in project cdap by caskdata.

the class RuntimeJobTwillPreparer method launch.

@Override
protected void launch(TwillRuntimeSpecification twillRuntimeSpec, RuntimeSpecification runtimeSpec, JvmOptions jvmOptions, Map<String, String> environments, Map<String, LocalFile> localFiles, TimeoutChecker timeoutChecker) throws Exception {
    try (RuntimeJobManager jobManager = jobManagerSupplier.get()) {
        timeoutChecker.throwIfTimeout();
        Map<String, LocalFile> localizeFiles = new HashMap<>(localFiles);
        for (Map.Entry<String, Location> secretFile : secretFiles.entrySet()) {
            Location secretFileLocation = secretFile.getValue();
            localizeFiles.put(secretFile.getKey(), new DefaultLocalFile(secretFile.getKey(), secretFileLocation.toURI(), secretFileLocation.lastModified(), secretFileLocation.length(), false, null));
        }
        RuntimeJobInfo runtimeJobInfo = createRuntimeJobInfo(runtimeSpec, localizeFiles, jvmOptions.getRunnableExtraOptions(runtimeSpec.getName()));
        LOG.info("Starting runnable {} for runId {} with job manager.", runtimeSpec.getName(), getProgramRunId());
        // launch job using job manager
        jobManager.launch(runtimeJobInfo);
    }
}

Example 3 with DefaultLocalFile

use of org.apache.twill.internal.DefaultLocalFile in project cdap by caskdata.

the class TetheringRuntimeJobManagerTest method testGetLocalFileAsCompressedString.

@Test
public void testGetLocalFileAsCompressedString() throws IOException {
    File file = File.createTempFile("test", "xml");
    file.deleteOnExit();
    String fileContents = "contents of test.xml";
    try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
        writer.write(fileContents);
    }
    LocalFile localfile = new DefaultLocalFile(file.getName(), file.toURI(), file.lastModified(), file.length(), false, null);
    byte[] compressedContents = runtimeJobManager.getLocalFileAsCompressedBytes(localfile);
    // test that uncompressed contents matches original file contents
    String uncompressedContents;
    try (GZIPInputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(compressedContents))) {
        uncompressedContents = IOUtils.toString(inputStream);
    }
    Assert.assertEquals(fileContents, uncompressedContents);
}

Example 4 with DefaultLocalFile

use of org.apache.twill.internal.DefaultLocalFile in project cdap by caskdata.

the class KubeTwillPreparer method populateRunnableLocalFiles.

/**
 * Based on the given {@link TwillSpecification}, upload LocalFiles to {@link Location}s.
 *
 * @param spec The {@link TwillSpecification} for populating resource.
 */
private Map<String, Collection<LocalFile>> populateRunnableLocalFiles(TwillSpecification spec) throws IOException {
    Map<String, Collection<LocalFile>> localFiles = new HashMap<>();
    String locationScheme = appLocation.toURI().getScheme();
    for (Map.Entry<String, RuntimeSpecification> entry : spec.getRunnables().entrySet()) {
        String runnableName = entry.getKey();
        Collection<LocalFile> runnableFiles = localFiles.computeIfAbsent(runnableName, k -> new ArrayList<>());
        for (LocalFile localFile : entry.getValue().getLocalFiles()) {
            Location location;
            URI uri = localFile.getURI();
            if (locationScheme.equals(uri.getScheme())) {
                // If the source file location is having the same scheme as the target location, no need to copy
                location = appLocation.getLocationFactory().create(uri);
            } else {
                URL url = uri.toURL();
                LOG.debug("Create and copy {} : {}", runnableName, url);
                // Preserves original suffix for expansion.
                location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName())));
                LOG.debug("Done {} : {}", runnableName, url);
            }
            runnableFiles.add(new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(), location.length(), localFile.isArchive(), localFile.getPattern()));
        }
    }
    return localFiles;
}

Example 5 with DefaultLocalFile

use of org.apache.twill.internal.DefaultLocalFile in project cdap by caskdata.

the class DataprocRuntimeJobManager method uploadFile.

/**
 * Uploads files to gcs.
 */
private LocalFile uploadFile(String bucket, String targetFilePath, LocalFile localFile) throws IOException, StorageException {
    BlobId blobId = BlobId.of(bucket, targetFilePath);
    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/octet-stream").build();
    LOG.debug("Uploading a file of size {} bytes from {} to gs://{}/{} ", localFile.getSize(), localFile.getURI(), bucket, targetFilePath);
    Bucket bucketObj = getStorageClient().get(bucket);
    if (bucketObj != null) {
        LOG.debug("File's Location type : {} and Location : {}. ", bucketObj.getLocationType(), bucketObj.getLocation());
    }
    try (InputStream inputStream = openStream(localFile.getURI());
        WriteChannel writer = getStorageClient().writer(blobInfo)) {
        ByteStreams.copy(inputStream, Channels.newOutputStream(writer));
    }
    LOG.debug("Successfully Uploaded file : {}.", localFile.getURI());
    return new DefaultLocalFile(localFile.getName(), URI.create(String.format("gs://%s/%s", bucket, targetFilePath)), localFile.getLastModified(), localFile.getSize(), localFile.isArchive(), localFile.getPattern());
}