Examples with LocalizeResource co.cask.cdap.internal.app.runtime.distributed.LocalizeResource used on opensource projects

Example 1 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class MapReduceRuntimeService method localizeUserResources.

/**
   * Localizes resources requested by users in the MapReduce Program's beforeSubmit phase.
   * In Local mode, also copies resources to a temporary directory.
   *
   * @param job the {@link Job} for this MapReduce program
   * @param targetDir in local mode, a temporary directory to copy the resources to
   * @return a {@link Map} of resource name to the resource path. The resource path will be absolute in local mode,
   * while it will just contain the file name in distributed mode.
   */
private Map<String, String> localizeUserResources(Job job, File targetDir) throws IOException {
    Map<String, String> localizedResources = new HashMap<>();
    Map<String, LocalizeResource> resourcesToLocalize = context.getResourcesToLocalize();
    for (Map.Entry<String, LocalizeResource> entry : resourcesToLocalize.entrySet()) {
        String localizedFilePath;
        String name = entry.getKey();
        Configuration mapredConf = job.getConfiguration();
        if (MapReduceTaskContextProvider.isLocal(mapredConf)) {
            // in local mode, also add localize resources in a temporary directory
            localizedFilePath = LocalizationUtils.localizeResource(entry.getKey(), entry.getValue(), targetDir).getAbsolutePath();
        } else {
            URI uri = entry.getValue().getURI();
            // in distributed mode, use the MapReduce Job object to localize resources
            URI actualURI;
            try {
                actualURI = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), name);
            } catch (URISyntaxException e) {
                // If it does though, there is nothing that clients can do to recover, so not propagating a checked exception.
                throw Throwables.propagate(e);
            }
            if (entry.getValue().isArchive()) {
                job.addCacheArchive(actualURI);
            } else {
                job.addCacheFile(actualURI);
            }
            localizedFilePath = name;
        }
        LOG.debug("MapReduce Localizing file {} {}", entry.getKey(), entry.getValue());
        localizedResources.put(name, localizedFilePath);
    }
    return localizedResources;
}

Example 2 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class MapReduceContainerHelper method localizeFramework.

/**
   * Sets the resources that need to be localized to program runner twill container that used as MapReduce client.
   *
   * @param hConf The hadoop configuration
   * @param result the map to be updated with the localized resources.
   * @return the result map
   */
public static <T extends Map<String, LocalizeResource>> T localizeFramework(Configuration hConf, T result) {
    try {
        URI frameworkURI = getFrameworkURI(hConf);
        // If MR Application framework is used, need to localize the framework file to Twill container
        if (frameworkURI != null) {
            URI uri = new URI(frameworkURI.getScheme(), frameworkURI.getAuthority(), frameworkURI.getPath(), null, null);
            result.put(frameworkURI.getFragment(), new LocalizeResource(uri, true));
        }
        return result;
    } catch (URISyntaxException e) {
        // Shouldn't happen since the frameworkURI is already parsed.
        throw Throwables.propagate(e);
    }
}

Example 3 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class LocalizationUtilsTest method testRemoteFile.

@Test
public void testRemoteFile() throws IOException {
    File directory = TEMP_FOLDER.newFolder("ftp");
    File ftpFile = new File(directory, "ftp_file");
    String ftpFileContents = "Contents of ftp_file";
    FileSystem fileSystem = new UnixFakeFileSystem();
    fileSystem.add(new FileEntry(ftpFile.getAbsolutePath(), ftpFileContents));
    String user = "user";
    String password = "password";
    FakeFtpServer ftpServer = new FakeFtpServer();
    // Use any available port
    ftpServer.setServerControlPort(0);
    ftpServer.addUserAccount(new UserAccount(user, password, directory.getAbsolutePath()));
    ftpServer.setFileSystem(fileSystem);
    ftpServer.start();
    try {
        URI uri = URI.create(String.format("ftp://%s:%s@localhost:%d/%s", user, password, ftpServer.getServerControlPort(), ftpFile.getName()));
        File localizationDir = TEMP_FOLDER.newFolder("localRemote");
        File localizedResource = LocalizationUtils.localizeResource("file1", new LocalizeResource(uri, false), localizationDir);
        Assert.assertTrue(localizedResource.exists());
        Assert.assertTrue(localizedResource.isFile());
        Assert.assertEquals(ftpFileContents, com.google.common.io.Files.toString(localizedResource, Charsets.UTF_8));
    } finally {
        ftpServer.stop();
    }
}

Example 4 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class LocalizationUtilsTest method testGz.

@Test(expected = IllegalArgumentException.class)
public void testGz() throws IOException {
    String gzFileName = "target";
    File directory = TEMP_FOLDER.newFolder("gz");
    File source = File.createTempFile("source", ".txt", directory);
    File gzFile = createGzFile(gzFileName, source);
    File localizationDir = TEMP_FOLDER.newFolder("localGz");
    LocalizationUtils.localizeResource(gzFileName, new LocalizeResource(gzFile, true), localizationDir);
}

Example 5 with LocalizeResource

use of co.cask.cdap.internal.app.runtime.distributed.LocalizeResource in project cdap by caskdata.

the class LocalizationUtilsTest method testZip.

@Test
public void testZip() throws IOException {
    String zipFileName = "target";
    File directory = TEMP_FOLDER.newFolder("zip");
    File file1 = File.createTempFile("file1", ".txt", directory);
    File file2 = File.createTempFile("file2", ".txt", directory);
    File zipFile = createZipFile(zipFileName, directory, false);
    File localizationDir = TEMP_FOLDER.newFolder("localZip");
    File localizedResource = LocalizationUtils.localizeResource(zipFileName, new LocalizeResource(zipFile, true), localizationDir);
    Assert.assertTrue(localizedResource.isDirectory());
    File[] files = localizedResource.listFiles();
    Assert.assertNotNull(files);
    Assert.assertEquals(2, files.length);
    if (file1.getName().equals(files[0].getName())) {
        Assert.assertEquals(file2.getName(), files[1].getName());
    } else {
        Assert.assertEquals(file1.getName(), files[1].getName());
        Assert.assertEquals(file2.getName(), files[0].getName());
    }
}