Examples with LocalResource org.apache.hadoop.yarn.api.records.LocalResource used on opensource projects

Example 1 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TaskAttemptImpl method createCommonContainerLaunchContext.

/**
   * Create the common {@link ContainerLaunchContext} for all attempts.
   * @param applicationACLs 
   */
private static ContainerLaunchContext createCommonContainerLaunchContext(Map<ApplicationAccessType, String> applicationACLs, Configuration conf, Token<JobTokenIdentifier> jobToken, final org.apache.hadoop.mapred.JobID oldJobId, Credentials credentials) {
    // Application resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    // Application environment
    Map<String, String> environment = new HashMap<String, String>();
    // Service data
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    // Tokens
    ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
    try {
        FileSystem remoteFS = FileSystem.get(conf);
        // //////////// Set up JobJar to be localized properly on the remote NM.
        String jobJar = conf.get(MRJobConfig.JAR);
        if (jobJar != null) {
            final Path jobJarPath = new Path(jobJar);
            final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
            Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), jobJarFs.getWorkingDirectory());
            LocalResource rc = createLocalResource(jobJarFs, remoteJobJar, LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
            String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
            rc.setPattern(pattern);
            localResources.put(MRJobConfig.JOB_JAR, rc);
            LOG.info("The job-jar file on the remote FS is " + remoteJobJar.toUri().toASCIIString());
        } else {
            // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
            // mapreduce jar itself which is already on the classpath.
            LOG.info("Job jar is not present. " + "Not adding any jar to the list of resources.");
        }
        // //////////// End of JobJar setup
        // //////////// Set up JobConf to be localized properly on the remote NM.
        Path path = MRApps.getStagingAreaDir(conf, UserGroupInformation.getCurrentUser().getShortUserName());
        Path remoteJobSubmitDir = new Path(path, oldJobId.toString());
        Path remoteJobConfPath = new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE);
        localResources.put(MRJobConfig.JOB_CONF_FILE, createLocalResource(remoteFS, remoteJobConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
        LOG.info("The job-conf file on the remote FS is " + remoteJobConfPath.toUri().toASCIIString());
        // //////////// End of JobConf setup
        // Setup DistributedCache
        MRApps.setupDistributedCache(conf, localResources);
        // Setup up task credentials buffer
        LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container");
        Credentials taskCredentials = new Credentials(credentials);
        // LocalStorageToken is needed irrespective of whether security is enabled
        // or not.
        TokenCache.setJobToken(jobToken, taskCredentials);
        DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
        LOG.info("Size of containertokens_dob is " + taskCredentials.numberOfTokens());
        taskCredentials.writeTokenStorageToStream(containerTokens_dob);
        taskCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
        // Add shuffle secret key
        // The secret key is converted to a JobToken to preserve backwards
        // compatibility with an older ShuffleHandler running on an NM.
        LOG.info("Putting shuffle token in serviceData");
        byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
        if (shuffleSecret == null) {
            LOG.warn("Cannot locate shuffle secret in credentials." + " Using job token as shuffle secret.");
            shuffleSecret = jobToken.getPassword();
        }
        Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(jobToken.getIdentifier(), shuffleSecret, jobToken.getKind(), jobToken.getService());
        serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, ShuffleHandler.serializeServiceData(shuffleToken));
        // add external shuffle-providers - if any
        Collection<String> shuffleProviders = conf.getStringCollection(MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
        if (!shuffleProviders.isEmpty()) {
            Collection<String> auxNames = conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES);
            for (final String shuffleProvider : shuffleProviders) {
                if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
                    // skip built-in shuffle-provider that was already inserted with shuffle secret key
                    continue;
                }
                if (auxNames.contains(shuffleProvider)) {
                    LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
                    // This only serves for INIT_APP notifications
                    // The shuffle service needs to be able to work with the host:port information provided by the AM
                    // (i.e. shuffle services which require custom location / other configuration are not supported)
                    serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
                } else {
                    throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider + " was NOT found in the list of aux-services that are available in this NM." + " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
                }
            }
        }
        MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(), getInitialClasspath(conf), conf);
        if (initialAppClasspath != null) {
            MRApps.addToEnvironment(environment, Environment.APP_CLASSPATH.name(), initialAppClasspath, conf);
        }
    } catch (IOException e) {
        throw new YarnRuntimeException(e);
    }
    // Shell
    environment.put(Environment.SHELL.name(), conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
    // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
    MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
    // Add the env variables passed by the admin
    MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV, MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
    // Construct the actual Container
    // The null fields are per-container and will be constructed for each
    // container separately.
    ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null, serviceData, taskCredentialsBuffer, applicationACLs);
    return container;
}

Example 2 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method createJarFile.

static LocalResource createJarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException {
    byte[] bytes = new byte[len];
    r.nextBytes(bytes);
    File archiveFile = new File(p.toUri().getPath() + ".jar");
    archiveFile.createNewFile();
    JarOutputStream out = new JarOutputStream(new FileOutputStream(archiveFile));
    out.putNextEntry(new JarEntry(p.getName()));
    out.write(bytes);
    out.closeEntry();
    out.close();
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    ret.setResource(URL.fromPath(new Path(p.toString() + ".jar")));
    ret.setSize(len);
    ret.setType(LocalResourceType.ARCHIVE);
    ret.setVisibility(vis);
    ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar")).getModificationTime());
    return ret;
}

Example 3 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method testUniqueDestinationPath.

@Test(timeout = 10000)
public void testUniqueDestinationPath() throws Exception {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    ExecutorService singleThreadedExec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    Path p = new Path(basedir, "dir" + 0 + ".jar");
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    LocalResource rsrc = createJar(files, p, vis);
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    Future<Path> rPath = singleThreadedExec.submit(fsd);
    singleThreadedExec.shutdown();
    while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(rPath.isDone());
    // Now FSDownload will not create a random directory to localize the
    // resource. Therefore the final localizedPath for the resource should be
    // destination directory (passed as an argument) + file name.
    Assert.assertEquals(destPath, rPath.get().getParent());
}

Example 4 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method testDownloadBadPublic.

@Test(timeout = 10000)
public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    pending.put(rsrc, exec.submit(fsd));
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(pending.get(rsrc).isDone());
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            p.getValue().get();
            Assert.fail("We localized a file that is not public.");
        }
    } catch (ExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof IOException);
    }
}

Example 5 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestFSDownload method testDirDownload.

@Test(timeout = 10000)
public void testDirDownload() throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
        LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
        if (i % 2 == 1) {
            vis = LocalResourceVisibility.APPLICATION;
        }
        Path p = new Path(basedir, "dir" + i + ".jar");
        LocalResource rsrc = createJar(files, p, vis);
        rsrcVis.put(rsrc, vis);
        Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
        destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
        FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
        pending.put(rsrc, exec.submit(fsd));
    }
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    for (Future<Path> path : pending.values()) {
        Assert.assertTrue(path.isDone());
    }
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            Path localized = p.getValue().get();
            FileStatus status = files.getFileStatus(localized);
            System.out.println("Testing path " + localized);
            assert (status.isDirectory());
            assert (rsrcVis.containsKey(p.getKey()));
            verifyPermsRecursively(localized.getFileSystem(conf), files, localized, rsrcVis.get(p.getKey()));
        }
    } catch (ExecutionException e) {
        throw new IOException("Failed exec", e);
    }
}