Examples with LocalResource org.apache.hadoop.yarn.api.records.LocalResource used on opensource projects

Example 46 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestContainer method getMockRsrc.

private static Entry<String, LocalResource> getMockRsrc(Random r, LocalResourceVisibility vis) {
    String name = Long.toHexString(r.nextLong());
    URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
    LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
    return new SimpleEntry<String, LocalResource>(name, rsrc);
}

Example 47 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestContainerManager method setupLocalResources.

private Map<String, LocalResource> setupLocalResources(String fileName, String symLink) throws Exception {
    // ////// Create the resources for the container
    File dir = new File(tmpDir, "dir");
    dir.mkdirs();
    File file = new File(dir, fileName);
    PrintWriter fileWriter = new PrintWriter(file);
    fileWriter.write("Hello World!");
    fileWriter.close();
    URL resourceURL = URL.fromPath(FileContext.getLocalFSFileContext().makeQualified(new Path(file.getAbsolutePath())));
    LocalResource resource = recordFactory.newRecordInstance(LocalResource.class);
    resource.setResource(resourceURL);
    resource.setSize(-1);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    resource.setType(LocalResourceType.FILE);
    resource.setTimestamp(file.lastModified());
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(symLink, resource);
    return localResources;
}

Example 48 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestPBRecordImpl method createResource.

static LocalResource createResource() {
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    assertTrue(ret instanceof LocalResourcePBImpl);
    ret.setResource(URL.fromPath(new Path("hdfs://y.ak:9820/foo/bar")));
    ret.setSize(4344L);
    ret.setTimestamp(3141592653589793L);
    ret.setVisibility(LocalResourceVisibility.PUBLIC);
    return ret;
}

Example 49 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TestNodeManagerReboot method testClearLocalDirWhenNodeReboot.

@Test(timeout = 2000000)
public void testClearLocalDirWhenNodeReboot() throws IOException, YarnException, InterruptedException {
    nm = new MyNodeManager();
    nm.start();
    final ContainerManagementProtocol containerManager = nm.getContainerManager();
    // create files under fileCache
    createFiles(nmLocalDir.getAbsolutePath(), ContainerLocalizer.FILECACHE, 100);
    localResourceDir.mkdirs();
    ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
    // Construct the Container-id
    ContainerId cId = createContainerId();
    URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(localResourceDir.getAbsolutePath())));
    LocalResource localResource = LocalResource.newInstance(localResourceUri, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, -1, localResourceDir.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, localResource);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();
    containerLaunchContext.setCommands(commands);
    NodeId nodeId = nm.getNMContext().getNodeId();
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, destinationFile, nm.getNMContext().getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    final StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    final UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId().toString());
    NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.getApplicationAttemptId(), nodeId, user, 123);
    currentUser.addTokenIdentifier(nmIdentifier);
    currentUser.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws YarnException, IOException {
            nm.getContainerManager().startContainers(allRequests);
            return null;
        }
    });
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    Container container = nm.getNMContext().getContainers().get(request.getContainerIds().get(0));
    final int MAX_TRIES = 20;
    int numTries = 0;
    while (!container.getContainerState().equals(ContainerState.DONE) && numTries <= MAX_TRIES) {
        try {
            Thread.sleep(500);
        } catch (InterruptedException ex) {
        // Do nothing
        }
        numTries++;
    }
    Assert.assertEquals(ContainerState.DONE, container.getContainerState());
    Assert.assertTrue("The container should create a subDir named currentUser: " + user + "under localDir/usercache", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer.USERCACHE) > 0);
    Assert.assertTrue("There should be files or Dirs under nm_private when " + "container is launched", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0);
    // restart the NodeManager
    restartNM(MAX_TRIES);
    checkNumOfLocalDirs();
    verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR + "_DEL_")));
    verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_")));
    verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(user, null, new String[] { destinationFile })));
    verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(null, ContainerLocalizer.USERCACHE + "_DEL_", new String[] {})));
    // restart the NodeManager again
    // this time usercache directory should be empty
    restartNM(MAX_TRIES);
    checkNumOfLocalDirs();
}

Example 50 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project hadoop by apache.

the class TaskAttemptImpl method createCommonContainerLaunchContext.

/**
   * Create the common {@link ContainerLaunchContext} for all attempts.
   * @param applicationACLs 
   */
private static ContainerLaunchContext createCommonContainerLaunchContext(Map<ApplicationAccessType, String> applicationACLs, Configuration conf, Token<JobTokenIdentifier> jobToken, final org.apache.hadoop.mapred.JobID oldJobId, Credentials credentials) {
    // Application resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    // Application environment
    Map<String, String> environment = new HashMap<String, String>();
    // Service data
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    // Tokens
    ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
    try {
        FileSystem remoteFS = FileSystem.get(conf);
        // //////////// Set up JobJar to be localized properly on the remote NM.
        String jobJar = conf.get(MRJobConfig.JAR);
        if (jobJar != null) {
            final Path jobJarPath = new Path(jobJar);
            final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
            Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), jobJarFs.getWorkingDirectory());
            LocalResource rc = createLocalResource(jobJarFs, remoteJobJar, LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
            String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
            rc.setPattern(pattern);
            localResources.put(MRJobConfig.JOB_JAR, rc);
            LOG.info("The job-jar file on the remote FS is " + remoteJobJar.toUri().toASCIIString());
        } else {
            // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
            // mapreduce jar itself which is already on the classpath.
            LOG.info("Job jar is not present. " + "Not adding any jar to the list of resources.");
        }
        // //////////// End of JobJar setup
        // //////////// Set up JobConf to be localized properly on the remote NM.
        Path path = MRApps.getStagingAreaDir(conf, UserGroupInformation.getCurrentUser().getShortUserName());
        Path remoteJobSubmitDir = new Path(path, oldJobId.toString());
        Path remoteJobConfPath = new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE);
        localResources.put(MRJobConfig.JOB_CONF_FILE, createLocalResource(remoteFS, remoteJobConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
        LOG.info("The job-conf file on the remote FS is " + remoteJobConfPath.toUri().toASCIIString());
        // //////////// End of JobConf setup
        // Setup DistributedCache
        MRApps.setupDistributedCache(conf, localResources);
        // Setup up task credentials buffer
        LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container");
        Credentials taskCredentials = new Credentials(credentials);
        // LocalStorageToken is needed irrespective of whether security is enabled
        // or not.
        TokenCache.setJobToken(jobToken, taskCredentials);
        DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
        LOG.info("Size of containertokens_dob is " + taskCredentials.numberOfTokens());
        taskCredentials.writeTokenStorageToStream(containerTokens_dob);
        taskCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
        // Add shuffle secret key
        // The secret key is converted to a JobToken to preserve backwards
        // compatibility with an older ShuffleHandler running on an NM.
        LOG.info("Putting shuffle token in serviceData");
        byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
        if (shuffleSecret == null) {
            LOG.warn("Cannot locate shuffle secret in credentials." + " Using job token as shuffle secret.");
            shuffleSecret = jobToken.getPassword();
        }
        Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(jobToken.getIdentifier(), shuffleSecret, jobToken.getKind(), jobToken.getService());
        serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, ShuffleHandler.serializeServiceData(shuffleToken));
        // add external shuffle-providers - if any
        Collection<String> shuffleProviders = conf.getStringCollection(MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
        if (!shuffleProviders.isEmpty()) {
            Collection<String> auxNames = conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES);
            for (final String shuffleProvider : shuffleProviders) {
                if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
                    // skip built-in shuffle-provider that was already inserted with shuffle secret key
                    continue;
                }
                if (auxNames.contains(shuffleProvider)) {
                    LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
                    // This only serves for INIT_APP notifications
                    // The shuffle service needs to be able to work with the host:port information provided by the AM
                    // (i.e. shuffle services which require custom location / other configuration are not supported)
                    serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
                } else {
                    throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider + " was NOT found in the list of aux-services that are available in this NM." + " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
                }
            }
        }
        MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(), getInitialClasspath(conf), conf);
        if (initialAppClasspath != null) {
            MRApps.addToEnvironment(environment, Environment.APP_CLASSPATH.name(), initialAppClasspath, conf);
        }
    } catch (IOException e) {
        throw new YarnRuntimeException(e);
    }
    // Shell
    environment.put(Environment.SHELL.name(), conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
    // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
    MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
    // Add the env variables passed by the admin
    MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV, MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
    // Construct the actual Container
    // The null fields are per-container and will be constructed for each
    // container separately.
    ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null, serviceData, taskCredentialsBuffer, applicationACLs);
    return container;
}