Examples with LocalResource org.apache.hadoop.yarn.api.records.LocalResource used on opensource projects

Example 51 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project kitten by cloudera.

the class LuaContainerLaunchParameters method constructExtraResource.

private LocalResource constructExtraResource(String key) {
    LocalResource rsrc = Records.newRecord(LocalResource.class);
    rsrc.setType(LocalResourceType.FILE);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    try {
        Path path = new Path(localFileUris.get(key));
        configureLocalResourceForPath(rsrc, path);
    } catch (IOException e) {
        LOG.error("Error constructing extra local resource: " + key, e);
        return null;
    }
    return rsrc;
}

Example 52 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project alluxio by Alluxio.

the class Client method setupContainerLaunchContext.

private void setupContainerLaunchContext() throws IOException, YarnException {
    Map<String, String> applicationMasterArgs = ImmutableMap.<String, String>of("-num_workers", Integer.toString(mNumWorkers), "-master_address", mMasterAddress, "-resource_path", mResourcePath);
    final String amCommand = YarnUtils.buildCommand(YarnContainerType.APPLICATION_MASTER, applicationMasterArgs);
    System.out.println("ApplicationMaster command: " + amCommand);
    mAmContainer.setCommands(Collections.singletonList(amCommand));
    // Setup local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put("alluxio.tar.gz", YarnUtils.createLocalResourceOfFile(mYarnConf, mResourcePath + "/alluxio.tar.gz"));
    localResources.put("alluxio-yarn-setup.sh", YarnUtils.createLocalResourceOfFile(mYarnConf, mResourcePath + "/alluxio-yarn-setup.sh"));
    localResources.put("alluxio.jar", YarnUtils.createLocalResourceOfFile(mYarnConf, mResourcePath + "/alluxio.jar"));
    mAmContainer.setLocalResources(localResources);
    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    mAmContainer.setEnvironment(appMasterEnv);
    // Set up security tokens for launching our ApplicationMaster container.
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials credentials = new Credentials();
        String tokenRenewer = mYarnConf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }
        org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(mYarnConf);
        // getting tokens for the default file-system.
        final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        // getting yarn resource manager token
        org.apache.hadoop.conf.Configuration config = mYarnClient.getConfig();
        Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(mYarnClient.getRMDelegationToken(new org.apache.hadoop.io.Text(tokenRenewer)), ClientRMProxy.getRMDelegationTokenService(config));
        LOG.info("Added RM delegation token: " + token);
        credentials.addToken(token.getService(), token);
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        mAmContainer.setTokens(buffer);
    }
}

Example 53 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project apex-core by apache.

the class LaunchContainerRunnable method run.

/**
   * Connects to CM, sets up container launch context and eventually dispatches the container start request to the CM.
   */
@Override
public void run() {
    LOG.info("Setting up container launch context for containerid={}", container.getId());
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    setClasspath(containerEnv);
    // Setup ACLs for the impersonating user
    try {
        String launchPrincipal = System.getenv("HADOOP_USER_NAME");
        LOG.debug("Launch principal {}", launchPrincipal);
        if (launchPrincipal != null) {
            String launchUserName = launchPrincipal;
            if (UserGroupInformation.isSecurityEnabled()) {
                try {
                    launchUserName = new HadoopKerberosName(launchPrincipal).getShortName();
                } catch (Exception ex) {
                    LOG.warn("Error resolving kerberos principal {}", launchPrincipal, ex);
                }
            }
            LOG.debug("ACL launch user {} current user {}", launchUserName, UserGroupInformation.getCurrentUser().getShortUserName());
            if (!UserGroupInformation.getCurrentUser().getShortUserName().equals(launchUserName)) {
                ACLManager.setupUserACLs(ctx, launchUserName, nmClient.getConfig());
            }
        }
    } catch (IOException e) {
        LOG.warn("Unable to setup user acls for container {}", container.getId(), e);
    }
    try {
        // propagate to replace node managers user name (effective in non-secure mode)
        containerEnv.put("HADOOP_USER_NAME", UserGroupInformation.getLoginUser().getUserName());
    } catch (Exception e) {
        LOG.error("Failed to retrieve principal name", e);
    }
    // Set the environment
    ctx.setEnvironment(containerEnv);
    ctx.setTokens(tokens);
    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<>();
    // add resources for child VM
    try {
        // child VM dependencies
        try (FileSystem fs = StramClientUtils.newFileSystemInstance(nmClient.getConfig())) {
            addFilesToLocalResources(LocalResourceType.FILE, dag.getAttributes().get(Context.DAGContext.LIBRARY_JARS), localResources, fs);
            String archives = dag.getAttributes().get(LogicalPlan.ARCHIVES);
            if (archives != null) {
                addFilesToLocalResources(LocalResourceType.ARCHIVE, archives, localResources, fs);
            }
            ctx.setLocalResources(localResources);
        }
    } catch (IOException e) {
        LOG.error("Failed to prepare local resources.", e);
        return;
    }
    // Set the necessary command to execute on the allocated container
    List<CharSequence> vargs = getChildVMCommand(container.getId().toString());
    // Get final command
    StringBuilder command = new StringBuilder(1024);
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }
    LOG.info("Launching on node: {} command: {}", container.getNodeId(), command);
    List<String> commands = new ArrayList<>();
    commands.add(command.toString());
    ctx.setCommands(commands);
    nmClient.startContainerAsync(container, ctx);
}

Example 54 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project asterixdb by apache.

the class AsterixApplicationMaster method distributeAsterixConfig.

/**
     * Sets up the parameters for the Asterix config.
     *
     * @throws IOException
     */
private void distributeAsterixConfig() throws IOException {
    FileSystem fs = FileSystem.get(conf);
    String pathSuffix = instanceConfPath + File.separator + ASTERIX_CONF_NAME;
    Path dst = new Path(dfsBasePath, pathSuffix);
    URI paramLocation = dst.toUri();
    FileStatus paramFileStatus = fs.getFileStatus(dst);
    Long paramLen = paramFileStatus.getLen();
    Long paramTimestamp = paramFileStatus.getModificationTime();
    LocalResource asterixParamLoc = Records.newRecord(LocalResource.class);
    asterixParamLoc.setType(LocalResourceType.FILE);
    asterixParamLoc.setVisibility(LocalResourceVisibility.PRIVATE);
    asterixParamLoc.setResource(ConverterUtils.getYarnUrlFromURI(paramLocation));
    asterixParamLoc.setTimestamp(paramTimestamp);
    asterixParamLoc.setSize(paramLen);
    localResources.put(ASTERIX_CONF_NAME, asterixParamLoc);
}

Example 55 with LocalResource

use of org.apache.hadoop.yarn.api.records.LocalResource in project ignite by apache.

the class IgniteYarnUtils method setupFile.

/**
     * @param file Path.
     * @param fs File system.
     * @param type Local resource type.
     * @throws Exception If failed.
     */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type) throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);
    file = fs.makeQualified(file);
    FileStatus stat = fs.getFileStatus(file);
    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    return resource;
}