Examples with ApplicationSubmissionContext org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext used on opensource projects

Example 56 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class TestClientRMService method testGetApplicationResourceUsageReportDummy.

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException, IOException {
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    YarnConfiguration config = new YarnConfiguration();
    RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, null);
    ApplicationResourceUsageReport report = rmAppAttemptImpl.getApplicationResourceUsageReport();
    assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Example 57 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class TestClientRMService method mockSubmitAppRequest.

@SuppressWarnings("deprecation")
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) {
    ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
    Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
    ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    submissionContext.setAMContainerSpec(amContainerSpec);
    submissionContext.setApplicationName(name);
    submissionContext.setQueue(queue);
    submissionContext.setApplicationId(appId);
    submissionContext.setResource(resource);
    submissionContext.setApplicationType(appType);
    submissionContext.setApplicationTags(tags);
    submissionContext.setUnmanagedAM(unmanaged);
    submissionContext.setPriority(Priority.newInstance(0));
    SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class);
    submitRequest.setApplicationSubmissionContext(submissionContext);
    return submitRequest;
}

Example 58 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class FairSchedulerTestBase method createMockRMApp.

protected RMApp createMockRMApp(ApplicationAttemptId attemptId) {
    RMApp app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(attemptId.getApplicationId());
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    when(attempt.getAppAttemptId()).thenReturn(attemptId);
    RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getUnmanagedAM()).thenReturn(false);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    resourceManager.getRMContext().getRMApps().put(attemptId.getApplicationId(), app);
    return app;
}

Example 59 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project jstorm by alibaba.

the class JstormOnYarn method run.

/**
     * Main run function for the client
     *
     * @return true if application completed successfully
     * @throws IOException
     * @throws YarnException
     */
public boolean run() throws IOException, YarnException {
    LOG.info("Running Client");
    jstormClientContext.yarnClient.start();
    YarnClusterMetrics clusterMetrics = jstormClientContext.yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers());
    List<NodeReport> clusterNodeReports = jstormClientContext.yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM, count is " + String.valueOf(clusterNodeReports.size()));
    QueueInfo queueInfo = jstormClientContext.yarnClient.getQueueInfo(this.jstormClientContext.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size());
    List<QueueUserACLInfo> listAclInfo = jstormClientContext.yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name());
        }
    }
    if (jstormClientContext.domainId != null && jstormClientContext.domainId.length() > 0 && jstormClientContext.toCreateDomain) {
        prepareTimelineDomain();
    }
    // Get a new application id
    YarnClientApplication app = jstormClientContext.yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
    // A resource ask cannot exceed the max.
    if (jstormClientContext.amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + jstormClientContext.amMemory + ", max=" + maxMem);
        jstormClientContext.amMemory = maxMem;
    }
    int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);
    if (jstormClientContext.amVCores > maxVCores) {
        LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + jstormClientContext.amVCores + ", max=" + maxVCores);
        jstormClientContext.amVCores = maxVCores;
    }
    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setKeepContainersAcrossApplicationAttempts(jstormClientContext.keepContainers);
    if (!StringUtils.isBlank(jstormClientContext.instanceName) && JOYConstants.CLIIENT_CLASS.equals(jstormClientContext.appName)) {
        appContext.setApplicationName(jstormClientContext.appName + JOYConstants.DOT + jstormClientContext.instanceName);
    } else {
        appContext.setApplicationName(jstormClientContext.appName);
    }
    if (jstormClientContext.attemptFailuresValidityInterval >= 0) {
        appContext.setAttemptFailuresValidityInterval(jstormClientContext.attemptFailuresValidityInterval);
    }
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    FileSystem fs = FileSystem.get(jstormClientContext.conf);
    addToLocalResources(fs, jstormClientContext.appMasterJar, JOYConstants.appMasterJarPath, appId.toString(), localResources, null);
    //add jstormOnYarn's configuration to resources which path is jstorm-yarn.xml
    if (jstormClientContext.confFile == null || jstormClientContext.confFile.isEmpty()) {
        addToLocalResources(fs, JOYConstants.CONF_NAME, JOYConstants.CONF_NAME, appId.toString(), localResources, null);
    } else {
        addToLocalResources(fs, jstormClientContext.confFile, JOYConstants.CONF_NAME, appId.toString(), localResources, null);
    }
    if (jstormClientContext.libJars != null && !jstormClientContext.libJars.isEmpty()) {
        for (String libPath : jstormClientContext.libJars.split(JOYConstants.COMMA)) {
            String[] strArr = libPath.split(JOYConstants.BACKLASH);
            String libName = strArr[strArr.length - 1];
            addToLocalResources(fs, libPath, libName, appId.toString(), localResources, null);
        }
    }
    // Set the log4j properties if needed
    if (!jstormClientContext.log4jPropFile.isEmpty()) {
        addToLocalResources(fs, jstormClientContext.log4jPropFile, JOYConstants.log4jPath, appId.toString(), localResources, null);
    }
    // The shell script has to be made available on the final container(s)
    // where it will be executed.
    String hdfsShellScriptLocation = JOYConstants.EMPTY;
    long hdfsShellScriptLen = 0;
    long hdfsShellScriptTimestamp = 0;
    if (!jstormClientContext.shellScriptPath.isEmpty()) {
        Path shellSrc = new Path(jstormClientContext.shellScriptPath);
        String shellPathSuffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId.toString() + JOYConstants.BACKLASH + JOYConstants.SCRIPT_PATH;
        Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix);
        fs.copyFromLocalFile(false, true, shellSrc, shellDst);
        hdfsShellScriptLocation = shellDst.toUri().toString();
        FileStatus shellFileStatus = fs.getFileStatus(shellDst);
        hdfsShellScriptLen = shellFileStatus.getLen();
        hdfsShellScriptTimestamp = shellFileStatus.getModificationTime();
    }
    if (!jstormClientContext.shellCommand.isEmpty()) {
        addToLocalResources(fs, null, JOYConstants.shellCommandPath, appId.toString(), localResources, jstormClientContext.shellCommand);
    }
    if (jstormClientContext.shellArgs.length > 0) {
        addToLocalResources(fs, null, JOYConstants.shellArgsPath, appId.toString(), localResources, StringUtils.join(jstormClientContext.shellArgs, JOYConstants.BLANK));
    }
    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    Path appMaterJar = new Path(fs.getHomeDirectory(), jstormClientContext.appName + JOYConstants.BACKLASH + appId.toString() + JOYConstants.BACKLASH + JOYConstants.appMasterJarPath);
    FileStatus jarFIleStatus = fs.getFileStatus(appMaterJar);
    // put location of shell script into env
    // using the env info, the application master will create the correct local resource for the
    // eventual containers that will be launched to execute the shell scripts
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp));
    env.put(JOYConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen));
    if (jstormClientContext.domainId != null && jstormClientContext.domainId.length() > 0) {
        env.put(JOYConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN, jstormClientContext.domainId);
    }
    env.put(JOYConstants.APPMASTERJARSCRIPTLOCATION, appMaterJar.toUri().toString());
    env.put(JOYConstants.APPMASTERLEN, Long.toString(jarFIleStatus.getLen()));
    env.put(JOYConstants.APPMASTERTIMESTAMP, Long.toString(jarFIleStatus.getModificationTime()));
    env.put(JOYConstants.APPMASTERJARSCRIPTLOCATION, appMaterJar.toUri().toString());
    env.put(JOYConstants.BINARYFILEDEPLOYPATH, jstormClientContext.deployPath);
    env.put(JOYConstants.INSTANCENAME, jstormClientContext.instanceName);
    // Add AppMaster.jar location to classpath
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()).append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(JOYConstants.DOT + JOYConstants.BACKLASH + JOYConstants.ASTERISK);
    for (String c : jstormClientContext.conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append(JOYConstants.DOT + JOYConstants.BACKLASH + JOYConstants.LOG_PROPERTIES);
    // add the runtime classpath needed for tests to work
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(JOYConstants.COLON);
        classPathEnv.append(System.getProperty(JOYConstants.JAVA_CLASS_PATH));
    }
    env.put(JOYConstants.CLASS_PATH, classPathEnv.toString());
    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);
    // Set java executable command
    vargs.add(Environment.JAVA_HOME.$$() + JOYConstants.JAVA);
    // Set Xmx based on am memory size
    vargs.add(JOYConstants.XMX + jstormClientContext.amMemory + JOYConstants.MB);
    // Set class name
    vargs.add(appMasterMainClass);
    // Set params for Application Master
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.CONTAINER_MEMORY + JOYConstants.BLANK + String.valueOf(jstormClientContext.containerMemory));
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.CONTAINER_VCORES + JOYConstants.BLANK + String.valueOf(jstormClientContext.containerVirtualCores));
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.NUM_CONTAINERS + JOYConstants.BLANK + String.valueOf(jstormClientContext.numContainers));
    if (null != jstormClientContext.nodeLabelExpression) {
        appContext.setNodeLabelExpression(jstormClientContext.nodeLabelExpression);
    }
    vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.PRIORITY + JOYConstants.BLANK + String.valueOf(jstormClientContext.shellCmdPriority));
    for (Map.Entry<String, String> entry : jstormClientContext.shellEnv.entrySet()) {
        vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.SHELL_ENV + JOYConstants.BLANK + entry.getKey() + JOYConstants.EQUAL + entry.getValue());
    }
    if (jstormClientContext.debugFlag) {
        vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.DEBUG);
    }
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + JOYConstants.APPMASTER_STDOUT);
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + JOYConstants.APPMASTER_STDERR);
    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(JOYConstants.BLANK);
    }
    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null);
    // Set up resource type requirements
    Resource capability = Resource.newInstance(jstormClientContext.amMemory, jstormClientContext.amVCores);
    appContext.setResource(capability);
    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce
        Credentials credentials = new Credentials();
        String tokenRenewer = jstormClientContext.conf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }
        // For now, only getting tokens for the default file-system.
        final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }
    appContext.setAMContainerSpec(amContainer);
    // Set the priority for the application master
    Priority pri = Priority.newInstance(jstormClientContext.amPriority);
    appContext.setPriority(pri);
    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(jstormClientContext.amQueue);
    LOG.info("Submitting application to ASM");
    //check configuration
    if (JstormYarnUtils.isUnset(jstormClientContext.conf.get(JOYConstants.INSTANCE_NAME_KEY))) {
        throw new IOException(JOYConstants.INSTANCE_NAME_KEY + " is not set");
    }
    if (JstormYarnUtils.isUnset(jstormClientContext.conf.get(JOYConstants.INSTANCE_DEPLOY_DIR_KEY))) {
        throw new IOException(JOYConstants.INSTANCE_DEPLOY_DIR_KEY + " is not set");
    }
    jstormClientContext.yarnClient.submitApplication(appContext);
    // Monitor the application
    return monitorApplication(appId);
}

Example 60 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class BuilderUtils method newApplicationSubmissionContext.

public static ApplicationSubmissionContext newApplicationSubmissionContext(ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) {
    ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    context.setApplicationId(applicationId);
    context.setApplicationName(applicationName);
    context.setQueue(queue);
    context.setPriority(priority);
    context.setAMContainerSpec(amContainer);
    context.setUnmanagedAM(isUnmanagedAM);
    context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
    context.setMaxAppAttempts(maxAppAttempts);
    context.setResource(resource);
    context.setApplicationType(applicationType);
    return context;
}