Examples with GetNewApplicationResponse org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse used on opensource projects

Example 6 with GetNewApplicationResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse in project kitten by cloudera.

the class YarnClientServiceImpl method startUp.

@Override
protected void startUp() throws IOException {
    ByteBuffer serializedTokens = null;
    if (UserGroupInformation.isSecurityEnabled()) {
        Configuration conf = this.yarnClientFactory.getConfig();
        FileSystem fs = FileSystem.get(conf);
        Credentials credentials = new Credentials();
        String tokenRenewer = this.yarnClientFactory.getConfig().get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }
        // For now, only getting tokens for the default file-system.
        final Token<?>[] tokens = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got delegation token for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        serializedTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    }
    this.yarnClient = yarnClientFactory.connect();
    YarnClientApplication clientApp = getNewApplication();
    GetNewApplicationResponse newApp = clientApp.getNewApplicationResponse();
    ContainerLaunchContextFactory clcFactory = new ContainerLaunchContextFactory(newApp.getMaximumResourceCapability(), serializedTokens);
    ApplicationSubmissionContext appContext = clientApp.getApplicationSubmissionContext();
    this.applicationId = appContext.getApplicationId();
    appContext.setApplicationName(parameters.getApplicationName());
    // Setup the container for the application master.
    ContainerLaunchParameters appMasterParams = parameters.getApplicationMasterParameters(applicationId);
    ContainerLaunchContext clc = clcFactory.create(appMasterParams);
    LOG.debug("Master context: " + clc);
    appContext.setResource(clcFactory.createResource(appMasterParams));
    appContext.setQueue(parameters.getQueue());
    appContext.setPriority(clcFactory.createPriority(appMasterParams.getPriority()));
    appContext.setAMContainerSpec(clc);
    submitApplication(appContext);
    // Make sure we stop the application in the case that it isn't done already.
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            if (YarnClientServiceImpl.this.isRunning()) {
                YarnClientServiceImpl.this.stop();
            }
        }
    });
    stopwatch.start();
}

Example 7 with GetNewApplicationResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse in project alluxio by Alluxio.

the class Client method submitApplication.

/**
   * Submits an application to the ResourceManager to run ApplicationMaster.
   *
   * The stable Yarn API provides a convenience method (YarnClient#createApplication) for creating
   * applications and setting up the application submission context. This was not available in the
   * alpha API.
   */
private void submitApplication() throws YarnException, IOException {
    // Initialize a YarnClient
    mYarnClient = YarnClient.createYarnClient();
    mYarnClient.init(mYarnConf);
    mYarnClient.start();
    // Create an application, get and check the information about the cluster
    YarnClientApplication app = mYarnClient.createApplication();
    // Get a response of this application, containing information of the cluster
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // Check if the cluster has enough resource to launch the ApplicationMaster
    checkClusterResource(appResponse);
    // Check that there are enough hosts in the cluster to support the desired number of workers
    checkNodesAvailable();
    // Set up the container launch context for the application master
    mAmContainer = Records.newRecord(ContainerLaunchContext.class);
    setupContainerLaunchContext();
    // Finally, set-up ApplicationSubmissionContext for the application
    mAppContext = app.getApplicationSubmissionContext();
    setupApplicationSubmissionContext();
    // Submit the application to the applications manager.
    // Ignore the response as either a valid response object is returned on success
    // or an exception thrown to denote some form of a failure
    mAppId = mAppContext.getApplicationId();
    System.out.println("Submitting application of id " + mAppId + " to ResourceManager");
    mYarnClient.submitApplication(mAppContext);
    monitorApplication();
}

Example 8 with GetNewApplicationResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse in project alluxio by Alluxio.

the class ClientTest method generateMaxAllocation.

private void generateMaxAllocation(Resource resource) throws Exception {
    YarnClientApplication mockYarnClientApplication = mock(YarnClientApplication.class);
    doReturn(mockYarnClientApplication).when(mYarnClient).createApplication();
    GetNewApplicationResponse mockNewApplicationResponse = mock(GetNewApplicationResponse.class);
    doReturn(mockNewApplicationResponse).when(mockYarnClientApplication).getNewApplicationResponse();
    doReturn(resource).when(mockNewApplicationResponse).getMaximumResourceCapability();
}

Example 9 with GetNewApplicationResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse in project hadoop by apache.

the class TestRM method testGetNewAppId.

@Test
public void testGetNewAppId() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    MockRM rm = new MockRM(conf);
    rm.start();
    GetNewApplicationResponse resp = rm.getNewAppId();
    assert (resp.getApplicationId().getId() != 0);
    assert (resp.getMaximumResourceCapability().getMemorySize() > 0);
    rm.stop();
}

Example 10 with GetNewApplicationResponse

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse in project weave by continuuity.

the class Hadoop20YarnAppClient method createLauncher.

@Override
public ProcessLauncher<ApplicationId> createLauncher(WeaveSpecification weaveSpec) throws Exception {
    // Request for new application
    final GetNewApplicationResponse response = yarnClient.getNewApplication();
    final ApplicationId appId = response.getApplicationId();
    // Setup the context for application submission
    final ApplicationSubmissionContext appSubmissionContext = Records.newRecord(ApplicationSubmissionContext.class);
    appSubmissionContext.setApplicationId(appId);
    appSubmissionContext.setApplicationName(weaveSpec.getName());
    appSubmissionContext.setUser(user);
    ApplicationSubmitter submitter = new ApplicationSubmitter() {

        @Override
        public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext, Resource capability) {
            ContainerLaunchContext context = launchContext.getLaunchContext();
            addRMToken(context);
            context.setUser(appSubmissionContext.getUser());
            context.setResource(adjustMemory(response, capability));
            appSubmissionContext.setAMContainerSpec(context);
            try {
                yarnClient.submitApplication(appSubmissionContext);
                return new ProcessControllerImpl(yarnClient, appId);
            } catch (YarnRemoteException e) {
                LOG.error("Failed to submit application {}", appId, e);
                throw Throwables.propagate(e);
            }
        }
    };
    return new ApplicationMasterProcessLauncher(appId, submitter);
}