Examples with GetNewApplicationRequest org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest used on opensource projects

Example 1 with GetNewApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest in project hadoop by apache.

the class TestApplicationPriorityACLs method prepareForAppSubmission.

private ApplicationSubmissionContext prepareForAppSubmission(String submitter, String queueName, Priority priority) throws Exception {
    GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance();
    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId();
    Resource resource = BuilderUtils.newResource(1024, 1);
    ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, null);
    ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType");
    appSubmissionContext.setApplicationId(applicationId);
    appSubmissionContext.setQueue(queueName);
    if (null != priority) {
        appSubmissionContext.setPriority(priority);
    }
    return appSubmissionContext;
}

Example 2 with GetNewApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest in project hadoop by apache.

the class RMWebServices method createNewApplication.

/**
   * Function that actually creates the ApplicationId by calling the
   * ClientRMService
   * 
   * @return returns structure containing the app-id and maximum resource
   *         capabilities
   */
private NewApplication createNewApplication() {
    GetNewApplicationRequest req = recordFactory.newRecordInstance(GetNewApplicationRequest.class);
    GetNewApplicationResponse resp;
    try {
        resp = rm.getClientRMService().getNewApplication(req);
    } catch (YarnException e) {
        String msg = "Unable to create new app from RM web service";
        LOG.error(msg, e);
        throw new YarnRuntimeException(msg, e);
    }
    NewApplication appId = new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(resp.getMaximumResourceCapability()));
    return appId;
}

Example 3 with GetNewApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest in project hadoop by apache.

the class QueueACLsTestBase method submitAppAndGetAppId.

private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception {
    GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance();
    ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
    ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId();
    Resource resource = BuilderUtils.newResource(1024, 1);
    Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
    ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls);
    ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType");
    appSubmissionContext.setApplicationId(applicationId);
    appSubmissionContext.setQueue(queueName);
    SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext);
    submitterClient.submitApplication(submitRequest);
    resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
    return applicationId;
}

Example 4 with GetNewApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest in project hadoop by apache.

the class TestClientRMTokens method testDelegationToken.

@Test
public void testDelegationToken() throws IOException, InterruptedException {
    final YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_PRINCIPAL, "testuser/localhost@apache.org");
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ResourceScheduler scheduler = createMockScheduler(conf);
    long initialInterval = 10000l;
    long maxLifetime = 20000l;
    long renewInterval = 10000l;
    RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval);
    rmDtSecretManager.startThreads();
    LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval);
    final ClientRMService clientRMService = new ClientRMServiceForTest(conf, scheduler, rmDtSecretManager);
    clientRMService.init(conf);
    clientRMService.start();
    ApplicationClientProtocol clientRMWithDT = null;
    try {
        // Create a user for the renewr and fake the authentication-method
        UserGroupInformation loggedInUser = UserGroupInformation.createRemoteUser("testrenewer@APACHE.ORG");
        Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
        // Default realm is APACHE.ORG
        loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
        org.apache.hadoop.yarn.api.records.Token token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        long tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser1", conf);
        GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Renew after 50% of token age.
        while (System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
            Thread.sleep(500l);
        }
        long nextExpTime = renewDelegationToken(loggedInUser, clientRMService, token);
        long renewalTime = System.currentTimeMillis();
        LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
        // Wait for first expiry, but before renewed expiry.
        while (System.currentTimeMillis() > tokenFetchTime + initialInterval && System.currentTimeMillis() < nextExpTime) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        // Valid token because of renewal.
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Wait for expiry.
        while (System.currentTimeMillis() < renewalTime + renewInterval) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
        // Token should have expired.      
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with an expired token");
        } catch (Exception e) {
            assertEquals(InvalidToken.class.getName(), e.getClass().getName());
            assertTrue(e.getMessage().contains("is expired"));
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        cancelDelegationToken(loggedInUser, clientRMService, token);
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        // Creating a new connection.
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
        // Verify cancellation worked.
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with a cancelled delegation token");
        } catch (IOException e) {
        } catch (YarnException e) {
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        byte[] tokenIdentifierContent = token.getIdentifier().array();
        RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier();
        DataInputBuffer dib = new DataInputBuffer();
        dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
        tokenIdentifier.readFields(dib);
        // Construct new version RMDelegationTokenIdentifier with additional field
        RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest(tokenIdentifier, "message");
        Token<RMDelegationTokenIdentifier> newRMDTtoken = new Token<RMDelegationTokenIdentifier>(newVersionTokenIdentifier, rmDtSecretManager);
        org.apache.hadoop.yarn.api.records.Token newToken = BuilderUtils.newDelegationToken(newRMDTtoken.getIdentifier(), newRMDTtoken.getKind().toString(), newRMDTtoken.getPassword(), newRMDTtoken.getService().toString());
        // Now try talking to RMService using the new version delegation token
        clientRMWithDT = getClientRMProtocolWithDT(newToken, clientRMService.getBindAddress(), "loginuser3", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
    } finally {
        rmDtSecretManager.stopThreads();
        // TODO PRECOMMIT Close proxies.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
        }
    }
}

Example 5 with GetNewApplicationRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest in project hadoop by apache.

the class AMSimulator method submitApp.

private void submitApp() throws YarnException, InterruptedException, IOException {
    // ask for new application
    GetNewApplicationRequest newAppRequest = Records.newRecord(GetNewApplicationRequest.class);
    GetNewApplicationResponse newAppResponse = rm.getClientRMService().getNewApplication(newAppRequest);
    appId = newAppResponse.getApplicationId();
    // submit the application
    final SubmitApplicationRequest subAppRequest = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext appSubContext = Records.newRecord(ApplicationSubmissionContext.class);
    appSubContext.setApplicationId(appId);
    appSubContext.setMaxAppAttempts(1);
    appSubContext.setQueue(queue);
    appSubContext.setPriority(Priority.newInstance(0));
    ContainerLaunchContext conLauContext = Records.newRecord(ContainerLaunchContext.class);
    conLauContext.setApplicationACLs(new HashMap<ApplicationAccessType, String>());
    conLauContext.setCommands(new ArrayList<String>());
    conLauContext.setEnvironment(new HashMap<String, String>());
    conLauContext.setLocalResources(new HashMap<String, LocalResource>());
    conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
    appSubContext.setAMContainerSpec(conLauContext);
    appSubContext.setResource(Resources.createResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB, MR_AM_CONTAINER_RESOURCE_VCORES));
    subAppRequest.setApplicationSubmissionContext(appSubContext);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    ugi.doAs(new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws YarnException, IOException {
            rm.getClientRMService().submitApplication(subAppRequest);
            return null;
        }
    });
    LOG.info(MessageFormat.format("Submit a new application {0}", appId));
}