Examples with Token org.apache.hadoop.yarn.api.records.Token used on opensource projects

Example 46 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class YarnClientImpl method getAMRMToken.

public org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> getAMRMToken(ApplicationId appId) throws YarnException, IOException {
    Token token = getApplicationReport(appId).getAMRMToken();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = null;
    if (token != null) {
        amrmToken = ConverterUtils.convertFromYarn(token, (Text) null);
    }
    return amrmToken;
}

Example 47 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class AMRMClientImpl method updateAMRMToken.

private void updateAMRMToken(Token token) throws IOException {
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token.getIdentifier().array(), token.getPassword().array(), new Text(token.getKind()), new Text(token.getService()));
    // Preserve the token service sent by the RM when adding the token
    // to ensure we replace the previous token setup by the RM.
    // Afterwards we can update the service address for the RPC layer.
    UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
    LOG.info("Updating with new AMRMToken");
    currentUGI.addToken(amrmToken);
    amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}

Example 48 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyTokenRenewal.

/*
   * This test validates the token renewal from the AMRMPRoxy. The test verifies
   * that the received token from AMRMProxy is different from the previous one
   * within 5 requests.
   */
@Test(timeout = 120000)
public void testAMRMProxyTokenRenewal() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
        // RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
        // RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
        conf.setInt(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        LOG.info("testAMRMProxyTokenRenewal - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        Token lastToken = null;
        AllocateResponse response = null;
        for (int i = 0; i < 5; i++) {
            response = client.allocate(request);
            request.setResponseId(request.getResponseId() + 1);
            if (response.getAMRMToken() != null && !response.getAMRMToken().equals(lastToken)) {
                break;
            }
            lastToken = response.getAMRMToken();
            // Time slot to be sure the AMRMProxy renew the token
            Thread.sleep(1500);
        }
        Assert.assertFalse(response.getAMRMToken().equals(lastToken));
        LOG.info("testAMRMPRoxy - Finish Application Master");
        client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    }
}

Example 49 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestYarnApiClasses method testCancelDelegationTokenRequestPBImpl.

/**
  * Test CancelDelegationTokenRequestPBImpl.
  * Test a transformation to prototype and back
  */
@Test
public void testCancelDelegationTokenRequestPBImpl() {
    Token token = getDelegationToken();
    CancelDelegationTokenRequestPBImpl original = new CancelDelegationTokenRequestPBImpl();
    original.setDelegationToken(token);
    CancelDelegationTokenRequestProto protoType = original.getProto();
    CancelDelegationTokenRequestPBImpl copy = new CancelDelegationTokenRequestPBImpl(protoType);
    assertNotNull(copy.getDelegationToken());
    //compare source and converted
    assertEquals(token, copy.getDelegationToken());
}

Example 50 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testOpportunisticAllocation.

/**
   * Tests allocation with requests comprising only opportunistic containers.
   */
@Test(timeout = 60000)
public void testOpportunisticAllocation() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    int oppContainersRequestedAny = amClient.getTable(0).get(priority3, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
    assertEquals(2, oppContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 10;
    Set<ContainerId> releases = new TreeSet<>();
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    while (allocatedContainerCount < oppContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        for (Container container : allocResponse.getAllocatedContainers()) {
            allocatedContainerCount++;
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < oppContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(oppContainersRequestedAny, allocatedContainerCount);
    assertEquals(1, receivedNMTokens.values().size());
}