Examples with ContainerTokenIdentifier org.apache.hadoop.yarn.security.ContainerTokenIdentifier used on opensource projects

Example 6 with ContainerTokenIdentifier

use of org.apache.hadoop.yarn.security.ContainerTokenIdentifier in project hadoop by apache.

the class TestNMContainerTokenSecretManager method testRecovery.

@Test
public void testRecovery() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    final NodeId nodeId = NodeId.newInstance("somehost", 1234);
    final ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    final ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2);
    ContainerTokenKeyGeneratorForTest keygen = new ContainerTokenKeyGeneratorForTest(conf);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NMContainerTokenSecretManager secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    MasterKey currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    ContainerTokenIdentifier tokenId1 = createContainerTokenId(cid1, nodeId, "user1", secretMgr);
    ContainerTokenIdentifier tokenId2 = createContainerTokenId(cid2, nodeId, "user2", secretMgr);
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // restart and verify tokens still valid
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key and start a container
    secretMgr.startContainerSuccessful(tokenId2);
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    // restart and verify tokens still valid due to prev key persist
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key again, restart, and verify keys no longer valid
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    try {
        secretMgr.retrievePassword(tokenId1);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    try {
        secretMgr.retrievePassword(tokenId2);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    stateStore.close();
}

Example 7 with ContainerTokenIdentifier

use of org.apache.hadoop.yarn.security.ContainerTokenIdentifier in project hadoop by apache.

the class TestLogAggregationService method createContainer.

private ContainerId createContainer(ApplicationAttemptId appAttemptId1, long cId, ContainerType containerType) {
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId1, cId);
    Resource r = BuilderUtils.newResource(1024, 1);
    ContainerTokenIdentifier containerToken = new ContainerTokenIdentifier(containerId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 100000L, 123, DUMMY_RM_IDENTIFIER, Priority.newInstance(0), 0, null, null, containerType);
    Container container = mock(Container.class);
    context.getContainers().put(containerId, container);
    when(container.getContainerTokenIdentifier()).thenReturn(containerToken);
    when(container.getContainerId()).thenReturn(containerId);
    return containerId;
}

Example 8 with ContainerTokenIdentifier

use of org.apache.hadoop.yarn.security.ContainerTokenIdentifier in project hadoop by apache.

the class RMContainerTokenSecretManager method createContainerToken.

/**
   * Helper function for creating ContainerTokens
   *
   * @param containerId Container Id
   * @param containerVersion Container version
   * @param nodeId Node Id
   * @param appSubmitter App Submitter
   * @param capability Capability
   * @param priority Priority
   * @param createTime Create Time
   * @param logAggregationContext Log Aggregation Context
   * @param nodeLabelExpression Node Label Expression
   * @param containerType Container Type
   * @return the container-token
   */
public Token createContainerToken(ContainerId containerId, int containerVersion, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, String nodeLabelExpression, ContainerType containerType) {
    byte[] password;
    ContainerTokenIdentifier tokenIdentifier;
    long expiryTimeStamp = System.currentTimeMillis() + containerTokenExpiryInterval;
    // Lock so that we use the same MasterKey's keyId and its bytes
    this.readLock.lock();
    try {
        tokenIdentifier = new ContainerTokenIdentifier(containerId, containerVersion, nodeId.toString(), appSubmitter, capability, expiryTimeStamp, this.currentMasterKey.getMasterKey().getKeyId(), ResourceManager.getClusterTimeStamp(), priority, createTime, logAggregationContext, nodeLabelExpression, containerType, ExecutionType.GUARANTEED);
        password = this.createPassword(tokenIdentifier);
    } finally {
        this.readLock.unlock();
    }
    return BuilderUtils.newContainerToken(nodeId, password, tokenIdentifier);
}

Example 9 with ContainerTokenIdentifier

use of org.apache.hadoop.yarn.security.ContainerTokenIdentifier in project hadoop by apache.

the class TestContainerManager method testNullTokens.

/* Test added to verify fix in YARN-644 */
@Test
public void testNullTokens() throws Exception {
    ContainerManagerImpl cMgrImpl = new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler);
    String strExceptionMsg = "";
    try {
        cMgrImpl.authorizeStartAndResourceIncreaseRequest(null, new ContainerTokenIdentifier(), true);
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
    strExceptionMsg = "";
    try {
        cMgrImpl.authorizeStartAndResourceIncreaseRequest(new NMTokenIdentifier(), null, true);
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_CONTAINERTOKEN_MSG);
    strExceptionMsg = "";
    try {
        cMgrImpl.authorizeGetAndStopContainerRequest(null, null, true, null);
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
    strExceptionMsg = "";
    try {
        cMgrImpl.authorizeUser(null, null);
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
    ContainerManagerImpl spyContainerMgr = Mockito.spy(cMgrImpl);
    UserGroupInformation ugInfo = UserGroupInformation.createRemoteUser("a");
    Mockito.when(spyContainerMgr.getRemoteUgi()).thenReturn(ugInfo);
    Mockito.when(spyContainerMgr.selectNMTokenIdentifier(ugInfo)).thenReturn(null);
    strExceptionMsg = "";
    try {
        spyContainerMgr.stopContainers(new StopContainersRequestPBImpl());
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
    strExceptionMsg = "";
    try {
        spyContainerMgr.getContainerStatuses(new GetContainerStatusesRequestPBImpl());
    } catch (YarnException ye) {
        strExceptionMsg = ye.getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
    Mockito.doNothing().when(spyContainerMgr).authorizeUser(ugInfo, null);
    List<StartContainerRequest> reqList = new ArrayList<>();
    reqList.add(StartContainerRequest.newInstance(null, null));
    StartContainersRequest reqs = new StartContainersRequestPBImpl();
    reqs.setStartContainerRequests(reqList);
    strExceptionMsg = "";
    try {
        spyContainerMgr.startContainers(reqs);
    } catch (YarnException ye) {
        strExceptionMsg = ye.getCause().getMessage();
    }
    Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_CONTAINERTOKEN_MSG);
}

Example 10 with ContainerTokenIdentifier

use of org.apache.hadoop.yarn.security.ContainerTokenIdentifier in project hadoop by apache.

the class TestApplication method testAppFinishedOnRunningContainers.

@Test
@SuppressWarnings("unchecked")
public void testAppFinishedOnRunningContainers() {
    WrappedApplication wa = null;
    try {
        wa = new WrappedApplication(4, 314159265358979L, "yak", 3);
        wa.initApplication();
        wa.initContainer(-1);
        assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
        wa.applicationInited();
        assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
        wa.containerFinished(0);
        assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
        assertEquals(2, wa.app.getContainers().size());
        wa.appFinished();
        assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT, wa.app.getApplicationState());
        assertEquals(2, wa.app.getContainers().size());
        for (int i = 1; i < wa.containers.size(); i++) {
            verify(wa.containerBus).handle(argThat(new ContainerKillMatcher(wa.containers.get(i).getContainerId())));
        }
        wa.containerFinished(1);
        assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT, wa.app.getApplicationState());
        assertEquals(1, wa.app.getContainers().size());
        reset(wa.localizerBus);
        wa.containerFinished(2);
        // All containers finished. Cleanup should be called.
        assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP, wa.app.getApplicationState());
        assertEquals(0, wa.app.getContainers().size());
        verify(wa.localizerBus).handle(refEq(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, wa.app)));
        verify(wa.auxBus).handle(refEq(new AuxServicesEvent(AuxServicesEventType.APPLICATION_STOP, wa.appId)));
        wa.appResourcesCleanedup();
        for (Container container : wa.containers) {
            ContainerTokenIdentifier identifier = wa.getContainerTokenIdentifier(container.getContainerId());
            waitForContainerTokenToExpire(identifier);
            Assert.assertTrue(wa.context.getContainerTokenSecretManager().isValidStartContainerRequest(identifier));
        }
        assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
    } finally {
        if (wa != null)
            wa.finished();
    }
}