Search in sources :

Example 1 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestContainerManagerSecurity method testContainerTokenWithEpoch.

/**
   * This tests whether a containerId is serialized/deserialized with epoch.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws YarnException
   */
private void testContainerTokenWithEpoch(Configuration conf) throws IOException, InterruptedException, YarnException {
    LOG.info("Running test for serializing/deserializing containerIds");
    NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext().getNMTokenSecretManager();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
    NodeManager nm = yarnCluster.getNodeManager(0);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager();
    String user = "test";
    waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
    NodeId nodeId = nm.getNMContext().getNodeId();
    // Both id should be equal.
    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
    // Creating a normal Container Token
    RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getContainerTokenSecretManager();
    Resource r = Resource.newInstance(1230, 2);
    Token containerToken = containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier();
    byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
    containerTokenIdentifier.readFields(dib);
    Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
    Assert.assertEquals(cId.toString(), containerTokenIdentifier.getContainerID().toString());
    Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
    YarnRPC rpc = YarnRPC.create(conf);
    testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
    List<ContainerId> containerIds = new LinkedList<ContainerId>();
    containerIds.add(cId);
    ContainerManagementProtocol proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    GetContainerStatusesResponse res = proxy.getContainerStatuses(GetContainerStatusesRequest.newInstance(containerIds));
    Assert.assertNotNull(res.getContainerStatuses().get(0));
    Assert.assertEquals(cId, res.getContainerStatuses().get(0).getContainerId());
    Assert.assertEquals(cId.toString(), res.getContainerStatuses().get(0).getContainerId().toString());
}
Also used : Resource(org.apache.hadoop.yarn.api.records.Resource) NMTokenSecretManagerInNM(org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) Token(org.apache.hadoop.yarn.api.records.Token) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) YarnRPC(org.apache.hadoop.yarn.ipc.YarnRPC) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) LinkedList(java.util.LinkedList) ContainerTokenIdentifier(org.apache.hadoop.yarn.security.ContainerTokenIdentifier) GetContainerStatusesResponse(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse) NodeManager(org.apache.hadoop.yarn.server.nodemanager.NodeManager) DataInputBuffer(org.apache.hadoop.io.DataInputBuffer) ContainerManagementProtocol(org.apache.hadoop.yarn.api.ContainerManagementProtocol) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) NodeId(org.apache.hadoop.yarn.api.records.NodeId) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId)

Example 2 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class ReservationSystemTestUtil method createRMContext.

public static RMContext createRMContext(Configuration conf) {
    RMContext mockRmContext = Mockito.spy(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null));
    RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
    when(nlm.getQueueResource(any(String.class), anySetOf(String.class), any(Resource.class))).thenAnswer(new Answer<Resource>() {

        @Override
        public Resource answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            return (Resource) args[2];
        }
    });
    when(nlm.getResourceByLabel(any(String.class), any(Resource.class))).thenAnswer(new Answer<Resource>() {

        @Override
        public Resource answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            return (Resource) args[1];
        }
    });
    mockRmContext.setNodeLabelManager(nlm);
    return mockRmContext;
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) ClientToAMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Resource(org.apache.hadoop.yarn.api.records.Resource) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) RMNodeLabelsManager(org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager)

Example 3 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestQueueParsing method testQueueParsingReinitializeWithLabels.

@Test
public void testQueueParsingReinitializeWithLabels() throws IOException {
    nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue"));
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithoutLabels(csConf);
    YarnConfiguration conf = new YarnConfiguration(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(nodeLabelManager);
    capacityScheduler.setConf(conf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(conf);
    capacityScheduler.start();
    csConf = new CapacitySchedulerConfiguration();
    setupQueueConfigurationWithLabels(csConf);
    conf = new YarnConfiguration(csConf);
    capacityScheduler.reinitialize(conf, rmContext);
    checkQueueLabels(capacityScheduler);
    ServiceOperations.stopQuietly(capacityScheduler);
}
Also used : ClientToAMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Test(org.junit.Test)

Example 4 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestQueueParsing method testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager.

@Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfigurationWithLabelsInherit(csConf);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null);
    RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
    nodeLabelsManager.init(conf);
    nodeLabelsManager.start();
    rmContext.setNodeLabelManager(nodeLabelsManager);
    capacityScheduler.setConf(csConf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(csConf);
    capacityScheduler.start();
    ServiceOperations.stopQuietly(capacityScheduler);
    ServiceOperations.stopQuietly(nodeLabelsManager);
}
Also used : ClientToAMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) NullRMNodeLabelsManager(org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) RMNodeLabelsManager(org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager) NullRMNodeLabelsManager(org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager) Test(org.junit.Test)

Example 5 with NMTokenSecretManagerInRM

use of org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM in project hadoop by apache.

the class TestQueueParsing method testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent.

/**
   * Test init a queue configuration, children's capacity for a given label
   * doesn't equals to 100%. This expect IllegalArgumentException thrown.
   */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent() throws IOException {
    nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue"));
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
    setupQueueConfiguration(csConf);
    csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);
    CapacityScheduler capacityScheduler = new CapacityScheduler();
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null);
    rmContext.setNodeLabelManager(nodeLabelManager);
    capacityScheduler.setConf(csConf);
    capacityScheduler.setRMContext(rmContext);
    capacityScheduler.init(csConf);
    capacityScheduler.start();
    ServiceOperations.stopQuietly(capacityScheduler);
}
Also used : ClientToAMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Test(org.junit.Test)

Aggregations

NMTokenSecretManagerInRM (org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM)35 RMContainerTokenSecretManager (org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager)34 RMContextImpl (org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl)30 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)26 ClientToAMTokenSecretManagerInRM (org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM)24 Test (org.junit.Test)22 RMContext (org.apache.hadoop.yarn.server.resourcemanager.RMContext)8 Configuration (org.apache.hadoop.conf.Configuration)7 RMNodeLabelsManager (org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager)7 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)6 Resource (org.apache.hadoop.yarn.api.records.Resource)6 NullRMNodeLabelsManager (org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager)6 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)5 InlineDispatcher (org.apache.hadoop.yarn.event.InlineDispatcher)5 NodeId (org.apache.hadoop.yarn.api.records.NodeId)4 Token (org.apache.hadoop.yarn.api.records.Token)4 NMLivelinessMonitor (org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor)4 NodesListManager (org.apache.hadoop.yarn.server.resourcemanager.NodesListManager)4 ResourceTrackerService (org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService)4 RMApplicationHistoryWriter (org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter)4