Examples with NodeId org.apache.hadoop.yarn.api.records.NodeId used on opensource projects

Example 31 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class NodeLabelTestBase method assertMapContains.

public static void assertMapContains(Map<NodeId, Set<String>> expected, ImmutableMap<NodeId, Set<String>> actual) {
    for (NodeId k : actual.keySet()) {
        Assert.assertTrue(expected.containsKey(k));
        assertCollectionEquals(expected.get(k), actual.get(k));
    }
}

Example 32 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestDistributedScheduler method testDistributedScheduler.

@Test
public void testDistributedScheduler() throws Exception {
    Configuration conf = new Configuration();
    DistributedScheduler distributedScheduler = new DistributedScheduler();
    RequestInterceptor finalReqIntcptr = setup(conf, distributedScheduler);
    registerAM(distributedScheduler, finalReqIntcptr, Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("a", 1), "http://a:1"), RemoteNode.newInstance(NodeId.newInstance("b", 2), "http://b:2")));
    final AtomicBoolean flipFlag = new AtomicBoolean(true);
    Mockito.when(finalReqIntcptr.allocateForDistributedScheduling(Mockito.any(DistributedSchedulingAllocateRequest.class))).thenAnswer(new Answer<DistributedSchedulingAllocateResponse>() {

        @Override
        public DistributedSchedulingAllocateResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
            flipFlag.set(!flipFlag.get());
            if (flipFlag.get()) {
                return createAllocateResponse(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("c", 3), "http://c:3"), RemoteNode.newInstance(NodeId.newInstance("d", 4), "http://d:4")));
            } else {
                return createAllocateResponse(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("d", 4), "http://d:4"), RemoteNode.newInstance(NodeId.newInstance("c", 3), "http://c:3")));
            }
        }
    });
    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
    ResourceRequest guaranteedReq = createResourceRequest(ExecutionType.GUARANTEED, 5, "*");
    ResourceRequest opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 4, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    // Verify 4 containers were allocated
    AllocateResponse allocateResponse = distributedScheduler.allocate(allocateRequest);
    Assert.assertEquals(4, allocateResponse.getAllocatedContainers().size());
    // Verify equal distribution on hosts a and b, and none on c or d
    Map<NodeId, List<ContainerId>> allocs = mapAllocs(allocateResponse, 4);
    Assert.assertEquals(2, allocs.get(NodeId.newInstance("a", 1)).size());
    Assert.assertEquals(2, allocs.get(NodeId.newInstance("b", 2)).size());
    Assert.assertNull(allocs.get(NodeId.newInstance("c", 3)));
    Assert.assertNull(allocs.get(NodeId.newInstance("d", 4)));
    // New Allocate request
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 6, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    // Verify 6 containers were allocated
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    Assert.assertEquals(6, allocateResponse.getAllocatedContainers().size());
    // Verify new containers are equally distribution on hosts c and d,
    // and none on a or b
    allocs = mapAllocs(allocateResponse, 6);
    Assert.assertEquals(3, allocs.get(NodeId.newInstance("c", 3)).size());
    Assert.assertEquals(3, allocs.get(NodeId.newInstance("d", 4)).size());
    Assert.assertNull(allocs.get(NodeId.newInstance("a", 1)));
    Assert.assertNull(allocs.get(NodeId.newInstance("b", 2)));
    // Ensure the DistributedScheduler respects the list order..
    // The first request should be allocated to "d" since it is ranked higher
    // The second request should be allocated to "c" since the ranking is
    // flipped on every allocate response.
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("d", 4)).size());
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("c", 3)).size());
    allocateRequest = Records.newRecord(AllocateRequest.class);
    opportunisticReq = createResourceRequest(ExecutionType.OPPORTUNISTIC, 1, "*");
    allocateRequest.setAskList(Arrays.asList(guaranteedReq, opportunisticReq));
    allocateResponse = distributedScheduler.allocate(allocateRequest);
    allocs = mapAllocs(allocateResponse, 1);
    Assert.assertEquals(1, allocs.get(NodeId.newInstance("d", 4)).size());
}

Example 33 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestLogAggregationService method setup.

@Override
@SuppressWarnings("unchecked")
public void setup() throws IOException {
    super.setup();
    NodeId nodeId = NodeId.newInstance("0.0.0.0", 5555);
    ((NMContext) context).setNodeId(nodeId);
    dispatcher = createDispatcher();
    appEventHandler = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, appEventHandler);
    UserGroupInformation.setConfiguration(conf);
}

Example 34 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestNMContainerTokenSecretManager method testRecovery.

@Test
public void testRecovery() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    final NodeId nodeId = NodeId.newInstance("somehost", 1234);
    final ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    final ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2);
    ContainerTokenKeyGeneratorForTest keygen = new ContainerTokenKeyGeneratorForTest(conf);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NMContainerTokenSecretManager secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    MasterKey currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    ContainerTokenIdentifier tokenId1 = createContainerTokenId(cid1, nodeId, "user1", secretMgr);
    ContainerTokenIdentifier tokenId2 = createContainerTokenId(cid2, nodeId, "user2", secretMgr);
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // restart and verify tokens still valid
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key and start a container
    secretMgr.startContainerSuccessful(tokenId2);
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    // restart and verify tokens still valid due to prev key persist
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    assertNotNull(secretMgr.retrievePassword(tokenId1));
    assertNotNull(secretMgr.retrievePassword(tokenId2));
    // roll master key again, restart, and verify keys no longer valid
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr = new NMContainerTokenSecretManager(conf, stateStore);
    secretMgr.setNodeId(nodeId);
    secretMgr.recover();
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isValidStartContainerRequest(tokenId1));
    assertFalse(secretMgr.isValidStartContainerRequest(tokenId2));
    try {
        secretMgr.retrievePassword(tokenId1);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    try {
        secretMgr.retrievePassword(tokenId2);
        fail("token should not be valid");
    } catch (InvalidToken e) {
    // expected
    }
    stateStore.close();
}

Example 35 with NodeId

use of org.apache.hadoop.yarn.api.records.NodeId in project hadoop by apache.

the class TestNMTokenSecretManagerInNM method testRecovery.

@Test
public void testRecovery() throws IOException {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    final NodeId nodeId = NodeId.newInstance("somehost", 1234);
    final ApplicationAttemptId attempt1 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
    final ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(ApplicationId.newInstance(2, 2), 2);
    NMTokenKeyGeneratorForTest keygen = new NMTokenKeyGeneratorForTest();
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NMTokenSecretManagerInNM secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.setNodeId(nodeId);
    MasterKey currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    NMTokenIdentifier attemptToken1 = getNMTokenId(secretMgr.createNMToken(attempt1, nodeId, "user1"));
    NMTokenIdentifier attemptToken2 = getNMTokenId(secretMgr.createNMToken(attempt2, nodeId, "user2"));
    secretMgr.appAttemptStartContainer(attemptToken1);
    secretMgr.appAttemptStartContainer(attemptToken2);
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // restart and verify key is still there and token still valid
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // roll master key and remove an app
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr.appFinished(attempt1.getApplicationId());
    // restart and verify attempt1 key is still valid due to prev key persist
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    assertNotNull(secretMgr.retrievePassword(attemptToken1));
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // roll master key again, restart, and verify attempt1 key is bad but
    // attempt2 is still good due to app key persist
    currentKey = keygen.generateKey();
    secretMgr.setMasterKey(currentKey);
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    try {
        secretMgr.retrievePassword(attemptToken1);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    assertNotNull(secretMgr.retrievePassword(attemptToken2));
    // remove last attempt, restart, verify both tokens are now bad
    secretMgr.appFinished(attempt2.getApplicationId());
    secretMgr = new NMTokenSecretManagerInNM(stateStore);
    secretMgr.recover();
    secretMgr.setNodeId(nodeId);
    assertEquals(currentKey, secretMgr.getCurrentKey());
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1));
    assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt2));
    try {
        secretMgr.retrievePassword(attemptToken1);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    try {
        secretMgr.retrievePassword(attemptToken2);
        fail("attempt token should not still be valid");
    } catch (InvalidToken e) {
    // expected
    }
    stateStore.close();
}