Examples with ApplicationAttemptId org.apache.hadoop.yarn.api.records.ApplicationAttemptId used on opensource projects

Example 26 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestFSAppAttempt method testDelayScheduling.

@Test
public void testDelayScheduling() {
    FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    double nodeLocalityThreshold = .5;
    double rackLocalityThreshold = .6;
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    RMContext rmContext = resourceManager.getRMContext();
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    // Default level should be node-local
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // First five scheduling opportunities should remain node local
    for (int i = 0; i < 5; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    // After five it should switch to rack local
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // Manually set back to node local
    schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
    schedulerApp.resetSchedulingOpportunities(prio);
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // Now escalate again to rack-local, then to off-switch
    for (int i = 0; i < 5; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    for (int i = 0; i < 6; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}

Example 27 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestFSAppAttempt method testLocalityLevelWithoutDelays.

@Test
public /**
   * Ensure that when negative paramaters are given (signaling delay scheduling
   * no tin use), the least restrictive locality level is returned.
   */
void testLocalityLevelWithoutDelays() {
    FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, -1.0, -1.0));
}

Example 28 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestFSAppAttempt method testDelaySchedulingForContinuousScheduling.

@Test
public void testDelaySchedulingForContinuousScheduling() throws InterruptedException {
    FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue("queue", true);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    ControlledClock clock = new ControlledClock();
    scheduler.setClock(clock);
    // 5 seconds
    long nodeLocalityDelayMs = 5 * 1000L;
    // 6 seconds
    long rackLocalityDelayMs = 6 * 1000L;
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    // Default level should be node-local
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // after 4 seconds should remain node local
    clock.tickSec(4);
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // after 6 seconds should switch to rack local
    clock.tickSec(2);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // manually set back to node local
    schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
    schedulerApp.resetSchedulingOpportunities(prio, clock.getTime());
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // Now escalate again to rack-local, then to off-switch
    clock.tickSec(6);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    clock.tickSec(7);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
}

Example 29 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestContainerManagerSecurity method testContainerTokenWithEpoch.

/**
   * This tests whether a containerId is serialized/deserialized with epoch.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws YarnException
   */
private void testContainerTokenWithEpoch(Configuration conf) throws IOException, InterruptedException, YarnException {
    LOG.info("Running test for serializing/deserializing containerIds");
    NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext().getNMTokenSecretManager();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
    NodeManager nm = yarnCluster.getNodeManager(0);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager();
    String user = "test";
    waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
    NodeId nodeId = nm.getNMContext().getNodeId();
    // Both id should be equal.
    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
    // Creating a normal Container Token
    RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getContainerTokenSecretManager();
    Resource r = Resource.newInstance(1230, 2);
    Token containerToken = containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier();
    byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
    containerTokenIdentifier.readFields(dib);
    Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
    Assert.assertEquals(cId.toString(), containerTokenIdentifier.getContainerID().toString());
    Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
    YarnRPC rpc = YarnRPC.create(conf);
    testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
    List<ContainerId> containerIds = new LinkedList<ContainerId>();
    containerIds.add(cId);
    ContainerManagementProtocol proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    GetContainerStatusesResponse res = proxy.getContainerStatuses(GetContainerStatusesRequest.newInstance(containerIds));
    Assert.assertNotNull(res.getContainerStatuses().get(0));
    Assert.assertEquals(cId, res.getContainerStatuses().get(0).getContainerId());
    Assert.assertEquals(cId.toString(), res.getContainerStatuses().get(0).getContainerId().toString());
}

Example 30 with ApplicationAttemptId

use of org.apache.hadoop.yarn.api.records.ApplicationAttemptId in project hadoop by apache.

the class TestFSRMStateStore method testFSRMStateStore.

@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
    HdfsConfiguration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    try {
        fsTester = new TestFSRMStateStoreTester(cluster, false);
        // If the state store is FileSystemRMStateStore then add corrupted entry.
        // It should discard the entry and remove it from file system.
        FSDataOutputStream fsOut = null;
        FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore) fsTester.getRMStateStore();
        String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
        ApplicationAttemptId attemptId3 = ApplicationAttemptId.fromString(appAttemptIdStr3);
        Path appDir = fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
        Path tempAppAttemptFile = new Path(appDir, attemptId3.toString() + ".tmp");
        fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
        fsOut.write("Some random data ".getBytes());
        fsOut.close();
        testRMAppStateStore(fsTester);
        Assert.assertFalse(fsTester.workingDirPathURI.getFileSystem(conf).exists(tempAppAttemptFile));
        testRMDTSecretManagerStateStore(fsTester);
        testCheckVersion(fsTester);
        testEpoch(fsTester);
        testAppDeletion(fsTester);
        testDeleteStore(fsTester);
        testRemoveApplication(fsTester);
        testRemoveAttempt(fsTester);
        testAMRMTokenSecretManagerStateStore(fsTester);
        testReservationStateStore(fsTester);
    } finally {
        cluster.shutdown();
    }
}