Examples with ApplicationAccessType org.apache.hadoop.yarn.api.records.ApplicationAccessType used on opensource projects

Example 41 with ApplicationAccessType

use of org.apache.hadoop.yarn.api.records.ApplicationAccessType in project hadoop by apache.

the class TestAbstractYarnScheduler method testResourceRequestRestoreWhenRMContainerIsAtAllocated.

@Test(timeout = 60000)
public void testResourceRequestRestoreWhenRMContainerIsAtAllocated() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm1 = new MockRM(conf);
    try {
        rm1.start();
        RMApp app1 = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null, "Test", false, true);
        MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService());
        nm1.registerNode();
        MockNM nm2 = new MockNM("127.0.0.1:2351", 10240, rm1.getResourceTrackerService());
        nm2.registerNode();
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
        int NUM_CONTAINERS = 1;
        // allocate NUM_CONTAINERS containers
        am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<ContainerId>());
        nm1.nodeHeartbeat(true);
        // wait for containers to be allocated.
        List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.size() != NUM_CONTAINERS) {
            nm1.nodeHeartbeat(true);
            containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
            Thread.sleep(200);
        }
        // launch the 2nd container, for testing running container transferred.
        nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
        ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
        rm1.waitForState(nm1, containerId2, RMContainerState.RUNNING);
        // 3rd container is in Allocated state.
        am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<ContainerId>());
        nm2.nodeHeartbeat(true);
        ContainerId containerId3 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
        rm1.waitForState(nm2, containerId3, RMContainerState.ALLOCATED);
        // NodeManager restart
        nm2.registerNode();
        // NM restart kills all allocated and running containers.
        rm1.waitForState(nm2, containerId3, RMContainerState.KILLED);
        // The killed RMContainer request should be restored. In successive
        // nodeHeartBeats AM should be able to get container allocated.
        containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.size() != NUM_CONTAINERS) {
            nm2.nodeHeartbeat(true);
            containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
            Thread.sleep(200);
        }
        nm2.nodeHeartbeat(am1.getApplicationAttemptId(), 4, ContainerState.RUNNING);
        ContainerId containerId4 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 4);
        rm1.waitForState(nm2, containerId4, RMContainerState.RUNNING);
    } finally {
        rm1.stop();
    }
}

Example 42 with ApplicationAccessType

use of org.apache.hadoop.yarn.api.records.ApplicationAccessType in project hadoop by apache.

the class TestAbstractYarnScheduler method testResourceRequestRecoveryToTheRightAppAttempt.

/**
   * Test to verify that ResourceRequests recovery back to the right app-attempt
   * after a container gets killed at ACQUIRED state: YARN-4502.
   *
   * @throws Exception
   */
@Test
public void testResourceRequestRecoveryToTheRightAppAttempt() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        RMApp rmApp = rm.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null, "Test", false, true);
        MockNM node = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
        node.registerNode();
        MockAM am1 = MockRM.launchAndRegisterAM(rmApp, rm, node);
        ApplicationAttemptId applicationAttemptOneID = am1.getApplicationAttemptId();
        ContainerId am1ContainerID = ContainerId.newContainerId(applicationAttemptOneID, 1);
        // allocate NUM_CONTAINERS containers
        am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
        node.nodeHeartbeat(true);
        // wait for containers to be allocated.
        List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.size() != 1) {
            node.nodeHeartbeat(true);
            containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
            Thread.sleep(200);
        }
        // launch a 2nd container, for testing running-containers transfer.
        node.nodeHeartbeat(applicationAttemptOneID, 2, ContainerState.RUNNING);
        ContainerId runningContainerID = ContainerId.newContainerId(applicationAttemptOneID, 2);
        rm.waitForState(node, runningContainerID, RMContainerState.RUNNING);
        // 3rd container is in Allocated state.
        int ALLOCATED_CONTAINER_PRIORITY = 1047;
        am1.allocate("127.0.0.1", 1024, 1, ALLOCATED_CONTAINER_PRIORITY, new ArrayList<ContainerId>(), null);
        node.nodeHeartbeat(true);
        ContainerId allocatedContainerID = ContainerId.newContainerId(applicationAttemptOneID, 3);
        rm.waitForState(node, allocatedContainerID, RMContainerState.ALLOCATED);
        RMContainer allocatedContainer = rm.getResourceScheduler().getRMContainer(allocatedContainerID);
        // Capture scheduler app-attempt before AM crash.
        SchedulerApplicationAttempt firstSchedulerAppAttempt = ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm.getResourceScheduler()).getApplicationAttempt(applicationAttemptOneID);
        // AM crashes, and a new app-attempt gets created
        node.nodeHeartbeat(applicationAttemptOneID, 1, ContainerState.COMPLETE);
        rm.drainEvents();
        RMAppAttempt rmAppAttempt2 = MockRM.waitForAttemptScheduled(rmApp, rm);
        ApplicationAttemptId applicationAttemptTwoID = rmAppAttempt2.getAppAttemptId();
        Assert.assertEquals(2, applicationAttemptTwoID.getAttemptId());
        // All outstanding allocated containers will be killed (irrespective of
        // keep-alive of container across app-attempts)
        Assert.assertEquals(RMContainerState.KILLED, allocatedContainer.getState());
        // original app-attempt, not the new one
        for (SchedulerRequestKey key : firstSchedulerAppAttempt.getSchedulerKeys()) {
            if (key.getPriority().getPriority() == 0) {
                Assert.assertEquals(0, firstSchedulerAppAttempt.getOutstandingAsksCount(key));
            } else if (key.getPriority().getPriority() == ALLOCATED_CONTAINER_PRIORITY) {
                Assert.assertEquals(1, firstSchedulerAppAttempt.getOutstandingAsksCount(key));
            }
        }
        // Also, only one running container should be transferred after AM
        // launches
        MockRM.launchAM(rmApp, rm, node);
        List<Container> transferredContainers = rm.getResourceScheduler().getTransferredContainers(applicationAttemptTwoID);
        Assert.assertEquals(1, transferredContainers.size());
        Assert.assertEquals(runningContainerID, transferredContainers.get(0).getId());
    } finally {
        rm.stop();
    }
}

Example 43 with ApplicationAccessType

use of org.apache.hadoop.yarn.api.records.ApplicationAccessType in project hadoop by apache.

the class TestAMAuthorization method testAuthorizedAccess.

@Test
public void testAuthorizedAccess() throws Exception {
    MyContainerManager containerManager = new MyContainerManager();
    rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    int waitCount = 0;
    while (containerManager.containerTokens == null && waitCount++ < 20) {
        LOG.info("Waiting for AM Launch to happen..");
        Thread.sleep(1000);
    }
    Assert.assertNotNull(containerManager.containerTokens);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    waitForLaunchedState(attempt);
    // Create a client to the RM.
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm.getApplicationMasterService().getBindAddress(), conf);
        }
    });
    RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
    RegisterApplicationMasterResponse response = client.registerApplicationMaster(request);
    Assert.assertNotNull(response.getClientToAMTokenMasterKey());
    if (UserGroupInformation.isSecurityEnabled()) {
        Assert.assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
    }
    Assert.assertEquals("Register response has bad ACLs", "*", response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
}

Example 44 with ApplicationAccessType

use of org.apache.hadoop.yarn.api.records.ApplicationAccessType in project hadoop by apache.

the class TestContainerResourceUsage method amRestartTests.

private void amRestartTests(boolean keepRunningContainers) throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    RMApp app = rm.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
    MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
    nm.registerNode();
    MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
    int NUM_CONTAINERS = 1;
    // allocate NUM_CONTAINERS containers
    am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<ContainerId>());
    nm.nodeHeartbeat(true);
    // wait for containers to be allocated.
    List<Container> containers = am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    while (containers.size() != NUM_CONTAINERS) {
        nm.nodeHeartbeat(true);
        containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        Thread.sleep(200);
    }
    // launch the 2nd container.
    ContainerId containerId2 = ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
    nm.nodeHeartbeat(am0.getApplicationAttemptId(), containerId2.getContainerId(), ContainerState.RUNNING);
    rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
    // Capture the containers here so the metrics can be calculated after the
    // app has completed.
    Collection<RMContainer> rmContainers = rm.scheduler.getSchedulerAppInfo(am0.getApplicationAttemptId()).getLiveContainers();
    // fail the first app attempt by sending CONTAINER_FINISHED event without
    // registering.
    ContainerId amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId();
    nm.nodeHeartbeat(am0.getApplicationAttemptId(), amContainerId.getContainerId(), ContainerState.COMPLETE);
    rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm.drainEvents();
    long memorySeconds = 0;
    long vcoreSeconds = 0;
    // Calculate container usage metrics for first attempt.
    if (keepRunningContainers) {
        // Only calculate the usage for the one container that has completed.
        for (RMContainer c : rmContainers) {
            if (c.getContainerId().equals(amContainerId)) {
                AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
                memorySeconds += ru.getMemorySeconds();
                vcoreSeconds += ru.getVcoreSeconds();
            } else {
                // The remaining container should be RUNNING.
                Assert.assertTrue("After first attempt failed, remaining container " + "should still be running. ", c.getContainerState().equals(ContainerState.RUNNING));
            }
        }
    } else {
        // be completed. Calculate the resource usage metrics for all of them.
        for (RMContainer c : rmContainers) {
            waitforContainerCompletion(rm, nm, amContainerId, c);
            AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
            memorySeconds += ru.getMemorySeconds();
            vcoreSeconds += ru.getVcoreSeconds();
        }
    }
    // wait for app to start a new attempt.
    rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
    // assert this is a new AM.
    RMAppAttempt attempt2 = app.getCurrentAppAttempt();
    Assert.assertFalse(attempt2.getAppAttemptId().equals(am0.getApplicationAttemptId()));
    rm.waitForState(attempt2.getAppAttemptId(), RMAppAttemptState.SCHEDULED);
    nm.nodeHeartbeat(true);
    MockAM am1 = rm.sendAMLaunched(attempt2.getAppAttemptId());
    am1.registerAppAttempt();
    rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
    // allocate NUM_CONTAINERS containers
    am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<ContainerId>());
    nm.nodeHeartbeat(true);
    // wait for containers to be allocated.
    containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    while (containers.size() != NUM_CONTAINERS) {
        nm.nodeHeartbeat(true);
        containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        Thread.sleep(200);
    }
    rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
    // Capture running containers for later use by metrics calculations.
    rmContainers = rm.scheduler.getSchedulerAppInfo(attempt2.getAppAttemptId()).getLiveContainers();
    // complete container by sending the container complete event which has
    // earlier attempt's attemptId
    amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId();
    nm.nodeHeartbeat(am0.getApplicationAttemptId(), amContainerId.getContainerId(), ContainerState.COMPLETE);
    MockRM.finishAMAndVerifyAppState(app, rm, nm, am1);
    // Calculate container usage metrics for second attempt.
    for (RMContainer c : rmContainers) {
        waitforContainerCompletion(rm, nm, amContainerId, c);
        AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
        memorySeconds += ru.getMemorySeconds();
        vcoreSeconds += ru.getVcoreSeconds();
    }
    RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
    Assert.assertEquals("Unexpected MemorySeconds value", memorySeconds, rmAppMetrics.getMemorySeconds());
    Assert.assertEquals("Unexpected VcoreSeconds value", vcoreSeconds, rmAppMetrics.getVcoreSeconds());
    rm.stop();
    return;
}

Example 45 with ApplicationAccessType

use of org.apache.hadoop.yarn.api.records.ApplicationAccessType in project hadoop by apache.

the class TestApplicationACLsManager method testCheckAccessWithPartialACLS.

@Test
public void testCheckAccessWithPartialACLS() {
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, ADMIN_USER);
    ApplicationACLsManager aclManager = new ApplicationACLsManager(conf);
    UserGroupInformation appOwner = UserGroupInformation.createRemoteUser(APP_OWNER);
    // Add only the VIEW ACLS
    Map<ApplicationAccessType, String> aclMap = new HashMap<ApplicationAccessType, String>();
    aclMap.put(ApplicationAccessType.VIEW_APP, TESTUSER1);
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    aclManager.addApplication(appId, aclMap);
    //Application Owner should have all access even if Application ACL is not added
    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP, APP_OWNER, appId));
    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP, APP_OWNER, appId));
    //Admin should have all access
    UserGroupInformation adminUser = UserGroupInformation.createRemoteUser(ADMIN_USER);
    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP, APP_OWNER, appId));
    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP, APP_OWNER, appId));
    // testuser1 should  have view access only
    UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser(TESTUSER1);
    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP, APP_OWNER, appId));
    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP, APP_OWNER, appId));
    // A testuser2 should Not have access
    UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser(TESTUSER2);
    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.VIEW_APP, APP_OWNER, appId));
    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.MODIFY_APP, APP_OWNER, appId));
}