Examples with YarnConfiguration org.apache.hadoop.yarn.conf.YarnConfiguration used on opensource projects

Example 51 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestApplicationMasterService method testInvalidIncreaseDecreaseRequest.

@Test(timeout = 60000)
public void testInvalidIncreaseDecreaseRequest() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(1024);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        RegisterApplicationMasterResponse registerResponse = am1.registerAppAttempt();
        sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
        // Ask for a normal increase should be successfull
        am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048), null)));
        // Target resource is negative, should fail
        AllocateResponse response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(-1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Target resource is more than maxAllocation, should fail
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.add(registerResponse.getMaximumResourceCapability(), Resources.createResource(1)), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Contains multiple increase/decrease requests for same contaienrId 
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048, 4), null), UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.DECREASE_RESOURCE, Resources.createResource(1024, 1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("UPDATE_OUTSTANDING_ERROR", response.getUpdateErrors().get(0).getReason());
    } finally {
        if (rm != null) {
            rm.close();
        }
    }
}

Example 52 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestApplicationMasterService method setup.

@BeforeClass
public static void setup() {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
}

Example 53 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestClientRMService method testRMStartWithDecommissionedNode.

@Test
public void testRMStartWithDecommissionedNode() throws Exception {
    String excludeFile = "excludeFile";
    createExcludeFile(excludeFile);
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeFile);
    MockRM rm = new MockRM(conf) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Make call
    GetClusterNodesRequest request = GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
    List<NodeReport> nodeReports = client.getClusterNodes(request).getNodeReports();
    Assert.assertEquals(1, nodeReports.size());
    rm.stop();
    rpc.stopProxy(client, conf);
    new File(excludeFile).delete();
}

Example 54 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestClientRMService method testForceKillApplication.

@Test
public void testForceKillApplication() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(MockRM.ENABLE_WEBAPP, true);
    MockRM rm = new MockRM(conf);
    rm.init(conf);
    rm.start();
    ClientRMService rmService = rm.getClientRMService();
    GetApplicationsRequest getRequest = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.KILLED));
    RMApp app1 = rm.submitApp(1024);
    RMApp app2 = rm.submitApp(1024, true);
    assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size());
    KillApplicationRequest killRequest1 = KillApplicationRequest.newInstance(app1.getApplicationId());
    String diagnostic = "message1";
    killRequest1.setDiagnostics(diagnostic);
    KillApplicationRequest killRequest2 = KillApplicationRequest.newInstance(app2.getApplicationId());
    int killAttemptCount = 0;
    for (int i = 0; i < 100; i++) {
        KillApplicationResponse killResponse1 = rmService.forceKillApplication(killRequest1);
        killAttemptCount++;
        if (killResponse1.getIsKillCompleted()) {
            break;
        }
        Thread.sleep(10);
    }
    assertTrue("Kill attempt count should be greater than 1 for managed AMs", killAttemptCount > 1);
    assertEquals("Incorrect number of apps in the RM", 1, rmService.getApplications(getRequest).getApplicationList().size());
    assertTrue("Diagnostic message is incorrect", app1.getDiagnostics().toString().contains(diagnostic));
    KillApplicationResponse killResponse2 = rmService.forceKillApplication(killRequest2);
    assertTrue("Killing UnmanagedAM should falsely acknowledge true", killResponse2.getIsKillCompleted());
    for (int i = 0; i < 100; i++) {
        if (2 == rmService.getApplications(getRequest).getApplicationList().size()) {
            break;
        }
        Thread.sleep(10);
    }
    assertEquals("Incorrect number of apps in the RM", 2, rmService.getApplications(getRequest).getApplicationList().size());
}

Example 55 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestAppManager method testStateStoreAppLimitLargerThanMemoryAppLimit.

@Test
public void testStateStoreAppLimitLargerThanMemoryAppLimit() {
    long now = System.currentTimeMillis();
    RMContext rmContext = mockRMContext(10, now - 20000);
    Configuration conf = new YarnConfiguration();
    int maxAppsInMemory = 8;
    conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, maxAppsInMemory);
    // larger than maxCompletedAppsInMemory, reset to RM_MAX_COMPLETED_APPLICATIONS.
    conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 1000);
    TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);
    addToCompletedApps(appMonitor, rmContext);
    Assert.assertEquals("Number of completed apps incorrect", 10, appMonitor.getCompletedAppsListSize());
    appMonitor.checkAppNumCompletedLimit();
    int numRemoveApps = 10 - maxAppsInMemory;
    Assert.assertEquals("Number of apps incorrect after # completed check", maxAppsInMemory, rmContext.getRMApps().size());
    Assert.assertEquals("Number of completed apps incorrect after check", maxAppsInMemory, appMonitor.getCompletedAppsListSize());
    verify(rmContext.getStateStore(), times(numRemoveApps)).removeApplication(isA(RMApp.class));
    Assert.assertEquals(maxAppsInMemory, appMonitor.getCompletedAppsInStateStore());
}