Examples with Token org.apache.hadoop.yarn.api.records.Token used on opensource projects

Example 51 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testDemotionFromAcquired.

@Test(timeout = 60000)
public void testDemotionFromAcquired() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3));
    int guarContainersRequestedAny = amClient.getTable(0).get(priority3, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
    assertEquals(1, guarContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    Map<ContainerId, Container> allocatedGuarContainers = new HashMap<>();
    int iterationsLeft = 50;
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    updateMetrics("Before Guar Allocation");
    while (allocatedContainerCount < guarContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            if (container.getExecutionType() == ExecutionType.GUARANTEED) {
                allocatedGuarContainers.put(container.getId(), container);
                removeCR(container);
            }
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < guarContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(guarContainersRequestedAny, allocatedContainerCount);
    assertEquals(guarContainersRequestedAny, allocatedGuarContainers.size());
    updateMetrics("After Guar Allocation / Before Demotion");
    try {
        Container c = allocatedGuarContainers.values().iterator().next();
        amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE, null, ExecutionType.GUARANTEED));
        Assert.fail("Should throw Exception..");
    } catch (IllegalArgumentException e) {
        System.out.println("## " + e.getMessage());
        Assert.assertTrue(e.getMessage().contains("target should be OPPORTUNISTIC and original should be GUARANTEED"));
    }
    Container c = allocatedGuarContainers.values().iterator().next();
    amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE, null, ExecutionType.OPPORTUNISTIC));
    iterationsLeft = 120;
    Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
    // do a few iterations to ensure RM is not going to send new containers
    while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
        // inform RM of rejection
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        // RM did not send new containers because AM does not need any
        if (allocResponse.getUpdatedContainers() != null) {
            for (UpdatedContainer updatedContainer : allocResponse.getUpdatedContainers()) {
                System.out.println("Got update..");
                updatedContainers.put(updatedContainer.getContainer().getId(), updatedContainer);
            }
        }
        if (iterationsLeft > 0) {
            // sleep to make sure NM's heartbeat
            sleep(100);
        }
    }
    updateMetrics("After Demotion");
    assertEquals(1, updatedContainers.size());
    for (ContainerId cId : allocatedGuarContainers.keySet()) {
        Container orig = allocatedGuarContainers.get(cId);
        UpdatedContainer updatedContainer = updatedContainers.get(cId);
        assertNotNull(updatedContainer);
        assertEquals(ExecutionType.OPPORTUNISTIC, updatedContainer.getContainer().getExecutionType());
        assertEquals(orig.getResource(), updatedContainer.getContainer().getResource());
        assertEquals(orig.getNodeId(), updatedContainer.getContainer().getNodeId());
        assertEquals(orig.getVersion() + 1, updatedContainer.getContainer().getVersion());
    }
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.ask.clear();
}

Example 52 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestContainerLaunchRPC method testRPCTimeout.

private void testRPCTimeout(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, server.getListenerAddress(), conf);
        ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
        StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
        list.add(scRequest);
        StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
        try {
            proxy.startContainers(allRequests);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            Assert.assertEquals("Error, exception is not: " + SocketTimeoutException.class.getName(), SocketTimeoutException.class.getName(), e.getClass().getName());
            return;
        }
    } finally {
        server.stop();
    }
    Assert.fail("timeout exception should have occurred!");
}

Example 53 with Token

use of org.apache.hadoop.yarn.api.records.Token in project hadoop by apache.

the class TestContainerResourceIncreaseRPC method testRPCTimeout.

private void testRPCTimeout(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, server.getListenerAddress(), conf);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
        // Construct container resource increase request,
        List<Token> increaseTokens = new ArrayList<>();
        increaseTokens.add(containerToken);
        IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
        try {
            proxy.increaseContainersResource(increaseRequest);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            Assert.assertEquals("Error, exception is not: " + SocketTimeoutException.class.getName(), SocketTimeoutException.class.getName(), e.getClass().getName());
            return;
        }
    } finally {
        server.stop();
    }
    Assert.fail("timeout exception should have occurred!");
}