Examples with NMToken org.apache.hadoop.yarn.api.records.NMToken used on opensource projects

Example 16 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class AMRMClientImpl method populateNMTokens.

@Private
@VisibleForTesting
protected void populateNMTokens(List<NMToken> nmTokens) {
    for (NMToken token : nmTokens) {
        String nodeId = token.getNodeId().toString();
        if (LOG.isDebugEnabled()) {
            if (getNMTokenCache().containsToken(nodeId)) {
                LOG.debug("Replacing token for : " + nodeId);
            } else {
                LOG.debug("Received new token for : " + nodeId);
            }
        }
        getNMTokenCache().setToken(nodeId, token.getToken());
    }
}

Example 17 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testOpportunisticAllocation.

/**
   * Tests allocation with requests comprising only opportunistic containers.
   */
@Test(timeout = 60000)
public void testOpportunisticAllocation() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
    int oppContainersRequestedAny = amClient.getTable(0).get(priority3, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
    assertEquals(2, oppContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 10;
    Set<ContainerId> releases = new TreeSet<>();
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    while (allocatedContainerCount < oppContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        for (Container container : allocResponse.getAllocatedContainers()) {
            allocatedContainerCount++;
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < oppContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(oppContainersRequestedAny, allocatedContainerCount);
    assertEquals(1, receivedNMTokens.values().size());
}

Example 18 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class TestOpportunisticContainerAllocation method testDemotionFromAcquired.

@Test(timeout = 60000)
public void testDemotionFromAcquired() throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority3));
    int guarContainersRequestedAny = amClient.getTable(0).get(priority3, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
    assertEquals(1, guarContainersRequestedAny);
    assertEquals(1, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    Map<ContainerId, Container> allocatedGuarContainers = new HashMap<>();
    int iterationsLeft = 50;
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<>();
    updateMetrics("Before Guar Allocation");
    while (allocatedContainerCount < guarContainersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            if (container.getExecutionType() == ExecutionType.GUARANTEED) {
                allocatedGuarContainers.put(container.getId(), container);
                removeCR(container);
            }
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < guarContainersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(100);
        }
    }
    assertEquals(guarContainersRequestedAny, allocatedContainerCount);
    assertEquals(guarContainersRequestedAny, allocatedGuarContainers.size());
    updateMetrics("After Guar Allocation / Before Demotion");
    try {
        Container c = allocatedGuarContainers.values().iterator().next();
        amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE, null, ExecutionType.GUARANTEED));
        Assert.fail("Should throw Exception..");
    } catch (IllegalArgumentException e) {
        System.out.println("## " + e.getMessage());
        Assert.assertTrue(e.getMessage().contains("target should be OPPORTUNISTIC and original should be GUARANTEED"));
    }
    Container c = allocatedGuarContainers.values().iterator().next();
    amClient.requestContainerUpdate(c, UpdateContainerRequest.newInstance(c.getVersion(), c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE, null, ExecutionType.OPPORTUNISTIC));
    iterationsLeft = 120;
    Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
    // do a few iterations to ensure RM is not going to send new containers
    while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
        // inform RM of rejection
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        // RM did not send new containers because AM does not need any
        if (allocResponse.getUpdatedContainers() != null) {
            for (UpdatedContainer updatedContainer : allocResponse.getUpdatedContainers()) {
                System.out.println("Got update..");
                updatedContainers.put(updatedContainer.getContainer().getId(), updatedContainer);
            }
        }
        if (iterationsLeft > 0) {
            // sleep to make sure NM's heartbeat
            sleep(100);
        }
    }
    updateMetrics("After Demotion");
    assertEquals(1, updatedContainers.size());
    for (ContainerId cId : allocatedGuarContainers.keySet()) {
        Container orig = allocatedGuarContainers.get(cId);
        UpdatedContainer updatedContainer = updatedContainers.get(cId);
        assertNotNull(updatedContainer);
        assertEquals(ExecutionType.OPPORTUNISTIC, updatedContainer.getContainer().getExecutionType());
        assertEquals(orig.getResource(), updatedContainer.getContainer().getResource());
        assertEquals(orig.getNodeId(), updatedContainer.getContainer().getNodeId());
        assertEquals(orig.getVersion() + 1, updatedContainer.getContainer().getVersion());
    }
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.ask.clear();
}

Example 19 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class TestNMClient method allocateContainers.

private Set<Container> allocateContainers(AMRMClientImpl<ContainerRequest> rmClient, int num) throws YarnException, IOException {
    // setup container request
    Resource capability = Resource.newInstance(1024, 0);
    Priority priority = Priority.newInstance(0);
    String node = nodeReports.get(0).getNodeId().getHost();
    String rack = nodeReports.get(0).getRackName();
    String[] nodes = new String[] { node };
    String[] racks = new String[] { rack };
    for (int i = 0; i < num; ++i) {
        rmClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    }
    int containersRequestedAny = rmClient.getTable(0).get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 2;
    Set<Container> containers = new TreeSet<Container>();
    while (allocatedContainerCount < containersRequestedAny && iterationsLeft > 0) {
        AllocateResponse allocResponse = rmClient.allocate(0.1f);
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            containers.add(container);
        }
        if (!allocResponse.getNMTokens().isEmpty()) {
            for (NMToken token : allocResponse.getNMTokens()) {
                rmClient.getNMTokenCache().setToken(token.getNodeId().toString(), token.getToken());
            }
        }
        if (allocatedContainerCount < containersRequestedAny) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(1000);
        }
        --iterationsLeft;
    }
    return containers;
}

Example 20 with NMToken

use of org.apache.hadoop.yarn.api.records.NMToken in project hadoop by apache.

the class AllocateResponsePBImpl method initLocalNewNMTokenList.

private synchronized void initLocalNewNMTokenList() {
    if (nmTokens != null) {
        return;
    }
    AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
    List<NMTokenProto> list = p.getNmTokensList();
    nmTokens = new ArrayList<NMToken>();
    for (NMTokenProto t : list) {
        nmTokens.add(convertFromProtoFormat(t));
    }
}