Examples with ContainerId org.apache.hadoop.yarn.api.records.ContainerId used on opensource projects

Example 46 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class NMTimelinePublisher method publishContainerCreatedEvent.

@SuppressWarnings("unchecked")
private void publishContainerCreatedEvent(ContainerEvent event) {
    ContainerId containerId = event.getContainerID();
    ContainerEntity entity = createContainerEntity(containerId);
    Container container = context.getContainers().get(containerId);
    Resource resource = container.getResource();
    Map<String, Object> entityInfo = new HashMap<String, Object>();
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, resource.getMemorySize());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, resource.getVirtualCores());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, nodeId.getHost());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, nodeId.getPort());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, container.getPriority().toString());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, httpAddress);
    entity.setInfo(entityInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    entity.addEvent(tEvent);
    entity.setCreatedTime(event.getTimestamp());
    dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId()));
}

Example 47 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class AMLauncher method connect.

private void connect() throws IOException {
    ContainerId masterContainerID = masterContainer.getId();
    containerMgrProxy = getContainerMgrProxy(masterContainerID);
}

Example 48 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class AMLauncher method launch.

private void launch() throws IOException, YarnException {
    connect();
    ContainerId masterContainerID = masterContainer.getId();
    ApplicationSubmissionContext applicationContext = application.getSubmissionContext();
    LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId());
    ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerMgrProxy.startContainers(allRequests);
    if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) {
        Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize();
        parseAndThrowException(t);
    } else {
        LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId());
    }
}

Example 49 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class AMLauncher method cleanup.

private void cleanup() throws IOException, YarnException {
    connect();
    ContainerId containerId = masterContainer.getId();
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse response = containerMgrProxy.stopContainers(stopRequest);
    if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(containerId)) {
        Throwable t = response.getFailedRequests().get(containerId).deSerialize();
        parseAndThrowException(t);
    }
}

Example 50 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class ReservedContainerCandidatesSelector method getPreemptionCandidatesOnNode.

/**
   * Try to check if we can preempt resources for reserved container in given node
   * @param node
   * @param cluster
   * @param queueToPreemptableResourceByPartition it's a map of
   *                 <queueName, <partition, preemptable-resource>>
   * @param readOnly do we want to modify preemptable resource after we selected
   *                 candidates
   * @return NodeForPreemption if it's possible to preempt containers on the node
   * to satisfy reserved resource
   */
private NodeForPreemption getPreemptionCandidatesOnNode(FiCaSchedulerNode node, Resource cluster, Map<String, Map<String, Resource>> queueToPreemptableResourceByPartition, Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates, Resource totalPreemptionAllowed, boolean readOnly) {
    RMContainer reservedContainer = node.getReservedContainer();
    Resource available = Resources.clone(node.getUnallocatedResource());
    Resource totalSelected = Resources.createResource(0);
    List<RMContainer> sortedRunningContainers = node.getCopiedListOfRunningContainers();
    List<RMContainer> selectedContainers = new ArrayList<>();
    Map<ContainerId, RMContainer> killableContainers = node.getKillableContainers();
    // Sort running container by launch time, we preferred to preempt recent
    // launched preempt container
    Collections.sort(sortedRunningContainers, new Comparator<RMContainer>() {

        @Override
        public int compare(RMContainer o1, RMContainer o2) {
            return -1 * o1.getContainerId().compareTo(o2.getContainerId());
        }
    });
    // First check: can we preempt containers to allocate the
    // reservedContainer?
    boolean canAllocateReservedContainer = false;
    // At least, we can get available + killable resources from this node
    Resource cur = Resources.add(available, node.getTotalKillableResources());
    String partition = node.getPartition();
    // Avoid preempt any container if required <= available + killable
    if (Resources.fitsIn(rc, cluster, reservedContainer.getReservedResource(), cur)) {
        return null;
    }
    // Extra cost of am container preemption
    float amPreemptionCost = 0f;
    for (RMContainer c : sortedRunningContainers) {
        String containerQueueName = c.getQueueName();
        // Skip container if it is already marked killable
        if (killableContainers.containsKey(c.getContainerId())) {
            continue;
        }
        // selected. Here for safety, avoid preempt AM container in any cases
        if (c.isAMContainer()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip selecting AM container on host=" + node.getNodeID() + " AM container=" + c.getContainerId());
            }
            continue;
        }
        // Can we preempt container c?
        // Check if we have quota to preempt this container
        boolean canPreempt = tryToPreemptFromQueue(cluster, containerQueueName, partition, queueToPreemptableResourceByPartition, c.getAllocatedResource(), totalPreemptionAllowed, readOnly);
        // If we can, add to selected container, and change resource accordingly.
        if (canPreempt) {
            if (!CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, selectedCandidates)) {
                if (!readOnly) {
                    selectedContainers.add(c);
                }
                Resources.addTo(totalSelected, c.getAllocatedResource());
            }
            Resources.addTo(cur, c.getAllocatedResource());
            if (Resources.fitsIn(rc, cluster, reservedContainer.getReservedResource(), cur)) {
                canAllocateReservedContainer = true;
                break;
            }
        }
    }
    if (!canAllocateReservedContainer) {
        if (!readOnly) {
            // Revert queue preemption quotas
            for (RMContainer c : selectedContainers) {
                Resource res = getPreemptableResource(c.getQueueName(), partition, queueToPreemptableResourceByPartition);
                if (null == res) {
                    // Ignore such failures.
                    continue;
                }
                Resources.addTo(res, c.getAllocatedResource());
            }
        }
        return null;
    }
    float ratio = Resources.ratio(rc, totalSelected, reservedContainer.getReservedResource());
    // Compute preemption score
    NodeForPreemption nfp = new NodeForPreemption(ratio + amPreemptionCost, node, selectedContainers);
    return nfp;
}