Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 46 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project weave by continuuity.

the class ApplicationMasterService method createCapability.

private Resource createCapability(ResourceSpecification resourceSpec) {
    Resource capability = Records.newRecord(Resource.class);
    if (!YarnUtils.setVirtualCores(capability, resourceSpec.getVirtualCores())) {
        LOG.debug("Virtual cores limit not supported.");
    }
    capability.setMemory(resourceSpec.getMemorySize());
    return capability;
}

Example 47 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project weave by continuuity.

the class AMRMClientImpl method decResourceRequest.

private void decResourceRequest(Priority priority, String resourceName, Resource capability, int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);
    if (remoteRequests == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as priority " + priority + " is not present in request table");
        }
        return;
    }
    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as " + resourceName + " is not present in request table");
        }
        return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (LOG.isDebugEnabled()) {
        LOG.debug("BEFORE decResourceRequest:" + " applicationId=" + appAttemptId + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() - containerCount);
    if (remoteRequest.getNumContainers() < 0) {
        // guard against spurious removals
        remoteRequest.setNumContainers(0);
    }
    // Send the ResourceRequest to RM even if is 0 because it needs to override
    // a previously sent value. If ResourceRequest was not sent previously then
    // sending 0 ought to be a no-op on RM.
    addResourceRequestToAsk(remoteRequest);
    // Delete entries from map if no longer needed.
    if (remoteRequest.getNumContainers() == 0) {
        reqMap.remove(capability);
        if (reqMap.size() == 0) {
            remoteRequests.remove(resourceName);
        }
        if (remoteRequests.size() == 0) {
            remoteRequestsTable.remove(priority);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.info("AFTER decResourceRequest:" + " applicationId=" + appAttemptId + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

Example 48 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class DistributedScheduler method updateParameters.

private void updateParameters(RegisterDistributedSchedulingAMResponse registerResponse) {
    Resource incrementResource = registerResponse.getIncrContainerResource();
    if (incrementResource == null) {
        incrementResource = registerResponse.getMinContainerResource();
    }
    oppContainerContext.updateAllocationParams(registerResponse.getMinContainerResource(), registerResponse.getMaxContainerResource(), incrementResource, registerResponse.getContainerTokenExpiryInterval());
    oppContainerContext.getContainerIdGenerator().resetContainerIdCounter(registerResponse.getContainerIdStart());
    setNodeList(registerResponse.getNodesForScheduling());
}

Example 49 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ContainersMonitorImpl method updateContainerMetrics.

private void updateContainerMetrics(ContainersMonitorEvent monitoringEvent) {
    if (!containerMetricsEnabled || monitoringEvent == null) {
        return;
    }
    ContainerId containerId = monitoringEvent.getContainerId();
    ContainerMetrics usageMetrics;
    int vmemLimitMBs;
    int pmemLimitMBs;
    int cpuVcores;
    switch(monitoringEvent.getType()) {
        case START_MONITORING_CONTAINER:
            usageMetrics = ContainerMetrics.forContainer(containerId, containerMetricsPeriodMs, containerMetricsUnregisterDelayMs);
            ContainerStartMonitoringEvent startEvent = (ContainerStartMonitoringEvent) monitoringEvent;
            usageMetrics.recordStateChangeDurations(startEvent.getLaunchDuration(), startEvent.getLocalizationDuration());
            cpuVcores = startEvent.getCpuVcores();
            vmemLimitMBs = (int) (startEvent.getVmemLimit() >> 20);
            pmemLimitMBs = (int) (startEvent.getPmemLimit() >> 20);
            usageMetrics.recordResourceLimit(vmemLimitMBs, pmemLimitMBs, cpuVcores);
            break;
        case STOP_MONITORING_CONTAINER:
            usageMetrics = ContainerMetrics.getContainerMetrics(containerId);
            if (usageMetrics != null) {
                usageMetrics.finished();
            }
            break;
        case CHANGE_MONITORING_CONTAINER_RESOURCE:
            usageMetrics = ContainerMetrics.forContainer(containerId, containerMetricsPeriodMs, containerMetricsUnregisterDelayMs);
            ChangeMonitoringContainerResourceEvent changeEvent = (ChangeMonitoringContainerResourceEvent) monitoringEvent;
            Resource resource = changeEvent.getResource();
            pmemLimitMBs = (int) resource.getMemorySize();
            vmemLimitMBs = (int) (pmemLimitMBs * vmemRatio);
            cpuVcores = resource.getVirtualCores();
            usageMetrics.recordResourceLimit(vmemLimitMBs, pmemLimitMBs, cpuVcores);
            break;
        default:
            break;
    }
}

Example 50 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class NMTimelinePublisher method publishContainerCreatedEvent.

@SuppressWarnings("unchecked")
private void publishContainerCreatedEvent(ContainerEvent event) {
    ContainerId containerId = event.getContainerID();
    ContainerEntity entity = createContainerEntity(containerId);
    Container container = context.getContainers().get(containerId);
    Resource resource = container.getResource();
    Map<String, Object> entityInfo = new HashMap<String, Object>();
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, resource.getMemorySize());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, resource.getVirtualCores());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, nodeId.getHost());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, nodeId.getPort());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, container.getPriority().toString());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, httpAddress);
    entity.setInfo(entityInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    entity.addEvent(tEvent);
    entity.setCreatedTime(event.getTimestamp());
    dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId()));
}