Examples with ContainerSimulator org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator used on opensource projects

Example 6 with ContainerSimulator

use of org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator in project hadoop by apache.

the class NMSimulator method addNewContainer.

/**
   * launch a new container with the given life time
   */
public void addNewContainer(Container container, long lifeTimeMS) {
    LOG.debug(MessageFormat.format("NodeManager {0} launches a new " + "container ({1}).", node.getNodeID(), container.getId()));
    if (lifeTimeMS != -1) {
        // normal container
        ContainerSimulator cs = new ContainerSimulator(container.getId(), container.getResource(), lifeTimeMS + System.currentTimeMillis(), lifeTimeMS);
        containerQueue.add(cs);
        runningContainers.put(cs.getId(), cs);
    } else {
        // -1 means AMContainer
        synchronized (amContainerList) {
            amContainerList.add(container.getId());
        }
    }
}

Example 7 with ContainerSimulator

use of org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator in project hadoop by apache.

the class NMSimulator method generateContainerStatusList.

/**
   * catch status of all containers located on current node
   */
private ArrayList<ContainerStatus> generateContainerStatusList() {
    ArrayList<ContainerStatus> csList = new ArrayList<ContainerStatus>();
    // add running containers
    for (ContainerSimulator container : runningContainers.values()) {
        csList.add(newContainerStatus(container.getId(), ContainerState.RUNNING, ContainerExitStatus.SUCCESS));
    }
    synchronized (amContainerList) {
        for (ContainerId cId : amContainerList) {
            csList.add(newContainerStatus(cId, ContainerState.RUNNING, ContainerExitStatus.SUCCESS));
        }
    }
    // add complete containers
    synchronized (completedContainerList) {
        for (ContainerId cId : completedContainerList) {
            LOG.debug(MessageFormat.format("NodeManager {0} completed" + " container ({1}).", node.getNodeID(), cId));
            csList.add(newContainerStatus(cId, ContainerState.COMPLETE, ContainerExitStatus.SUCCESS));
        }
        completedContainerList.clear();
    }
    // released containers
    synchronized (releasedContainerList) {
        for (ContainerId cId : releasedContainerList) {
            LOG.debug(MessageFormat.format("NodeManager {0} released container" + " ({1}).", node.getNodeID(), cId));
            csList.add(newContainerStatus(cId, ContainerState.COMPLETE, ContainerExitStatus.ABORTED));
        }
        releasedContainerList.clear();
    }
    return csList;
}

Example 8 with ContainerSimulator

use of org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator in project hadoop by apache.

the class NMSimulator method middleStep.

@Override
public void middleStep() throws Exception {
    // we check the lifetime for each running containers
    ContainerSimulator cs = null;
    synchronized (completedContainerList) {
        while ((cs = containerQueue.poll()) != null) {
            runningContainers.remove(cs.getId());
            completedContainerList.add(cs.getId());
            LOG.debug(MessageFormat.format("Container {0} has completed", cs.getId()));
        }
    }
    // send heart beat
    NodeHeartbeatRequest beatRequest = Records.newRecord(NodeHeartbeatRequest.class);
    beatRequest.setLastKnownNMTokenMasterKey(masterKey);
    NodeStatus ns = Records.newRecord(NodeStatus.class);
    ns.setContainersStatuses(generateContainerStatusList());
    ns.setNodeId(node.getNodeID());
    ns.setKeepAliveApplications(new ArrayList<ApplicationId>());
    ns.setResponseId(RESPONSE_ID++);
    ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
    beatRequest.setNodeStatus(ns);
    NodeHeartbeatResponse beatResponse = rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
    if (!beatResponse.getContainersToCleanup().isEmpty()) {
        // remove from queue
        synchronized (releasedContainerList) {
            for (ContainerId containerId : beatResponse.getContainersToCleanup()) {
                if (amContainerList.contains(containerId)) {
                    // AM container (not killed?, only release)
                    synchronized (amContainerList) {
                        amContainerList.remove(containerId);
                    }
                    LOG.debug(MessageFormat.format("NodeManager {0} releases " + "an AM ({1}).", node.getNodeID(), containerId));
                } else {
                    cs = runningContainers.remove(containerId);
                    containerQueue.remove(cs);
                    releasedContainerList.add(containerId);
                    LOG.debug(MessageFormat.format("NodeManager {0} releases a " + "container ({1}).", node.getNodeID(), containerId));
                }
            }
        }
    }
    if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
        lastStep();
    }
}

Example 9 with ContainerSimulator

use of org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator in project hadoop by apache.

the class SLSRunner method startAMFromSLSTraces.

/**
   * parse workload information from sls trace files
   */
@SuppressWarnings("unchecked")
private void startAMFromSLSTraces(Resource containerResource, int heartbeatInterval) throws IOException {
    // parse from sls traces
    JsonFactory jsonF = new JsonFactory();
    ObjectMapper mapper = new ObjectMapper();
    for (String inputTrace : inputTraces) {
        Reader input = new InputStreamReader(new FileInputStream(inputTrace), "UTF-8");
        try {
            Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
            while (i.hasNext()) {
                Map jsonJob = i.next();
                // load job information
                long jobStartTime = Long.parseLong(jsonJob.get("job.start.ms").toString());
                long jobFinishTime = Long.parseLong(jsonJob.get("job.end.ms").toString());
                String user = (String) jsonJob.get("job.user");
                if (user == null)
                    user = "default";
                String queue = jsonJob.get("job.queue.name").toString();
                String oldAppId = jsonJob.get("job.id").toString();
                boolean isTracked = trackedApps.contains(oldAppId);
                int queueSize = queueAppNumMap.containsKey(queue) ? queueAppNumMap.get(queue) : 0;
                queueSize++;
                queueAppNumMap.put(queue, queueSize);
                // tasks
                List tasks = (List) jsonJob.get("job.tasks");
                if (tasks == null || tasks.size() == 0) {
                    continue;
                }
                List<ContainerSimulator> containerList = new ArrayList<ContainerSimulator>();
                for (Object o : tasks) {
                    Map jsonTask = (Map) o;
                    String hostname = jsonTask.get("container.host").toString();
                    long taskStart = Long.parseLong(jsonTask.get("container.start.ms").toString());
                    long taskFinish = Long.parseLong(jsonTask.get("container.end.ms").toString());
                    long lifeTime = taskFinish - taskStart;
                    // Set memory and vcores from job trace file
                    Resource res = Resources.clone(containerResource);
                    if (jsonTask.containsKey("container.memory")) {
                        int containerMemory = Integer.parseInt(jsonTask.get("container.memory").toString());
                        res.setMemorySize(containerMemory);
                    }
                    if (jsonTask.containsKey("container.vcores")) {
                        int containerVCores = Integer.parseInt(jsonTask.get("container.vcores").toString());
                        res.setVirtualCores(containerVCores);
                    }
                    int priority = Integer.parseInt(jsonTask.get("container.priority").toString());
                    String type = jsonTask.get("container.type").toString();
                    containerList.add(new ContainerSimulator(res, lifeTime, hostname, priority, type));
                }
                // create a new AM
                String amType = jsonJob.get("am.type").toString();
                AMSimulator amSim = (AMSimulator) ReflectionUtils.newInstance(amClassMap.get(amType), new Configuration());
                if (amSim != null) {
                    amSim.init(AM_ID++, heartbeatInterval, containerList, rm, this, jobStartTime, jobFinishTime, user, queue, isTracked, oldAppId);
                    runner.schedule(amSim);
                    maxRuntime = Math.max(maxRuntime, jobFinishTime);
                    numTasks += containerList.size();
                    amMap.put(oldAppId, amSim);
                }
            }
        } finally {
            input.close();
        }
    }
}

Example 10 with ContainerSimulator

use of org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator in project hadoop by apache.

the class AMSimulator method packageRequests.

protected List<ResourceRequest> packageRequests(List<ContainerSimulator> csList, int priority) {
    // create requests
    Map<String, ResourceRequest> rackLocalRequestMap = new HashMap<String, ResourceRequest>();
    Map<String, ResourceRequest> nodeLocalRequestMap = new HashMap<String, ResourceRequest>();
    ResourceRequest anyRequest = null;
    for (ContainerSimulator cs : csList) {
        String[] rackHostNames = SLSUtils.getRackHostName(cs.getHostname());
        // check rack local
        String rackname = "/" + rackHostNames[0];
        if (rackLocalRequestMap.containsKey(rackname)) {
            rackLocalRequestMap.get(rackname).setNumContainers(rackLocalRequestMap.get(rackname).getNumContainers() + 1);
        } else {
            ResourceRequest request = createResourceRequest(cs.getResource(), rackname, priority, 1);
            rackLocalRequestMap.put(rackname, request);
        }
        // check node local
        String hostname = rackHostNames[1];
        if (nodeLocalRequestMap.containsKey(hostname)) {
            nodeLocalRequestMap.get(hostname).setNumContainers(nodeLocalRequestMap.get(hostname).getNumContainers() + 1);
        } else {
            ResourceRequest request = createResourceRequest(cs.getResource(), hostname, priority, 1);
            nodeLocalRequestMap.put(hostname, request);
        }
        // any
        if (anyRequest == null) {
            anyRequest = createResourceRequest(cs.getResource(), ResourceRequest.ANY, priority, 1);
        } else {
            anyRequest.setNumContainers(anyRequest.getNumContainers() + 1);
        }
    }
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ask.addAll(nodeLocalRequestMap.values());
    ask.addAll(rackLocalRequestMap.values());
    if (anyRequest != null) {
        ask.add(anyRequest);
    }
    return ask;
}