Examples with AMSimulator org.apache.hadoop.yarn.sls.appmaster.AMSimulator used on opensource projects

Example 1 with AMSimulator

use of org.apache.hadoop.yarn.sls.appmaster.AMSimulator in project hadoop by apache.

the class SLSRunner method printSimulationInfo.

private void printSimulationInfo() {
    if (printSimulation) {
        // node
        LOG.info("------------------------------------");
        LOG.info(MessageFormat.format("# nodes = {0}, # racks = {1}, capacity " + "of each node {2} MB memory and {3} vcores.", numNMs, numRacks, nmMemoryMB, nmVCores));
        LOG.info("------------------------------------");
        // job
        LOG.info(MessageFormat.format("# applications = {0}, # total " + "tasks = {1}, average # tasks per application = {2}", numAMs, numTasks, (int) (Math.ceil((numTasks + 0.0) / numAMs))));
        LOG.info("JobId\tQueue\tAMType\tDuration\t#Tasks");
        for (Map.Entry<String, AMSimulator> entry : amMap.entrySet()) {
            AMSimulator am = entry.getValue();
            LOG.info(entry.getKey() + "\t" + am.getQueue() + "\t" + am.getAMType() + "\t" + am.getDuration() + "\t" + am.getNumTasks());
        }
        LOG.info("------------------------------------");
        // queue
        LOG.info(MessageFormat.format("number of queues = {0}  average " + "number of apps = {1}", queueAppNumMap.size(), (int) (Math.ceil((numAMs + 0.0) / queueAppNumMap.size()))));
        LOG.info("------------------------------------");
        // runtime
        LOG.info(MessageFormat.format("estimated simulation time is {0}" + " seconds", (long) (Math.ceil(maxRuntime / 1000.0))));
        LOG.info("------------------------------------");
    }
    // package these information in the simulateInfoMap used by other places
    simulateInfoMap.put("Number of racks", numRacks);
    simulateInfoMap.put("Number of nodes", numNMs);
    simulateInfoMap.put("Node memory (MB)", nmMemoryMB);
    simulateInfoMap.put("Node VCores", nmVCores);
    simulateInfoMap.put("Number of applications", numAMs);
    simulateInfoMap.put("Number of tasks", numTasks);
    simulateInfoMap.put("Average tasks per applicaion", (int) (Math.ceil((numTasks + 0.0) / numAMs)));
    simulateInfoMap.put("Number of queues", queueAppNumMap.size());
    simulateInfoMap.put("Average applications per queue", (int) (Math.ceil((numAMs + 0.0) / queueAppNumMap.size())));
    simulateInfoMap.put("Estimated simulate time (s)", (long) (Math.ceil(maxRuntime / 1000.0)));
}

Example 2 with AMSimulator

use of org.apache.hadoop.yarn.sls.appmaster.AMSimulator in project hadoop by apache.

the class SLSRunner method startAMFromRumenTraces.

/**
   * parse workload information from rumen trace files
   */
@SuppressWarnings("unchecked")
private void startAMFromRumenTraces(Resource containerResource, int heartbeatInterval) throws IOException {
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", "file:///");
    long baselineTimeMS = 0;
    for (String inputTrace : inputTraces) {
        File fin = new File(inputTrace);
        JobTraceReader reader = new JobTraceReader(new Path(fin.getAbsolutePath()), conf);
        try {
            LoggedJob job = null;
            while ((job = reader.getNext()) != null) {
                // only support MapReduce currently
                String jobType = "mapreduce";
                String user = job.getUser() == null ? "default" : job.getUser().getValue();
                String jobQueue = job.getQueue().getValue();
                String oldJobId = job.getJobID().toString();
                long jobStartTimeMS = job.getSubmitTime();
                long jobFinishTimeMS = job.getFinishTime();
                if (baselineTimeMS == 0) {
                    baselineTimeMS = jobStartTimeMS;
                }
                jobStartTimeMS -= baselineTimeMS;
                jobFinishTimeMS -= baselineTimeMS;
                if (jobStartTimeMS < 0) {
                    LOG.warn("Warning: reset job " + oldJobId + " start time to 0.");
                    jobFinishTimeMS = jobFinishTimeMS - jobStartTimeMS;
                    jobStartTimeMS = 0;
                }
                boolean isTracked = trackedApps.contains(oldJobId);
                int queueSize = queueAppNumMap.containsKey(jobQueue) ? queueAppNumMap.get(jobQueue) : 0;
                queueSize++;
                queueAppNumMap.put(jobQueue, queueSize);
                List<ContainerSimulator> containerList = new ArrayList<ContainerSimulator>();
                // map tasks
                for (LoggedTask mapTask : job.getMapTasks()) {
                    if (mapTask.getAttempts().size() == 0) {
                        continue;
                    }
                    LoggedTaskAttempt taskAttempt = mapTask.getAttempts().get(mapTask.getAttempts().size() - 1);
                    String hostname = taskAttempt.getHostName().getValue();
                    long containerLifeTime = taskAttempt.getFinishTime() - taskAttempt.getStartTime();
                    containerList.add(new ContainerSimulator(containerResource, containerLifeTime, hostname, 10, "map"));
                }
                // reduce tasks
                for (LoggedTask reduceTask : job.getReduceTasks()) {
                    if (reduceTask.getAttempts().size() == 0) {
                        continue;
                    }
                    LoggedTaskAttempt taskAttempt = reduceTask.getAttempts().get(reduceTask.getAttempts().size() - 1);
                    String hostname = taskAttempt.getHostName().getValue();
                    long containerLifeTime = taskAttempt.getFinishTime() - taskAttempt.getStartTime();
                    containerList.add(new ContainerSimulator(containerResource, containerLifeTime, hostname, 20, "reduce"));
                }
                // create a new AM
                AMSimulator amSim = (AMSimulator) ReflectionUtils.newInstance(amClassMap.get(jobType), conf);
                if (amSim != null) {
                    amSim.init(AM_ID++, heartbeatInterval, containerList, rm, this, jobStartTimeMS, jobFinishTimeMS, user, jobQueue, isTracked, oldJobId);
                    runner.schedule(amSim);
                    maxRuntime = Math.max(maxRuntime, jobFinishTimeMS);
                    numTasks += containerList.size();
                    amMap.put(oldJobId, amSim);
                }
            }
        } finally {
            reader.close();
        }
    }
}

Example 3 with AMSimulator

use of org.apache.hadoop.yarn.sls.appmaster.AMSimulator in project hadoop by apache.

the class MockAMLauncher method handle.

@Override
@SuppressWarnings("unchecked")
public void handle(AMLauncherEvent event) {
    if (AMLauncherEventType.LAUNCH == event.getType()) {
        ApplicationId appId = event.getAppAttempt().getAppAttemptId().getApplicationId();
        // find AMSimulator
        for (AMSimulator ams : amMap.values()) {
            if (ams.getApplicationId() != null && ams.getApplicationId().equals(appId)) {
                try {
                    Container amContainer = event.getAppAttempt().getMasterContainer();
                    setupAMRMToken(event.getAppAttempt());
                    // Notify RMAppAttempt to change state
                    super.context.getDispatcher().getEventHandler().handle(new RMAppAttemptEvent(event.getAppAttempt().getAppAttemptId(), RMAppAttemptEventType.LAUNCHED));
                    ams.notifyAMContainerLaunched(event.getAppAttempt().getMasterContainer());
                    LOG.info("Notify AM launcher launched:" + amContainer.getId());
                    se.getNmMap().get(amContainer.getNodeId()).addNewContainer(amContainer, 100000000L);
                    return;
                } catch (Exception e) {
                    throw new YarnRuntimeException(e);
                }
            }
        }
        throw new YarnRuntimeException("Didn't find any AMSimulator for applicationId=" + appId);
    }
}

Example 4 with AMSimulator

use of org.apache.hadoop.yarn.sls.appmaster.AMSimulator in project hadoop by apache.

the class SLSRunner method startAMFromSLSTraces.

/**
   * parse workload information from sls trace files
   */
@SuppressWarnings("unchecked")
private void startAMFromSLSTraces(Resource containerResource, int heartbeatInterval) throws IOException {
    // parse from sls traces
    JsonFactory jsonF = new JsonFactory();
    ObjectMapper mapper = new ObjectMapper();
    for (String inputTrace : inputTraces) {
        Reader input = new InputStreamReader(new FileInputStream(inputTrace), "UTF-8");
        try {
            Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
            while (i.hasNext()) {
                Map jsonJob = i.next();
                // load job information
                long jobStartTime = Long.parseLong(jsonJob.get("job.start.ms").toString());
                long jobFinishTime = Long.parseLong(jsonJob.get("job.end.ms").toString());
                String user = (String) jsonJob.get("job.user");
                if (user == null)
                    user = "default";
                String queue = jsonJob.get("job.queue.name").toString();
                String oldAppId = jsonJob.get("job.id").toString();
                boolean isTracked = trackedApps.contains(oldAppId);
                int queueSize = queueAppNumMap.containsKey(queue) ? queueAppNumMap.get(queue) : 0;
                queueSize++;
                queueAppNumMap.put(queue, queueSize);
                // tasks
                List tasks = (List) jsonJob.get("job.tasks");
                if (tasks == null || tasks.size() == 0) {
                    continue;
                }
                List<ContainerSimulator> containerList = new ArrayList<ContainerSimulator>();
                for (Object o : tasks) {
                    Map jsonTask = (Map) o;
                    String hostname = jsonTask.get("container.host").toString();
                    long taskStart = Long.parseLong(jsonTask.get("container.start.ms").toString());
                    long taskFinish = Long.parseLong(jsonTask.get("container.end.ms").toString());
                    long lifeTime = taskFinish - taskStart;
                    // Set memory and vcores from job trace file
                    Resource res = Resources.clone(containerResource);
                    if (jsonTask.containsKey("container.memory")) {
                        int containerMemory = Integer.parseInt(jsonTask.get("container.memory").toString());
                        res.setMemorySize(containerMemory);
                    }
                    if (jsonTask.containsKey("container.vcores")) {
                        int containerVCores = Integer.parseInt(jsonTask.get("container.vcores").toString());
                        res.setVirtualCores(containerVCores);
                    }
                    int priority = Integer.parseInt(jsonTask.get("container.priority").toString());
                    String type = jsonTask.get("container.type").toString();
                    containerList.add(new ContainerSimulator(res, lifeTime, hostname, priority, type));
                }
                // create a new AM
                String amType = jsonJob.get("am.type").toString();
                AMSimulator amSim = (AMSimulator) ReflectionUtils.newInstance(amClassMap.get(amType), new Configuration());
                if (amSim != null) {
                    amSim.init(AM_ID++, heartbeatInterval, containerList, rm, this, jobStartTime, jobFinishTime, user, queue, isTracked, oldAppId);
                    runner.schedule(amSim);
                    maxRuntime = Math.max(maxRuntime, jobFinishTime);
                    numTasks += containerList.size();
                    amMap.put(oldAppId, amSim);
                }
            }
        } finally {
            input.close();
        }
    }
}