Examples with TaskAttemptInfo org.apache.tez.history.parser.datamodel.TaskAttemptInfo used on opensource projects

Example 11 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class SlowTaskIdentifier method analyze.

@Override
public void analyze(DagInfo dagInfo) throws TezException {
    List<TaskAttemptInfo> taskAttempts = Lists.newArrayList();
    for (VertexInfo vertexInfo : dagInfo.getVertices()) {
        taskAttempts.addAll(vertexInfo.getTaskAttempts());
    }
    // sort them by runtime in descending order
    Collections.sort(taskAttempts, new Comparator<TaskAttemptInfo>() {

        @Override
        public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) {
            return (o1.getTimeTaken() > o2.getTimeTaken()) ? -1 : ((o1.getTimeTaken() == o2.getTimeTaken()) ? 0 : 1);
        }
    });
    int limit = Math.min(taskAttempts.size(), Math.max(0, config.getInt(NO_OF_TASKS, NO_OF_TASKS_DEFAULT)));
    if (limit == 0) {
        return;
    }
    for (int i = 0; i < limit - 1; i++) {
        List<String> record = Lists.newLinkedList();
        record.add(taskAttempts.get(i).getTaskInfo().getVertexInfo().getVertexName());
        record.add(taskAttempts.get(i).getTaskAttemptId());
        record.add(taskAttempts.get(i).getContainer().getHost());
        record.add(taskAttempts.get(i).getTimeTaken() + "");
        record.add(taskAttempts.get(i).getStatus());
        record.add(taskAttempts.get(i).getDiagnostics());
        record.add(taskAttempts.get(i).getTaskInfo().getVertexInfo().getInputEdges().size() + "");
        csvResult.addRecord(record.toArray(new String[record.size()]));
    }
}

Example 12 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class TaskConcurrencyAnalyzer method analyze.

@Override
public void analyze(DagInfo dagInfo) throws TezException {
    // For each vertex find the concurrent tasks running at any point
    for (VertexInfo vertexInfo : dagInfo.getVertices()) {
        List<TaskAttemptInfo> taskAttempts = Lists.newLinkedList(vertexInfo.getTaskAttempts(true, null));
        String vertexName = vertexInfo.getVertexName();
        /**
         * - Get sorted multi-set of timestamps (S1, S2,...E1, E2..). Possible to have multiple
         * tasks starting/ending at same time.
         * - Walk through the set
         * - Increment concurrent tasks when start event is encountered
         * - Decrement concurrent tasks when start event is encountered
         */
        TreeMultiset<TimeInfo> timeInfoSet = TreeMultiset.create(new Comparator<TimeInfo>() {

            @Override
            public int compare(TimeInfo o1, TimeInfo o2) {
                if (o1.timestamp < o2.timestamp) {
                    return -1;
                }
                if (o1.timestamp > o2.timestamp) {
                    return 1;
                }
                if (o1.timestamp == o2.timestamp) {
                    // check event type
                    if (o1.eventType.equals(o2.eventType)) {
                        return 0;
                    }
                    if (o1.eventType.equals(EventType.START) && o2.eventType.equals(EventType.FINISH)) {
                        return -1;
                    } else {
                        return 1;
                    }
                }
                return 0;
            }
        });
        for (TaskAttemptInfo attemptInfo : taskAttempts) {
            TimeInfo startTimeInfo = new TimeInfo(EventType.START, attemptInfo.getStartTime());
            TimeInfo stopTimeInfo = new TimeInfo(EventType.FINISH, attemptInfo.getFinishTime());
            timeInfoSet.add(startTimeInfo);
            timeInfoSet.add(stopTimeInfo);
        }
        // Compute concurrent tasks in the list now.
        int concurrentTasks = 0;
        for (TimeInfo timeInfo : timeInfoSet.elementSet()) {
            switch(timeInfo.eventType) {
                case START:
                    concurrentTasks += timeInfoSet.count(timeInfo);
                    break;
                case FINISH:
                    concurrentTasks -= timeInfoSet.count(timeInfo);
                    break;
                default:
                    break;
            }
            timeInfo.concurrentTasks = concurrentTasks;
            addToResult(vertexName, timeInfo.timestamp, timeInfo.concurrentTasks);
        }
    }
}

Example 13 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class TaskAssignmentAnalyzer method analyze.

@Override
public void analyze(DagInfo dagInfo) throws TezException {
    Map<String, Integer> map = new HashMap<>();
    for (VertexInfo vertex : dagInfo.getVertices()) {
        map.clear();
        for (TaskAttemptInfo attempt : vertex.getTaskAttempts()) {
            Integer previousValue = map.get(attempt.getNodeId());
            map.put(attempt.getNodeId(), previousValue == null ? 1 : previousValue + 1);
        }
        double mean = vertex.getTaskAttempts().size() / Math.max(1.0, map.size());
        for (Map.Entry<String, Integer> assignment : map.entrySet()) {
            addARecord(vertex.getVertexName(), assignment.getKey(), assignment.getValue(), assignment.getValue() * 100 / mean);
        }
    }
}

Example 14 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class CriticalPathAnalyzer method determineConcurrency.

private void determineConcurrency(DagInfo dag) {
    ArrayList<TimeInfo> timeInfo = Lists.newArrayList();
    for (VertexInfo v : dag.getVertices()) {
        for (TaskInfo t : v.getTasks()) {
            for (TaskAttemptInfo a : t.getTaskAttempts()) {
                if (a.getStartTime() > 0) {
                    timeInfo.add(new TimeInfo(a.getStartTime(), true));
                    timeInfo.add(new TimeInfo(a.getFinishTime(), false));
                }
            }
        }
    }
    Collections.sort(timeInfo);
    int concurrency = 0;
    TimeInfo lastTimeInfo = null;
    for (TimeInfo t : timeInfo) {
        concurrency += (t.start) ? 1 : -1;
        maxConcurrency = (concurrency > maxConcurrency) ? concurrency : maxConcurrency;
        if (lastTimeInfo == null || lastTimeInfo.timestamp < t.timestamp) {
            lastTimeInfo = t;
            lastTimeInfo.count = concurrency;
            concurrencyByTime.add(lastTimeInfo);
        } else {
            // lastTimeInfo.timestamp == t.timestamp
            lastTimeInfo.count = concurrency;
        }
    }
// for (TimeInfo t : concurrencyByTime) {
// System.out.println(t.timestamp + " " + t.count);
// }
}

Example 15 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class CriticalPathAnalyzer method analyzeWaves.

private void analyzeWaves(DagInfo dag) {
    for (int i = 0; i < criticalPath.size(); ++i) {
        CriticalPathStep step = criticalPath.get(i);
        TaskAttemptInfo attempt = step.attempt;
        if (step.getType() != EntityType.ATTEMPT) {
            continue;
        }
        long creationTime = attempt.getCreationTime();
        long finishTime = attempt.getFinishTime();
        int numVertexTasks = attempt.getTaskInfo().getVertexInfo().getNumTasks();
        if (numVertexTasks <= 1) {
            continue;
        }
        int intervalMaxConcurrency = getIntervalMaxConcurrency(creationTime, finishTime);
        double numWaves = getWaves(numVertexTasks, intervalMaxConcurrency);
        step.notes.add("Vertex ran " + numVertexTasks + " tasks in " + numWaves + " waves with available concurrency of " + intervalMaxConcurrency);
        if (numWaves > 1) {
            if (numWaves % 1 < 0.5) {
                // more than 1 wave needed and last wave is small
                step.notes.add("Last partial wave did not use full concurrency. ");
            }
        }
    }
}