Examples with HadoopCounters org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters used on opensource projects

Example 1 with HadoopCounters

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters in project ignite by apache.

the class HadoopJobTracker method processJobMetaUpdate.

/**
 * @param jobId Job ID.
 * @param meta Job metadata.
 * @param locNodeId Local node ID.
 * @throws IgniteCheckedException If failed.
 */
private void processJobMetaUpdate(HadoopJobId jobId, HadoopJobMetadata meta, UUID locNodeId) throws IgniteCheckedException {
    JobLocalState state = activeJobs.get(jobId);
    HadoopJobEx job = job(jobId, meta.jobInfo());
    HadoopMapReducePlan plan = meta.mapReducePlan();
    switch(meta.phase()) {
        case PHASE_SETUP:
            {
                if (ctx.jobUpdateLeader()) {
                    Collection<HadoopTaskInfo> setupTask = setupTask(jobId);
                    if (setupTask != null)
                        ctx.taskExecutor().run(job, setupTask);
                }
                break;
            }
        case PHASE_MAP:
            {
                // Check if we should initiate new task on local node.
                Collection<HadoopTaskInfo> tasks = mapperTasks(plan.mappers(locNodeId), meta);
                if (tasks != null)
                    ctx.taskExecutor().run(job, tasks);
                break;
            }
        case PHASE_REDUCE:
            {
                if (meta.pendingReducers().isEmpty() && ctx.jobUpdateLeader()) {
                    HadoopTaskInfo info = new HadoopTaskInfo(COMMIT, jobId, 0, 0, null);
                    if (log.isDebugEnabled())
                        log.debug("Submitting COMMIT task for execution [locNodeId=" + locNodeId + ", jobId=" + jobId + ']');
                    ctx.taskExecutor().run(job, Collections.singletonList(info));
                    break;
                }
                Collection<HadoopTaskInfo> tasks = reducerTasks(plan.reducers(locNodeId), job);
                if (tasks != null)
                    ctx.taskExecutor().run(job, tasks);
                break;
            }
        case PHASE_CANCELLING:
            {
                // Prevent multiple task executor notification.
                if (state != null && state.onCancel()) {
                    if (log.isDebugEnabled())
                        log.debug("Cancelling local task execution for job: " + meta);
                    ctx.taskExecutor().cancelTasks(jobId);
                }
                if (meta.pendingSplits().isEmpty() && meta.pendingReducers().isEmpty()) {
                    if (ctx.jobUpdateLeader()) {
                        if (state == null)
                            state = initState(jobId);
                        // Prevent running multiple abort tasks.
                        if (state.onAborted()) {
                            HadoopTaskInfo info = new HadoopTaskInfo(ABORT, jobId, 0, 0, null);
                            if (log.isDebugEnabled())
                                log.debug("Submitting ABORT task for execution [locNodeId=" + locNodeId + ", jobId=" + jobId + ']');
                            ctx.taskExecutor().run(job, Collections.singletonList(info));
                        }
                    }
                    break;
                } else {
                    // Check if there are unscheduled mappers or reducers.
                    Collection<HadoopInputSplit> cancelMappers = new ArrayList<>();
                    Collection<Integer> cancelReducers = new ArrayList<>();
                    Collection<HadoopInputSplit> mappers = plan.mappers(ctx.localNodeId());
                    if (mappers != null) {
                        for (HadoopInputSplit b : mappers) {
                            if (state == null || !state.mapperScheduled(b))
                                cancelMappers.add(b);
                        }
                    }
                    int[] rdc = plan.reducers(ctx.localNodeId());
                    if (rdc != null) {
                        for (int r : rdc) {
                            if (state == null || !state.reducerScheduled(r))
                                cancelReducers.add(r);
                        }
                    }
                    if (!cancelMappers.isEmpty() || !cancelReducers.isEmpty())
                        transform(jobId, new CancelJobProcessor(null, cancelMappers, cancelReducers));
                }
                break;
            }
        case PHASE_COMPLETE:
            {
                if (log.isDebugEnabled())
                    log.debug("Job execution is complete, will remove local state from active jobs " + "[jobId=" + jobId + ", meta=" + meta + ']');
                if (state != null) {
                    state = activeJobs.remove(jobId);
                    assert state != null;
                    ctx.shuffle().jobFinished(jobId);
                }
                GridFutureAdapter<HadoopJobId> finishFut = activeFinishFuts.remove(jobId);
                if (finishFut != null) {
                    if (log.isDebugEnabled())
                        log.debug("Completing job future [locNodeId=" + locNodeId + ", meta=" + meta + ']');
                    finishFut.onDone(jobId, meta.failCause());
                }
                assert job != null;
                if (ctx.jobUpdateLeader())
                    job.cleanupStagingDirectory();
                jobs.remove(jobId);
                if (ctx.jobUpdateLeader()) {
                    ClassLoader ldr = job.getClass().getClassLoader();
                    try {
                        String statWriterClsName = job.info().property(HadoopCommonUtils.JOB_COUNTER_WRITER_PROPERTY);
                        if (statWriterClsName != null) {
                            Class<?> cls = ldr.loadClass(statWriterClsName);
                            HadoopCounterWriter writer = (HadoopCounterWriter) cls.newInstance();
                            HadoopCounters cntrs = meta.counters();
                            writer.write(job, cntrs);
                        }
                    } catch (Exception e) {
                        log.error("Can't write statistic due to: ", e);
                    }
                }
                job.dispose(false);
                break;
            }
        default:
            throw new IllegalStateException("Unknown phase: " + meta.phase());
    }
}

Example 2 with HadoopCounters

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters in project ignite by apache.

the class HadoopAbstractMapReduceTest method checkJobStatistics.

/**
 * Simple test job statistics.
 *
 * @param jobId Job id.
 * @throws IgniteCheckedException
 */
private void checkJobStatistics(HadoopJobId jobId) throws IgniteCheckedException, IOException {
    HadoopCounters cntrs = grid(0).hadoop().counters(jobId);
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
    Map<String, SortedMap<Integer, Long>> tasks = new TreeMap<>();
    Map<String, Integer> phaseOrders = new HashMap<>();
    phaseOrders.put("submit", 0);
    phaseOrders.put("prepare", 1);
    phaseOrders.put("start", 2);
    phaseOrders.put("Cstart", 3);
    phaseOrders.put("finish", 4);
    String prevTaskId = null;
    long apiEvtCnt = 0;
    for (T2<String, Long> evt : perfCntr.evts()) {
        // We expect string pattern: COMBINE 1 run 7fa86a14-5a08-40e3-a7cb-98109b52a706
        String[] parsedEvt = evt.get1().split(" ");
        String taskId;
        String taskPhase;
        if ("JOB".equals(parsedEvt[0])) {
            taskId = parsedEvt[0];
            taskPhase = parsedEvt[1];
        } else {
            taskId = ("COMBINE".equals(parsedEvt[0]) ? "MAP" : parsedEvt[0].substring(0, 3)) + parsedEvt[1];
            taskPhase = ("COMBINE".equals(parsedEvt[0]) ? "C" : "") + parsedEvt[2];
        }
        if (!taskId.equals(prevTaskId))
            tasks.put(taskId, new TreeMap<Integer, Long>());
        Integer pos = phaseOrders.get(taskPhase);
        assertNotNull("Invalid phase " + taskPhase, pos);
        tasks.get(taskId).put(pos, evt.get2());
        prevTaskId = taskId;
        apiEvtCnt++;
    }
    for (Map.Entry<String, SortedMap<Integer, Long>> task : tasks.entrySet()) {
        Map<Integer, Long> order = task.getValue();
        long prev = 0;
        for (Map.Entry<Integer, Long> phase : order.entrySet()) {
            assertTrue("Phase order of " + task.getKey() + " is invalid", phase.getValue() >= prev);
            prev = phase.getValue();
        }
    }
    final IgfsPath statPath = new IgfsPath("/xxx/" + USER + "/zzz/" + jobId + "/performance");
    assert GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return igfs.exists(statPath);
        }
    }, 20_000);
    final long apiEvtCnt0 = apiEvtCnt;
    boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            try {
                try (BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)))) {
                    return apiEvtCnt0 == HadoopTestUtils.simpleCheckJobStatFile(reader);
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }, 10000);
    if (!res) {
        BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)));
        assert false : "Invalid API events count [exp=" + apiEvtCnt0 + ", actual=" + HadoopTestUtils.simpleCheckJobStatFile(reader) + ']';
    }
}