Examples with HadoopPerformanceCounter org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter used on opensource projects

Example 1 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopJobTracker method submit.

/**
 * Submits execution of Hadoop job to grid.
 *
 * @param jobId Job ID.
 * @param info Job info.
 * @return Job completion future.
 */
@SuppressWarnings("unchecked")
public IgniteInternalFuture<HadoopJobId> submit(HadoopJobId jobId, HadoopJobInfo info) {
    if (!busyLock.tryReadLock()) {
        return new GridFinishedFuture<>(new IgniteCheckedException("Failed to execute map-reduce job " + "(grid is stopping): " + info));
    }
    try {
        long jobPrepare = U.currentTimeMillis();
        if (jobs.containsKey(jobId) || jobMetaCache().containsKey(jobId))
            throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
        HadoopJobEx job = job(jobId, info);
        HadoopMapReducePlan mrPlan = mrPlanner.preparePlan(job, ctx.nodes(), null);
        logPlan(info, mrPlan);
        HadoopJobMetadata meta = new HadoopJobMetadata(ctx.localNodeId(), jobId, info);
        meta.mapReducePlan(mrPlan);
        meta.pendingSplits(allSplits(mrPlan));
        meta.pendingReducers(allReducers(mrPlan));
        GridFutureAdapter<HadoopJobId> completeFut = new GridFutureAdapter<>();
        GridFutureAdapter<HadoopJobId> old = activeFinishFuts.put(jobId, completeFut);
        assert old == null : "Duplicate completion future [jobId=" + jobId + ", old=" + old + ']';
        if (log.isDebugEnabled())
            log.debug("Submitting job metadata [jobId=" + jobId + ", meta=" + meta + ']');
        long jobStart = U.currentTimeMillis();
        HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(meta.counters(), ctx.localNodeId());
        perfCntr.clientSubmissionEvents(info);
        perfCntr.onJobPrepare(jobPrepare);
        perfCntr.onJobStart(jobStart);
        if (jobMetaCache().getAndPutIfAbsent(jobId, meta) != null)
            throw new IgniteCheckedException("Failed to submit job. Job with the same ID already exists: " + jobId);
        return completeFut;
    } catch (IgniteCheckedException e) {
        U.error(log, "Failed to submit job: " + jobId, e);
        return new GridFinishedFuture<>(e);
    } finally {
        busyLock.readUnlock();
    }
}

Example 2 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopRunnableTask method call0.

/**
 * Implements actual task running.
 * @throws IgniteCheckedException On error.
 */
void call0() throws IgniteCheckedException {
    execStartTs = U.currentTimeMillis();
    Throwable err = null;
    HadoopTaskState state = HadoopTaskState.COMPLETED;
    HadoopPerformanceCounter perfCntr = null;
    try {
        perfCntr = HadoopPerformanceCounter.getCounter(ctx.counters(), nodeId);
        perfCntr.onTaskSubmit(info, submitTs);
        perfCntr.onTaskPrepare(info, execStartTs);
        ctx.prepareTaskEnvironment();
        runTask(perfCntr);
        if (info.type() == MAP && job.info().hasCombiner()) {
            // Switch to combiner.
            HadoopTaskInfo combineTaskInfo = new HadoopTaskInfo(COMBINE, info.jobId(), info.taskNumber(), info.attempt(), null);
            // Mapper and combiner share the same index.
            if (ctx.taskInfo().hasMapperIndex())
                combineTaskInfo.mapperIndex(ctx.taskInfo().mapperIndex());
            ctx.taskInfo(combineTaskInfo);
            try {
                runTask(perfCntr);
            } finally {
                ctx.taskInfo(info);
            }
        }
    } catch (HadoopTaskCancelledException ignored) {
        state = HadoopTaskState.CANCELED;
    } catch (Throwable e) {
        state = HadoopTaskState.FAILED;
        err = e;
        U.error(log, "Task execution failed.", e);
        if (e instanceof Error)
            throw e;
    } finally {
        execEndTs = U.currentTimeMillis();
        if (perfCntr != null)
            perfCntr.onTaskFinish(info, execEndTs);
        onTaskFinished(new HadoopTaskStatus(state, err, ctx == null ? null : ctx.counters()));
        if (combinerInput != null)
            combinerInput.close();
        if (ctx != null)
            ctx.cleanupTaskEnvironment();
    }
}

Example 3 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopAbstractMapReduceTest method checkJobStatistics.

/**
 * Simple test job statistics.
 *
 * @param jobId Job id.
 * @throws IgniteCheckedException
 */
private void checkJobStatistics(HadoopJobId jobId) throws IgniteCheckedException, IOException {
    HadoopCounters cntrs = grid(0).hadoop().counters(jobId);
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
    Map<String, SortedMap<Integer, Long>> tasks = new TreeMap<>();
    Map<String, Integer> phaseOrders = new HashMap<>();
    phaseOrders.put("submit", 0);
    phaseOrders.put("prepare", 1);
    phaseOrders.put("start", 2);
    phaseOrders.put("Cstart", 3);
    phaseOrders.put("finish", 4);
    String prevTaskId = null;
    long apiEvtCnt = 0;
    for (T2<String, Long> evt : perfCntr.evts()) {
        // We expect string pattern: COMBINE 1 run 7fa86a14-5a08-40e3-a7cb-98109b52a706
        String[] parsedEvt = evt.get1().split(" ");
        String taskId;
        String taskPhase;
        if ("JOB".equals(parsedEvt[0])) {
            taskId = parsedEvt[0];
            taskPhase = parsedEvt[1];
        } else {
            taskId = ("COMBINE".equals(parsedEvt[0]) ? "MAP" : parsedEvt[0].substring(0, 3)) + parsedEvt[1];
            taskPhase = ("COMBINE".equals(parsedEvt[0]) ? "C" : "") + parsedEvt[2];
        }
        if (!taskId.equals(prevTaskId))
            tasks.put(taskId, new TreeMap<Integer, Long>());
        Integer pos = phaseOrders.get(taskPhase);
        assertNotNull("Invalid phase " + taskPhase, pos);
        tasks.get(taskId).put(pos, evt.get2());
        prevTaskId = taskId;
        apiEvtCnt++;
    }
    for (Map.Entry<String, SortedMap<Integer, Long>> task : tasks.entrySet()) {
        Map<Integer, Long> order = task.getValue();
        long prev = 0;
        for (Map.Entry<Integer, Long> phase : order.entrySet()) {
            assertTrue("Phase order of " + task.getKey() + " is invalid", phase.getValue() >= prev);
            prev = phase.getValue();
        }
    }
    final IgfsPath statPath = new IgfsPath("/xxx/" + USER + "/zzz/" + jobId + "/performance");
    assert GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            return igfs.exists(statPath);
        }
    }, 20_000);
    final long apiEvtCnt0 = apiEvtCnt;
    boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {

        @Override
        public boolean apply() {
            try {
                try (BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)))) {
                    return apiEvtCnt0 == HadoopTestUtils.simpleCheckJobStatFile(reader);
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }, 10000);
    if (!res) {
        BufferedReader reader = new BufferedReader(new InputStreamReader(igfs.open(statPath)));
        assert false : "Invalid API events count [exp=" + apiEvtCnt0 + ", actual=" + HadoopTestUtils.simpleCheckJobStatFile(reader) + ']';
    }
}

Example 4 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopFileSystemCounterWriterDelegateImpl method write.

/**
 * {@inheritDoc}
 */
public void write(HadoopJobEx job, HadoopCounters cntrs) throws IgniteCheckedException {
    Configuration hadoopCfg = HadoopUtils.safeCreateConfiguration();
    final HadoopJobInfo jobInfo = job.info();
    final HadoopJobId jobId = job.id();
    for (Map.Entry<String, String> e : ((HadoopDefaultJobInfo) jobInfo).properties().entrySet()) hadoopCfg.set(e.getKey(), e.getValue());
    String user = jobInfo.user();
    user = IgfsUtils.fixUserName(user);
    String dir = jobInfo.property(IgniteHadoopFileSystemCounterWriter.COUNTER_WRITER_DIR_PROPERTY);
    if (dir == null)
        dir = DEFAULT_COUNTER_WRITER_DIR;
    Path jobStatPath = new Path(new Path(dir.replace(USER_MACRO, user)), jobId.toString());
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
    try {
        hadoopCfg.set(MRJobConfig.USER_NAME, user);
        FileSystem fs = ((HadoopV2Job) job).fileSystem(jobStatPath.toUri(), hadoopCfg);
        fs.mkdirs(jobStatPath);
        try (PrintStream out = new PrintStream(fs.create(new Path(jobStatPath, IgniteHadoopFileSystemCounterWriter.PERFORMANCE_COUNTER_FILE_NAME)))) {
            for (T2<String, Long> evt : perfCntr.evts()) {
                out.print(evt.get1());
                out.print(':');
                out.println(evt.get2().toString());
            }
            out.flush();
        }
    } catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}

Example 5 with HadoopPerformanceCounter

use of org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCounter in project ignite by apache.

the class HadoopShuffleJob method onShuffleMessage.

/**
 * @param src Source.
 * @param msg Message.
 * @throws IgniteCheckedException Exception.
 */
public void onShuffleMessage(T src, HadoopShuffleMessage msg) throws IgniteCheckedException {
    assert msg.buffer() != null;
    assert msg.offset() > 0;
    HadoopTaskContext taskCtx = locReducersCtx.get(msg.reducer()).get();
    HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(taskCtx.counters(), null);
    perfCntr.onShuffleMessage(msg.reducer(), U.currentTimeMillis());
    HadoopMultimap map = getOrCreateMap(locMaps, msg.reducer());
    // Add data from message to the map.
    try (HadoopMultimap.Adder adder = map.startAdding(taskCtx)) {
        final GridUnsafeDataInput dataInput = new GridUnsafeDataInput();
        final UnsafeValue val = new UnsafeValue(msg.buffer());
        msg.visit(new HadoopShuffleMessage.Visitor() {

            /**
             */
            private HadoopMultimap.Key key;

            @Override
            public void onKey(byte[] buf, int off, int len) throws IgniteCheckedException {
                dataInput.bytes(buf, off, off + len);
                key = adder.addKey(dataInput, key);
            }

            @Override
            public void onValue(byte[] buf, int off, int len) {
                val.off = off;
                val.size = len;
                key.add(val);
            }
        });
    }
    if (embedded) {
        // No immediate response.
        if (localShuffleState(src).onShuffleMessage())
            sendFinishResponse(src, msg.jobId());
    } else
        // Response for every message.
        io.apply(src, new HadoopShuffleAck(msg.id(), msg.jobId()));
}