Examples with HadoopJobPhase org.apache.ignite.internal.processors.hadoop.HadoopJobPhase used on opensource projects

Example 1 with HadoopJobPhase

use of org.apache.ignite.internal.processors.hadoop.HadoopJobPhase in project ignite by apache.

the class HadoopJobTracker method processNodeLeft.

/**
 * Processes node leave (or fail) event.
 *
 * @param evt Discovery event.
 */
@SuppressWarnings("ConstantConditions")
private void processNodeLeft(DiscoveryEvent evt) {
    if (log.isDebugEnabled())
        log.debug("Processing discovery event [locNodeId=" + ctx.localNodeId() + ", evt=" + evt + ']');
    // Check only if this node is responsible for job status updates.
    if (ctx.jobUpdateLeader()) {
        boolean checkSetup = evt.eventNode().order() < ctx.localNodeOrder();
        Iterable<IgniteCache.Entry<HadoopJobId, HadoopJobMetadata>> entries;
        try {
            entries = jobMetaCache().localEntries(OFFHEAP_PEEK_MODE);
        } catch (IgniteCheckedException e) {
            U.error(log, "Failed to get local entries", e);
            return;
        }
        // Iteration over all local entries is correct since system cache is REPLICATED.
        for (IgniteCache.Entry<HadoopJobId, HadoopJobMetadata> entry : entries) {
            HadoopJobMetadata meta = entry.getValue();
            HadoopJobId jobId = meta.jobId();
            HadoopMapReducePlan plan = meta.mapReducePlan();
            HadoopJobPhase phase = meta.phase();
            try {
                if (checkSetup && phase == PHASE_SETUP && !activeJobs.containsKey(jobId)) {
                    // Failover setup task.
                    HadoopJobEx job = job(jobId, meta.jobInfo());
                    Collection<HadoopTaskInfo> setupTask = setupTask(jobId);
                    assert setupTask != null;
                    ctx.taskExecutor().run(job, setupTask);
                } else if (phase == PHASE_MAP || phase == PHASE_REDUCE) {
                    // Must check all nodes, even that are not event node ID due to
                    // multiple node failure possibility.
                    Collection<HadoopInputSplit> cancelSplits = null;
                    for (UUID nodeId : plan.mapperNodeIds()) {
                        if (ctx.kernalContext().discovery().node(nodeId) == null) {
                            // Node has left the grid.
                            Collection<HadoopInputSplit> mappers = plan.mappers(nodeId);
                            if (cancelSplits == null)
                                cancelSplits = new HashSet<>();
                            cancelSplits.addAll(mappers);
                        }
                    }
                    Collection<Integer> cancelReducers = null;
                    for (UUID nodeId : plan.reducerNodeIds()) {
                        if (ctx.kernalContext().discovery().node(nodeId) == null) {
                            // Node has left the grid.
                            int[] reducers = plan.reducers(nodeId);
                            if (cancelReducers == null)
                                cancelReducers = new HashSet<>();
                            for (int rdc : reducers) cancelReducers.add(rdc);
                        }
                    }
                    if (cancelSplits != null || cancelReducers != null)
                        jobMetaCache().invoke(meta.jobId(), new CancelJobProcessor(null, new IgniteCheckedException("One or more nodes participating in map-reduce job execution failed."), cancelSplits, cancelReducers));
                }
            } catch (IgniteCheckedException e) {
                U.error(log, "Failed to cancel job: " + meta, e);
            }
        }
    }
}