Examples with HadoopJobId org.apache.ignite.internal.processors.hadoop.HadoopJobId used on opensource projects

Example 1 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopShuffleFinishRequest method readExternal.

/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobId = new HadoopJobId();
    jobId.readExternal(in);
    msgCnt = in.readLong();
}

Example 2 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopShuffleFinishResponse method readExternal.

/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobId = new HadoopJobId();
    jobId.readExternal(in);
}

Example 3 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopExternalTaskExecutionSelfTest method testSimpleTaskSubmit.

/**
     * @throws Exception If failed.
     */
public void testSimpleTaskSubmit() throws Exception {
    String testInputFile = "/test";
    prepareTestFile(testInputFile);
    Configuration cfg = new Configuration();
    setupFileSystems(cfg);
    Job job = Job.getInstance(cfg);
    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestReducer.class);
    job.setReducerClass(TestReducer.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(1);
    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestIgniteInstanceName(0) + "@/" + testInputFile));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestIgniteInstanceName(0) + "@/output"));
    job.setJarByClass(getClass());
    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()));
    fut.get();
}

Example 4 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopJobTracker method reducerTasks.

/**
     * Creates reducer tasks based on job information.
     *
     * @param reducers Reducers (may be {@code null}).
     * @param job Job instance.
     * @return Collection of task infos.
     */
private Collection<HadoopTaskInfo> reducerTasks(int[] reducers, HadoopJobEx job) {
    UUID locNodeId = ctx.localNodeId();
    HadoopJobId jobId = job.id();
    JobLocalState state = activeJobs.get(jobId);
    Collection<HadoopTaskInfo> tasks = null;
    if (reducers != null) {
        if (state == null)
            state = initState(job.id());
        for (int rdc : reducers) {
            if (state.addReducer(rdc)) {
                if (log.isDebugEnabled())
                    log.debug("Submitting REDUCE task for execution [locNodeId=" + locNodeId + ", rdc=" + rdc + ']');
                HadoopTaskInfo taskInfo = new HadoopTaskInfo(REDUCE, jobId, rdc, 0, null);
                if (tasks == null)
                    tasks = new ArrayList<>();
                tasks.add(taskInfo);
            }
        }
    }
    return tasks;
}

Example 5 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopJobTracker method mapperTasks.

/**
     * Creates mapper tasks based on job information.
     *
     * @param mappers Mapper blocks.
     * @param meta Job metadata.
     * @return Collection of created task infos or {@code null} if no mapper tasks scheduled for local node.
     */
private Collection<HadoopTaskInfo> mapperTasks(Iterable<HadoopInputSplit> mappers, HadoopJobMetadata meta) {
    UUID locNodeId = ctx.localNodeId();
    HadoopJobId jobId = meta.jobId();
    JobLocalState state = activeJobs.get(jobId);
    Collection<HadoopTaskInfo> tasks = null;
    if (mappers != null) {
        if (state == null)
            state = initState(jobId);
        int mapperIdx = 0;
        for (HadoopInputSplit split : mappers) {
            if (state.addMapper(split)) {
                if (log.isDebugEnabled())
                    log.debug("Submitting MAP task for execution [locNodeId=" + locNodeId + ", split=" + split + ']');
                HadoopTaskInfo taskInfo = new HadoopTaskInfo(MAP, jobId, meta.taskNumber(split), 0, split);
                taskInfo.mapperIndex(mapperIdx++);
                if (tasks == null)
                    tasks = new ArrayList<>();
                tasks.add(taskInfo);
            }
        }
    }
    return tasks;
}