Examples with HadoopJobId org.apache.ignite.internal.processors.hadoop.HadoopJobId used on opensource projects

Example 36 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopExternalTaskExecutionSelfTest method testMapperException.

/**
     * @throws Exception If failed.
     */
public void testMapperException() throws Exception {
    String testInputFile = "/test";
    prepareTestFile(testInputFile);
    Configuration cfg = new Configuration();
    setupFileSystems(cfg);
    Job job = Job.getInstance(cfg);
    job.setMapperClass(TestFailingMapper.class);
    job.setCombinerClass(TestReducer.class);
    job.setReducerClass(TestReducer.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(1);
    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestIgniteInstanceName(0) + "@/" + testInputFile));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestIgniteInstanceName(0) + "@/output"));
    job.setJarByClass(getClass());
    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1), createJobInfo(job.getConfiguration()));
    try {
        fut.get();
    } catch (IgniteCheckedException e) {
        IOException exp = X.cause(e, IOException.class);
        assertNotNull(exp);
        assertEquals("Test failure", exp.getMessage());
    }
}

Example 37 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopShuffleMessage method readExternal.

/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobId = new HadoopJobId();
    jobId.readExternal(in);
    msgId = in.readLong();
    reducer = in.readInt();
    off = in.readInt();
    buf = U.readByteArray(in);
}

Example 38 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopDirectShuffleMessage method readExternal.

/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobId = new HadoopJobId();
    jobId.readExternal(in);
    reducer = in.readInt();
    cnt = in.readInt();
    buf = U.readByteArray(in);
    bufLen = buf != null ? buf.length : 0;
    dataLen = in.readInt();
}

Example 39 with HadoopJobId

use of org.apache.ignite.internal.processors.hadoop.HadoopJobId in project ignite by apache.

the class HadoopShuffleAck method readExternal.

/** {@inheritDoc} */
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobId = new HadoopJobId();
    jobId.readExternal(in);
    msgId = in.readLong();
}