Examples with Reporter org.apache.hadoop.mapred.Reporter used on opensource projects

Example 1 with Reporter

use of org.apache.hadoop.mapred.Reporter in project hadoop by apache.

the class PipeMapRed method waitOutputThreads.

void waitOutputThreads() throws IOException {
    try {
        if (outThread_ == null) {
            // This happens only when reducer has empty input(So reduce() is not
            // called at all in this task). If reducer still generates output,
            // which is very uncommon and we may not have to support this case.
            // So we don't write this output to HDFS, but we consume/collect
            // this output just to avoid reducer hanging forever.
            OutputCollector collector = new OutputCollector() {

                public void collect(Object key, Object value) throws IOException {
                //just consume it, no need to write the record anywhere
                }
            };
            //dummy reporter
            Reporter reporter = Reporter.NULL;
            startOutputThreads(collector, reporter);
        }
        int exitVal = sim.waitFor();
        // how'd it go?
        if (exitVal != 0) {
            if (nonZeroExitIsFailure_) {
                throw new RuntimeException("PipeMapRed.waitOutputThreads(): subprocess failed with code " + exitVal);
            } else {
                LOG.info("PipeMapRed.waitOutputThreads(): subprocess exited with " + "code " + exitVal + " in " + PipeMapRed.class.getName());
            }
        }
        if (outThread_ != null) {
            outThread_.join(joinDelay_);
        }
        if (errThread_ != null) {
            errThread_.join(joinDelay_);
        }
        if (outerrThreadsThrowable != null) {
            throw new RuntimeException(outerrThreadsThrowable);
        }
    } catch (InterruptedException e) {
    //ignore
    }
}

Example 2 with Reporter

use of org.apache.hadoop.mapred.Reporter in project hadoop by apache.

the class TestShuffleScheduler method TestSucceedAndFailedCopyMap.

@SuppressWarnings("rawtypes")
@Test
public <K, V> void TestSucceedAndFailedCopyMap() throws Exception {
    JobConf job = new JobConf();
    job.setNumMapTasks(2);
    //mock creation
    TaskUmbilicalProtocol mockUmbilical = mock(TaskUmbilicalProtocol.class);
    Reporter mockReporter = mock(Reporter.class);
    FileSystem mockFileSystem = mock(FileSystem.class);
    Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass = job.getCombinerClass();
    // needed for mock with generic
    @SuppressWarnings("unchecked") CombineOutputCollector<K, V> mockCombineOutputCollector = (CombineOutputCollector<K, V>) mock(CombineOutputCollector.class);
    org.apache.hadoop.mapreduce.TaskAttemptID mockTaskAttemptID = mock(org.apache.hadoop.mapreduce.TaskAttemptID.class);
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);
    @SuppressWarnings("unchecked") MapOutput<K, V> output = mock(MapOutput.class);
    ShuffleConsumerPlugin.Context<K, V> context = new ShuffleConsumerPlugin.Context<K, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
    TaskStatus status = new TaskStatus() {

        @Override
        public boolean getIsMap() {
            return false;
        }

        @Override
        public void addFetchFailedMap(TaskAttemptID mapTaskId) {
        }
    };
    Progress progress = new Progress();
    ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null, null, progress, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter());
    MapHost host1 = new MapHost("host1", null);
    TaskAttemptID failedAttemptID = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 0), 0);
    TaskAttemptID succeedAttemptID = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 1), 1);
    // handle output fetch failure for failedAttemptID, part I
    scheduler.hostFailed(host1.getHostName());
    // handle output fetch succeed for succeedAttemptID
    long bytes = (long) 500 * 1024 * 1024;
    scheduler.copySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
    // handle output fetch failure for failedAttemptID, part II
    // for MAPREDUCE-6361: verify no NPE exception get thrown out
    scheduler.copyFailed(failedAttemptID, host1, true, false);
}

Example 3 with Reporter

use of org.apache.hadoop.mapred.Reporter in project hadoop by apache.

the class TestPipesNonJavaInputFormat method testFormat.

/**
   *  test PipesNonJavaInputFormat
    */
@Test
public void testFormat() throws IOException {
    PipesNonJavaInputFormat inputFormat = new PipesNonJavaInputFormat();
    JobConf conf = new JobConf();
    Reporter reporter = mock(Reporter.class);
    RecordReader<FloatWritable, NullWritable> reader = inputFormat.getRecordReader(new FakeSplit(), conf, reporter);
    assertEquals(0.0f, reader.getProgress(), 0.001);
    // input and output files
    File input1 = new File(workSpace + File.separator + "input1");
    if (!input1.getParentFile().exists()) {
        Assert.assertTrue(input1.getParentFile().mkdirs());
    }
    if (!input1.exists()) {
        Assert.assertTrue(input1.createNewFile());
    }
    File input2 = new File(workSpace + File.separator + "input2");
    if (!input2.exists()) {
        Assert.assertTrue(input2.createNewFile());
    }
    // set data for splits
    conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR, StringUtils.escapeString(input1.getAbsolutePath()) + "," + StringUtils.escapeString(input2.getAbsolutePath()));
    InputSplit[] splits = inputFormat.getSplits(conf, 2);
    assertEquals(2, splits.length);
    PipesNonJavaInputFormat.PipesDummyRecordReader dummyRecordReader = new PipesNonJavaInputFormat.PipesDummyRecordReader(conf, splits[0]);
    // empty dummyRecordReader
    assertNull(dummyRecordReader.createKey());
    assertNull(dummyRecordReader.createValue());
    assertEquals(0, dummyRecordReader.getPos());
    assertEquals(0.0, dummyRecordReader.getProgress(), 0.001);
    // test method next
    assertTrue(dummyRecordReader.next(new FloatWritable(2.0f), NullWritable.get()));
    assertEquals(2.0, dummyRecordReader.getProgress(), 0.001);
    dummyRecordReader.close();
}

Example 4 with Reporter

use of org.apache.hadoop.mapred.Reporter in project hive by apache.

the class RCFileOutputFormat method getRecordWriter.

/** {@inheritDoc} */
@Override
public RecordWriter<WritableComparable, BytesRefArrayWritable> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
    Path outputPath = getWorkOutputPath(job);
    FileSystem fs = outputPath.getFileSystem(job);
    Path file = new Path(outputPath, name);
    CompressionCodec codec = null;
    if (getCompressOutput(job)) {
        Class<?> codecClass = getOutputCompressorClass(job, DefaultCodec.class);
        codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, job);
    }
    final RCFile.Writer out = new RCFile.Writer(fs, job, file, progress, codec);
    return new RecordWriter<WritableComparable, BytesRefArrayWritable>() {

        @Override
        public void close(Reporter reporter) throws IOException {
            out.close();
        }

        @Override
        public void write(WritableComparable key, BytesRefArrayWritable value) throws IOException {
            out.append(value);
        }
    };
}

Example 5 with Reporter

use of org.apache.hadoop.mapred.Reporter in project hbase by apache.

the class TestGroupingTableMap method shouldNotCallCollectonSinceFindUniqueKeyValueMoreThanOnes.

@Test
@SuppressWarnings({ "deprecation", "unchecked" })
public void shouldNotCallCollectonSinceFindUniqueKeyValueMoreThanOnes() throws Exception {
    GroupingTableMap gTableMap = null;
    try {
        Result result = mock(Result.class);
        Reporter reporter = mock(Reporter.class);
        gTableMap = new GroupingTableMap();
        Configuration cfg = new Configuration();
        cfg.set(GroupingTableMap.GROUP_COLUMNS, "familyA:qualifierA familyB:qualifierB");
        JobConf jobConf = new JobConf(cfg);
        gTableMap.configure(jobConf);
        byte[] row = {};
        List<Cell> keyValues = ImmutableList.<Cell>of(new KeyValue(row, "familyA".getBytes(), "qualifierA".getBytes(), Bytes.toBytes("1111")), new KeyValue(row, "familyA".getBytes(), "qualifierA".getBytes(), Bytes.toBytes("2222")), new KeyValue(row, "familyB".getBytes(), "qualifierB".getBytes(), Bytes.toBytes("3333")));
        when(result.listCells()).thenReturn(keyValues);
        OutputCollector<ImmutableBytesWritable, Result> outputCollectorMock = mock(OutputCollector.class);
        gTableMap.map(null, result, outputCollectorMock, reporter);
        verify(result).listCells();
        verifyZeroInteractions(outputCollectorMock);
    } finally {
        if (gTableMap != null)
            gTableMap.close();
    }
}