Examples with TaskReporter org.apache.hadoop.mapred.Task.TaskReporter used on opensource projects

Example 1 with TaskReporter

use of org.apache.hadoop.mapred.Task.TaskReporter in project hadoop by apache.

the class TestCombineOutputCollector method testDefaultCollect.

@Test
public void testDefaultCollect() throws Throwable {
    //mock creation
    TaskReporter mockTaskReporter = mock(TaskReporter.class);
    @SuppressWarnings("unchecked") Writer<String, Integer> mockWriter = mock(Writer.class);
    Configuration conf = new Configuration();
    coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
    coc.setWriter(mockWriter);
    verify(mockTaskReporter, never()).progress();
    for (int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
        coc.collect("dummy", i);
    }
    verify(mockTaskReporter, times(1)).progress();
    for (int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
        coc.collect("dummy", i);
    }
    verify(mockTaskReporter, times(2)).progress();
}

Example 2 with TaskReporter

use of org.apache.hadoop.mapred.Task.TaskReporter in project hadoop by apache.

the class TestCombineOutputCollector method testCustomCollect.

@Test
public void testCustomCollect() throws Throwable {
    //mock creation
    TaskReporter mockTaskReporter = mock(TaskReporter.class);
    @SuppressWarnings("unchecked") Writer<String, Integer> mockWriter = mock(Writer.class);
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.COMBINE_RECORDS_BEFORE_PROGRESS, "2");
    coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
    coc.setWriter(mockWriter);
    verify(mockTaskReporter, never()).progress();
    coc.collect("dummy", 1);
    verify(mockTaskReporter, never()).progress();
    coc.collect("dummy", 2);
    verify(mockTaskReporter, times(1)).progress();
}

Example 3 with TaskReporter

use of org.apache.hadoop.mapred.Task.TaskReporter in project hadoop by apache.

the class TestCombineFileRecordReader method testProgressIsReportedIfInputASeriesOfEmptyFiles.

@SuppressWarnings("unchecked")
@Test
public void testProgressIsReportedIfInputASeriesOfEmptyFiles() throws IOException, InterruptedException {
    JobConf conf = new JobConf();
    Path[] paths = new Path[3];
    File[] files = new File[3];
    long[] fileLength = new long[3];
    try {
        for (int i = 0; i < 3; i++) {
            File dir = new File(outDir.toString());
            dir.mkdir();
            files[i] = new File(dir, "testfile" + i);
            FileWriter fileWriter = new FileWriter(files[i]);
            fileWriter.flush();
            fileWriter.close();
            fileLength[i] = i;
            paths[i] = new Path(outDir + "/testfile" + i);
        }
        CombineFileSplit combineFileSplit = new CombineFileSplit(paths, fileLength);
        TaskAttemptID taskAttemptID = Mockito.mock(TaskAttemptID.class);
        TaskReporter reporter = Mockito.mock(TaskReporter.class);
        TaskAttemptContextImpl taskAttemptContext = new TaskAttemptContextImpl(conf, taskAttemptID, reporter);
        CombineFileRecordReader cfrr = new CombineFileRecordReader(combineFileSplit, taskAttemptContext, TextRecordReaderWrapper.class);
        cfrr.initialize(combineFileSplit, taskAttemptContext);
        verify(reporter).progress();
        Assert.assertFalse(cfrr.nextKeyValue());
        verify(reporter, times(3)).progress();
    } finally {
        FileUtil.fullyDelete(new File(outDir.toString()));
    }
}