Examples with JobConf org.apache.hadoop.mapred.JobConf used on opensource projects

Example 31 with JobConf

use of org.apache.hadoop.mapred.JobConf in project hadoop by apache.

the class TestMergeManager method testMemoryMerge.

@Test(timeout = 10000)
public void testMemoryMerge() throws Exception {
    final int TOTAL_MEM_BYTES = 10000;
    final int OUTPUT_SIZE = 7950;
    JobConf conf = new JobConf();
    conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f);
    conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, TOTAL_MEM_BYTES);
    conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.8f);
    conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.9f);
    TestExceptionReporter reporter = new TestExceptionReporter();
    CyclicBarrier mergeStart = new CyclicBarrier(2);
    CyclicBarrier mergeComplete = new CyclicBarrier(2);
    StubbedMergeManager mgr = new StubbedMergeManager(conf, reporter, mergeStart, mergeComplete);
    // reserve enough map output to cause a merge when it is committed
    MapOutput<Text, Text> out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertTrue("Should be a memory merge", (out1 instanceof InMemoryMapOutput));
    InMemoryMapOutput<Text, Text> mout1 = (InMemoryMapOutput<Text, Text>) out1;
    fillOutput(mout1);
    MapOutput<Text, Text> out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertTrue("Should be a memory merge", (out2 instanceof InMemoryMapOutput));
    InMemoryMapOutput<Text, Text> mout2 = (InMemoryMapOutput<Text, Text>) out2;
    fillOutput(mout2);
    // next reservation should be a WAIT
    MapOutput<Text, Text> out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertEquals("Should be told to wait", null, out3);
    // trigger the first merge and wait for merge thread to start merging
    // and free enough output to reserve more
    mout1.commit();
    mout2.commit();
    mergeStart.await();
    Assert.assertEquals(1, mgr.getNumMerges());
    // reserve enough map output to cause another merge when committed
    out1 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertTrue("Should be a memory merge", (out1 instanceof InMemoryMapOutput));
    mout1 = (InMemoryMapOutput<Text, Text>) out1;
    fillOutput(mout1);
    out2 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertTrue("Should be a memory merge", (out2 instanceof InMemoryMapOutput));
    mout2 = (InMemoryMapOutput<Text, Text>) out2;
    fillOutput(mout2);
    // next reservation should be null
    out3 = mgr.reserve(null, OUTPUT_SIZE, 0);
    Assert.assertEquals("Should be told to wait", null, out3);
    // commit output *before* merge thread completes
    mout1.commit();
    mout2.commit();
    // allow the first merge to complete
    mergeComplete.await();
    // start the second merge and verify
    mergeStart.await();
    Assert.assertEquals(2, mgr.getNumMerges());
    // trigger the end of the second merge
    mergeComplete.await();
    Assert.assertEquals(2, mgr.getNumMerges());
    Assert.assertEquals("exception reporter invoked", 0, reporter.getNumExceptions());
}

Example 32 with JobConf

use of org.apache.hadoop.mapred.JobConf in project hadoop by apache.

the class TestMergeManager method testZeroShuffleMemoryLimitPercent.

@Test
public void testZeroShuffleMemoryLimitPercent() throws Exception {
    final JobConf jobConf = new JobConf();
    jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f);
    final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<>(null, jobConf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles());
    verifyReservedMapOutputType(mgr, 10L, "DISK");
}

Example 33 with JobConf

use of org.apache.hadoop.mapred.JobConf in project hadoop by apache.

the class TestMergeManager method testIoSortDefaults.

@Test
public void testIoSortDefaults() {
    final JobConf jobConf = new JobConf();
    assertEquals(10, jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100));
    assertEquals(100, jobConf.getInt(MRJobConfig.IO_SORT_MB, 10));
}

Example 34 with JobConf

use of org.apache.hadoop.mapred.JobConf in project hadoop by apache.

the class TestShuffleScheduler method TestSucceedAndFailedCopyMap.

@SuppressWarnings("rawtypes")
@Test
public <K, V> void TestSucceedAndFailedCopyMap() throws Exception {
    JobConf job = new JobConf();
    job.setNumMapTasks(2);
    //mock creation
    TaskUmbilicalProtocol mockUmbilical = mock(TaskUmbilicalProtocol.class);
    Reporter mockReporter = mock(Reporter.class);
    FileSystem mockFileSystem = mock(FileSystem.class);
    Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass = job.getCombinerClass();
    // needed for mock with generic
    @SuppressWarnings("unchecked") CombineOutputCollector<K, V> mockCombineOutputCollector = (CombineOutputCollector<K, V>) mock(CombineOutputCollector.class);
    org.apache.hadoop.mapreduce.TaskAttemptID mockTaskAttemptID = mock(org.apache.hadoop.mapreduce.TaskAttemptID.class);
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);
    @SuppressWarnings("unchecked") MapOutput<K, V> output = mock(MapOutput.class);
    ShuffleConsumerPlugin.Context<K, V> context = new ShuffleConsumerPlugin.Context<K, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
    TaskStatus status = new TaskStatus() {

        @Override
        public boolean getIsMap() {
            return false;
        }

        @Override
        public void addFetchFailedMap(TaskAttemptID mapTaskId) {
        }
    };
    Progress progress = new Progress();
    ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null, null, progress, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter());
    MapHost host1 = new MapHost("host1", null);
    TaskAttemptID failedAttemptID = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 0), 0);
    TaskAttemptID succeedAttemptID = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 1), 1);
    // handle output fetch failure for failedAttemptID, part I
    scheduler.hostFailed(host1.getHostName());
    // handle output fetch succeed for succeedAttemptID
    long bytes = (long) 500 * 1024 * 1024;
    scheduler.copySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
    // handle output fetch failure for failedAttemptID, part II
    // for MAPREDUCE-6361: verify no NPE exception get thrown out
    scheduler.copyFailed(failedAttemptID, host1, true, false);
}

Example 35 with JobConf

use of org.apache.hadoop.mapred.JobConf in project hadoop by apache.

the class HSAdmin method addSecurityConfiguration.

private Configuration addSecurityConfiguration(Configuration conf) {
    conf = new JobConf(conf);
    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(JHAdminConfig.MR_HISTORY_PRINCIPAL, ""));
    return conf;
}