Examples with HadoopFlowProcess cascading.flow.hadoop.HadoopFlowProcess used on opensource projects

Example 1 with HadoopFlowProcess

use of cascading.flow.hadoop.HadoopFlowProcess in project elephant-bird by twitter.

the class TestCombinedSequenceFile method testHadoopConf.

@Test
public void testHadoopConf() {
    CombinedSequenceFile csfScheme = new CombinedSequenceFile(Fields.ALL);
    JobConf conf = new JobConf();
    FlowProcess fp = new HadoopFlowProcess();
    Tap<JobConf, RecordReader, OutputCollector> tap = new TempHfs(conf, "test", CombinedSequenceFile.class, false);
    csfScheme.sourceConfInit(fp, tap, conf);
    assertEquals("MapReduceInputFormatWrapper shold wrap mapred.SequenceFileinputFormat", "org.apache.hadoop.mapred.SequenceFileInputFormat", conf.get(MapReduceInputFormatWrapper.CLASS_CONF_KEY));
    assertEquals("Delegate combiner should wrap MapReduceInputFormatWrapper", "com.twitter.elephantbird.mapreduce.input.MapReduceInputFormatWrapper", conf.get(DelegateCombineFileInputFormat.COMBINED_INPUT_FORMAT_DELEGATE));
    assertEquals("DeprecatedInputFormatWrapper should wrap Delegate combiner", "com.twitter.elephantbird.mapreduce.input.combine.DelegateCombineFileInputFormat", conf.get(DeprecatedInputFormatWrapper.CLASS_CONF_KEY));
}

Example 2 with HadoopFlowProcess

use of cascading.flow.hadoop.HadoopFlowProcess in project elephant-bird by twitter.

the class TestCombinedSequenceFile method testHadoopConf.

@Test
public void testHadoopConf() {
    CombinedSequenceFile csfScheme = new CombinedSequenceFile(Fields.ALL);
    JobConf conf = new JobConf();
    FlowProcess fp = new HadoopFlowProcess();
    Tap<Configuration, RecordReader, OutputCollector> tap = new TempHfs(conf, "test", CombinedSequenceFile.class, false);
    csfScheme.sourceConfInit(fp, tap, conf);
    assertEquals("MapReduceInputFormatWrapper shold wrap mapred.SequenceFileinputFormat", "org.apache.hadoop.mapred.SequenceFileInputFormat", conf.get(MapReduceInputFormatWrapper.CLASS_CONF_KEY));
    assertEquals("Delegate combiner should wrap MapReduceInputFormatWrapper", "com.twitter.elephantbird.mapreduce.input.MapReduceInputFormatWrapper", conf.get(DelegateCombineFileInputFormat.COMBINED_INPUT_FORMAT_DELEGATE));
    assertEquals("DeprecatedInputFormatWrapper should wrap Delegate combiner", "com.twitter.elephantbird.mapreduce.input.combine.DelegateCombineFileInputFormat", conf.get(DeprecatedInputFormatWrapper.CLASS_CONF_KEY));
}