use of org.apache.tez.dag.api.OutputDescriptor in project tez by apache.
the class TestMROutputLegacy method testOldAPI_MR.
// simulate the behavior of translating MR to DAG using MR old API
@Test(timeout = 5000)
public void testOldAPI_MR() throws Exception {
String outputPath = "/tmp/output";
JobConf conf = new JobConf();
conf.setOutputKeyClass(NullWritable.class);
conf.setOutputValueClass(Text.class);
conf.setOutputFormat(org.apache.hadoop.mapred.SequenceFileOutputFormat.class);
org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(conf, new Path(outputPath));
// the output is attached to reducer
conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(conf);
OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName()).setUserPayload(vertexPayload);
DataSinkDescriptor sink = DataSinkDescriptor.create(od, OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null);
OutputContext outputContext = createMockOutputContext(sink.getOutputDescriptor().getUserPayload());
MROutputLegacy output = new MROutputLegacy(outputContext, 2);
output.initialize();
assertEquals(false, output.useNewApi);
assertEquals(org.apache.hadoop.mapred.SequenceFileOutputFormat.class, output.oldOutputFormat.getClass());
assertNull(output.newOutputFormat);
assertEquals(NullWritable.class, output.oldApiTaskAttemptContext.getOutputKeyClass());
assertEquals(Text.class, output.oldApiTaskAttemptContext.getOutputValueClass());
assertNull(output.newApiTaskAttemptContext);
assertNotNull(output.oldRecordWriter);
assertNull(output.newRecordWriter);
assertEquals(org.apache.hadoop.mapred.FileOutputCommitter.class, output.committer.getClass());
}
use of org.apache.tez.dag.api.OutputDescriptor in project tez by apache.
the class TestMROutputLegacy method testOldAPI_MapperOnly.
// simulate the behavior of translating Mapper-only job to DAG using MR old API
@Test(timeout = 5000)
public void testOldAPI_MapperOnly() throws Exception {
String outputPath = "/tmp/output";
JobConf conf = new JobConf();
conf.setOutputKeyClass(NullWritable.class);
conf.setOutputValueClass(Text.class);
conf.setOutputFormat(org.apache.hadoop.mapred.SequenceFileOutputFormat.class);
org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(conf, new Path(outputPath));
// the output is attached to mapper
conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(conf);
OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName()).setUserPayload(vertexPayload);
DataSinkDescriptor sink = DataSinkDescriptor.create(od, OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null);
OutputContext outputContext = createMockOutputContext(sink.getOutputDescriptor().getUserPayload());
MROutputLegacy output = new MROutputLegacy(outputContext, 2);
output.initialize();
assertEquals(false, output.useNewApi);
assertEquals(org.apache.hadoop.mapred.SequenceFileOutputFormat.class, output.oldOutputFormat.getClass());
assertNull(output.newOutputFormat);
assertEquals(NullWritable.class, output.oldApiTaskAttemptContext.getOutputKeyClass());
assertEquals(Text.class, output.oldApiTaskAttemptContext.getOutputValueClass());
assertNull(output.newApiTaskAttemptContext);
assertNotNull(output.oldRecordWriter);
assertNull(output.newRecordWriter);
assertEquals(org.apache.hadoop.mapred.FileOutputCommitter.class, output.committer.getClass());
}
Aggregations