Examples with DataSinkDescriptor org.apache.tez.dag.api.DataSinkDescriptor used on opensource projects

Example 16 with DataSinkDescriptor

use of org.apache.tez.dag.api.DataSinkDescriptor in project tez by apache.

the class TezClientUtils method setupDAGCredentials.

/**
 * Obtains tokens for the DAG based on the list of URIs setup in the DAG. The
 * fetched credentials are populated back into the DAG and can be retrieved
 * via dag.getCredentials
 *
 * @param dag
 *          the dag for which credentials need to be setup
 * @param sessionCredentials
 *          session credentials which have already been obtained, and will be
 *          required for the DAG
 * @param conf
 * @throws IOException
 */
@Private
static Credentials setupDAGCredentials(DAG dag, Credentials sessionCredentials, Configuration conf) throws IOException {
    Preconditions.checkNotNull(sessionCredentials);
    TezCommonUtils.logCredentials(LOG, sessionCredentials, "session");
    Credentials dagCredentials = new Credentials();
    // All session creds are required for the DAG.
    dagCredentials.mergeAll(sessionCredentials);
    // Add additional credentials based on any URIs that the user may have specified.
    // Obtain Credentials for any paths that the user may have configured.
    addFileSystemCredentialsFromURIs(dag.getURIsForCredentials(), dagCredentials, conf);
    // Obtain Credentials for the local resources configured on the DAG
    try {
        Set<Path> lrPaths = new HashSet<Path>();
        for (Vertex v : dag.getVertices()) {
            for (LocalResource lr : v.getTaskLocalFiles().values()) {
                lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource()));
            }
            List<DataSourceDescriptor> dataSources = v.getDataSources();
            for (DataSourceDescriptor dataSource : dataSources) {
                addFileSystemCredentialsFromURIs(dataSource.getURIsForCredentials(), dagCredentials, conf);
            }
            List<DataSinkDescriptor> dataSinks = v.getDataSinks();
            for (DataSinkDescriptor dataSink : dataSinks) {
                addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), dagCredentials, conf);
            }
        }
        for (LocalResource lr : dag.getTaskLocalFiles().values()) {
            lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource()));
        }
        Path[] paths = lrPaths.toArray(new Path[lrPaths.size()]);
        TokenCache.obtainTokensForFileSystems(dagCredentials, paths, conf);
    } catch (URISyntaxException e) {
        throw new IOException(e);
    }
    return dagCredentials;
}

Example 17 with DataSinkDescriptor

use of org.apache.tez.dag.api.DataSinkDescriptor in project tez by apache.

the class TestMultiMROutput method createMROutputs.

private MultiMROutput createMROutputs(Class outputFormat, boolean isMapper, boolean useLazyOutputFormat) throws InterruptedException, IOException {
    String outputPath = "/tmp/output";
    JobConf conf = new JobConf();
    conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, isMapper);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);
    DataSinkDescriptor dataSink = MultiMROutput.createConfigBuilder(conf, outputFormat, outputPath, useLazyOutputFormat).build();
    OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload());
    MultiMROutput output = new MultiMROutput(outputContext, 2);
    output.initialize();
    return output;
}

Example 18 with DataSinkDescriptor

use of org.apache.tez.dag.api.DataSinkDescriptor in project tez by apache.

the class TestMROutput method testNewAPI_SequenceFileOutputFormat.

@Test(timeout = 5000)
public void testNewAPI_SequenceFileOutputFormat() throws Exception {
    String outputPath = "/tmp/output";
    JobConf conf = new JobConf();
    conf.setOutputKeyClass(NullWritable.class);
    conf.setOutputValueClass(Text.class);
    DataSinkDescriptor dataSink = MROutput.createConfigBuilder(conf, SequenceFileOutputFormat.class, outputPath).build();
    OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload());
    MROutput output = new MROutput(outputContext, 2);
    output.initialize();
    assertEquals(true, output.useNewApi);
    assertEquals(SequenceFileOutputFormat.class, output.newOutputFormat.getClass());
    assertNull(output.oldOutputFormat);
    assertEquals(NullWritable.class, output.newApiTaskAttemptContext.getOutputKeyClass());
    assertEquals(Text.class, output.newApiTaskAttemptContext.getOutputValueClass());
    assertNull(output.oldApiTaskAttemptContext);
    assertNotNull(output.newRecordWriter);
    assertNull(output.oldRecordWriter);
    assertEquals(FileOutputCommitter.class, output.committer.getClass());
}

Example 19 with DataSinkDescriptor

use of org.apache.tez.dag.api.DataSinkDescriptor in project tez by apache.

the class TestMROutput method testOldAPI_SequenceFileOutputFormat.

@Test(timeout = 5000)
public void testOldAPI_SequenceFileOutputFormat() throws Exception {
    String outputPath = "/tmp/output";
    JobConf conf = new JobConf();
    conf.setOutputKeyClass(NullWritable.class);
    conf.setOutputValueClass(Text.class);
    DataSinkDescriptor dataSink = MROutput.createConfigBuilder(conf, org.apache.hadoop.mapred.SequenceFileOutputFormat.class, outputPath).build();
    OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload());
    MROutput output = new MROutput(outputContext, 2);
    output.initialize();
    assertEquals(false, output.useNewApi);
    assertEquals(org.apache.hadoop.mapred.SequenceFileOutputFormat.class, output.oldOutputFormat.getClass());
    assertNull(output.newOutputFormat);
    assertEquals(NullWritable.class, output.oldApiTaskAttemptContext.getOutputKeyClass());
    assertEquals(Text.class, output.oldApiTaskAttemptContext.getOutputValueClass());
    assertNull(output.newApiTaskAttemptContext);
    assertNotNull(output.oldRecordWriter);
    assertNull(output.newRecordWriter);
    assertEquals(org.apache.hadoop.mapred.FileOutputCommitter.class, output.committer.getClass());
}

Example 20 with DataSinkDescriptor

use of org.apache.tez.dag.api.DataSinkDescriptor in project tez by apache.

the class TestMROutput method testNewAPI_TextOutputFormat.

@Test(timeout = 5000)
public void testNewAPI_TextOutputFormat() throws Exception {
    String outputPath = "/tmp/output";
    Configuration conf = new Configuration();
    conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
    DataSinkDescriptor dataSink = MROutput.createConfigBuilder(conf, TextOutputFormat.class, outputPath).build();
    OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload());
    MROutput output = new MROutput(outputContext, 2);
    output.initialize();
    assertEquals(true, output.isMapperOutput);
    assertEquals(true, output.useNewApi);
    assertEquals(TextOutputFormat.class, output.newOutputFormat.getClass());
    assertNull(output.oldOutputFormat);
    assertNotNull(output.newApiTaskAttemptContext);
    assertNull(output.oldApiTaskAttemptContext);
    assertNotNull(output.newRecordWriter);
    assertNull(output.oldRecordWriter);
    assertEquals(FileOutputCommitter.class, output.committer.getClass());
}