Examples with Output io.cdap.cdap.api.data.batch.Output used on opensource projects

Example 1 with Output

use of io.cdap.cdap.api.data.batch.Output in project cdap by caskdata.

the class MapReduceBatchContext method addOutput.

@Override
public void addOutput(Output output) {
    // Skip SQLEngineOutput as this is not supported in MapReduce.
    if (output instanceof SQLEngineOutput) {
        return;
    }
    Output actualOutput = suffixOutput(getOutput(output));
    Output trackableOutput = CALLER.callUnchecked(() -> {
        Output trackableOutput1 = isPreviewEnabled ? actualOutput : ExternalDatasets.makeTrackable(mrContext.getAdmin(), actualOutput);
        mrContext.addOutput(trackableOutput1);
        return trackableOutput1;
    });
    outputNames.add(trackableOutput.getAlias());
}

Example 2 with Output

use of io.cdap.cdap.api.data.batch.Output in project cdap by caskdata.

the class BasicMapReduceContext method addOutput.

@Override
public void addOutput(Output output) {
    if (output.getNamespace() != null && output.getNamespace().equals(NamespaceId.SYSTEM.getNamespace()) && !getProgram().getNamespaceId().equals(NamespaceId.SYSTEM.getNamespace())) {
        // trying to access system namespace from a program outside system namespace is not allowed
        throw new IllegalArgumentException(String.format("Accessing Output %s in system namespace " + "is not allowed from the namespace %s", output.getName(), getProgram().getNamespaceId()));
    }
    String alias = output.getAlias();
    if (this.outputs.containsKey(alias)) {
        throw new IllegalArgumentException("Output already configured: " + alias);
    }
    ProvidedOutput providedOutput;
    if (output instanceof Output.DatasetOutput) {
        providedOutput = Outputs.transform((Output.DatasetOutput) output, this);
    } else if (output instanceof Output.OutputFormatProviderOutput) {
        OutputFormatProvider outputFormatProvider = ((Output.OutputFormatProviderOutput) output).getOutputFormatProvider();
        if (outputFormatProvider instanceof DatasetOutputCommitter) {
            // be able to call its methods in MainOutputCommitter. It needs to be a DatasetOutput.
            throw new IllegalArgumentException("Cannot add a DatasetOutputCommitter as an OutputFormatProviderOutput. " + "Add the output as a DatasetOutput.");
        }
        providedOutput = new ProvidedOutput(output, outputFormatProvider);
    } else if (output.getClass().getCanonicalName().startsWith(CDAP_PACKAGE_PREFIX)) {
        // Skip unsupported outputs from within CDAP packages.
        // This is used to ignore unsupported outputs in MapReduce (such as the SQL Engine Output for Spark).
        LOG.info("Unsupported output in MapReduce: {}", output.getClass().getCanonicalName());
        return;
    } else {
        // shouldn't happen unless user defines their own Output class
        throw new IllegalArgumentException(String.format("Output %s has unknown output class %s", output.getName(), output.getClass().getCanonicalName()));
    }
    this.outputs.put(alias, providedOutput);
}

Example 3 with Output

use of io.cdap.cdap.api.data.batch.Output in project cdap by caskdata.

the class MockExternalSink method prepareRun.

@Override
public void prepareRun(BatchSinkContext context) {
    OutputFormatProvider outputFormatProvider = new Provider(config.dirName);
    if (config.name != null) {
        Output output = Output.of(config.name, outputFormatProvider);
        output.alias(config.alias);
        context.addOutput(output);
    } else {
        context.addOutput(Output.of(config.alias, outputFormatProvider));
    }
    if (config.name2 != null) {
        context.addOutput(Output.of(config.name2, new Provider(config.dirName2)).alias(config.alias2));
    } else if (config.alias2 != null) {
        context.addOutput(Output.of(config.alias2, new Provider(config.dirName2)));
    }
}

Example 4 with Output

use of io.cdap.cdap.api.data.batch.Output in project cdap by caskdata.

the class MapReduceContextConfig method setOutputs.

private void setOutputs(List<ProvidedOutput> providedOutputs) {
    // we only need to serialize the original Output objects, not the entire ProvidedOutput
    List<Output.DatasetOutput> datasetOutputs = new ArrayList<>();
    for (ProvidedOutput providedOutput : providedOutputs) {
        Output output = providedOutput.getOutput();
        if (output instanceof Output.DatasetOutput) {
            datasetOutputs.add((Output.DatasetOutput) output);
        }
    }
    hConf.set(HCONF_ATTR_OUTPUTS, GSON.toJson(datasetOutputs));
}

Example 5 with Output

use of io.cdap.cdap.api.data.batch.Output in project cdap by caskdata.

the class SparkBatchSinkContext method addOutput.

@Override
public void addOutput(Output output) {
    Output actualOutput = suffixOutput(getOutput(output));
    // Wrap the output provider with tracking counter for metrics collection via MR counter.
    if (actualOutput instanceof Output.OutputFormatProviderOutput) {
        OutputFormatProvider provider = ((Output.OutputFormatProviderOutput) actualOutput).getOutputFormatProvider();
        Map<String, String> conf = new HashMap<>(provider.getOutputFormatConfiguration());
        conf.put(TrackingOutputFormat.DELEGATE_CLASS_NAME, provider.getOutputFormatClassName());
        provider = new BasicOutputFormatProvider(TrackingOutputFormat.class.getName(), conf);
        actualOutput = Output.of(actualOutput.getName(), provider).alias(actualOutput.getAlias());
    }
    sinkFactory.addOutput(getStageName(), actualOutput);
}