Examples with DatasetOutputCommitter io.cdap.cdap.api.data.batch.DatasetOutputCommitter used on opensource projects

Example 1 with DatasetOutputCommitter

use of io.cdap.cdap.api.data.batch.DatasetOutputCommitter in project cdap by caskdata.

the class BasicMapReduceContext method addOutput.

@Override
public void addOutput(Output output) {
    if (output.getNamespace() != null && output.getNamespace().equals(NamespaceId.SYSTEM.getNamespace()) && !getProgram().getNamespaceId().equals(NamespaceId.SYSTEM.getNamespace())) {
        // trying to access system namespace from a program outside system namespace is not allowed
        throw new IllegalArgumentException(String.format("Accessing Output %s in system namespace " + "is not allowed from the namespace %s", output.getName(), getProgram().getNamespaceId()));
    }
    String alias = output.getAlias();
    if (this.outputs.containsKey(alias)) {
        throw new IllegalArgumentException("Output already configured: " + alias);
    }
    ProvidedOutput providedOutput;
    if (output instanceof Output.DatasetOutput) {
        providedOutput = Outputs.transform((Output.DatasetOutput) output, this);
    } else if (output instanceof Output.OutputFormatProviderOutput) {
        OutputFormatProvider outputFormatProvider = ((Output.OutputFormatProviderOutput) output).getOutputFormatProvider();
        if (outputFormatProvider instanceof DatasetOutputCommitter) {
            // be able to call its methods in MainOutputCommitter. It needs to be a DatasetOutput.
            throw new IllegalArgumentException("Cannot add a DatasetOutputCommitter as an OutputFormatProviderOutput. " + "Add the output as a DatasetOutput.");
        }
        providedOutput = new ProvidedOutput(output, outputFormatProvider);
    } else if (output.getClass().getCanonicalName().startsWith(CDAP_PACKAGE_PREFIX)) {
        // Skip unsupported outputs from within CDAP packages.
        // This is used to ignore unsupported outputs in MapReduce (such as the SQL Engine Output for Spark).
        LOG.info("Unsupported output in MapReduce: {}", output.getClass().getCanonicalName());
        return;
    } else {
        // shouldn't happen unless user defines their own Output class
        throw new IllegalArgumentException(String.format("Output %s has unknown output class %s", output.getName(), output.getClass().getCanonicalName()));
    }
    this.outputs.put(alias, providedOutput);
}

Example 2 with DatasetOutputCommitter

use of io.cdap.cdap.api.data.batch.DatasetOutputCommitter in project cdap by caskdata.

the class MainOutputCommitter method failOutputs.

/**
 * Calls onFailure for all of the DatasetOutputCommitters. If any operation throws an exception, it is suppressed
 * until onFailure is called on all of the DatasetOutputCommitters.
 */
private void failOutputs(Map<String, DatasetOutputCommitter> datasetOutputCommiters) throws Exception {
    if (completedCallingOnFailure) {
        // guard against multiple calls to OutputCommitter#abortJob
        LOG.info("Not calling onFailure on outputs, as it has they have already been executed.");
        return;
    }
    Exception e = null;
    for (Map.Entry<String, DatasetOutputCommitter> datasetOutputCommitter : datasetOutputCommiters.entrySet()) {
        try {
            datasetOutputCommitter.getValue().onFailure();
        } catch (Exception suppressedException) {
            LOG.error(String.format("Error from onFailure method of output dataset %s.", datasetOutputCommitter.getKey()), suppressedException);
            if (e != null) {
                e.addSuppressed(suppressedException);
            } else {
                e = suppressedException;
            }
        }
    }
    completedCallingOnFailure = true;
    if (e != null) {
        throw e;
    }
}