Examples with ConnectorSourceEmitter co.cask.cdap.etl.batch.mapreduce.ConnectorSourceEmitter used on opensource projects

Example 1 with ConnectorSourceEmitter

use of co.cask.cdap.etl.batch.mapreduce.ConnectorSourceEmitter in project cdap by caskdata.

the class TransformExecutorFactory method addTransformation.

private void addTransformation(PipelinePhase pipeline, String stageName, Map<String, PipeTransformDetail> transformations, Map<String, ErrorOutputWriter<Object, Object>> transformErrorSinkMap) throws Exception {
    StageInfo stageInfo = pipeline.getStage(stageName);
    String pluginType = stageInfo.getPluginType();
    ErrorOutputWriter<Object, Object> errorOutputWriter = transformErrorSinkMap.containsKey(stageName) ? transformErrorSinkMap.get(stageName) : null;
    // If stageName is a connector source, it will have stageName along with record so use ConnectorSourceEmitter
    if (pipeline.getSources().contains(stageName) && pluginType.equals(Constants.CONNECTOR_TYPE)) {
        transformations.put(stageName, new PipeTransformDetail(stageName, true, false, getTransformation(stageInfo), new ConnectorSourceEmitter(stageName)));
    } else if (pluginType.equals(BatchJoiner.PLUGIN_TYPE) && isMapPhase) {
        // Do not remove stageName only for Map phase of BatchJoiner
        transformations.put(stageName, new PipeTransformDetail(stageName, false, false, getTransformation(stageInfo), new TransformEmitter(stageName, errorOutputWriter)));
    } else {
        boolean isErrorConsumer = ErrorTransform.PLUGIN_TYPE.equals(pluginType);
        transformations.put(stageName, new PipeTransformDetail(stageName, true, isErrorConsumer, getTransformation(stageInfo), new TransformEmitter(stageName, errorOutputWriter)));
    }
}