Examples with LimitingInputFormatProvider io.cdap.cdap.etl.batch.preview.LimitingInputFormatProvider used on opensource projects

Example 1 with LimitingInputFormatProvider

use of io.cdap.cdap.etl.batch.preview.LimitingInputFormatProvider in project cdap by caskdata.

the class MapReduceBatchContext method setInput.

@Override
public void setInput(Input input) {
    Input wrapped = CALLER.callUnchecked(() -> {
        Input trackableInput = input;
        if (isPreviewEnabled && input instanceof Input.InputFormatProviderInput) {
            InputFormatProvider inputFormatProvider = ((Input.InputFormatProviderInput) input).getInputFormatProvider();
            LimitingInputFormatProvider wrapper = new LimitingInputFormatProvider(inputFormatProvider, getMaxPreviewRecords());
            trackableInput = Input.of(input.getName(), wrapper).alias(input.getAlias());
        }
        trackableInput = ExternalDatasets.makeTrackable(mrContext.getAdmin(), suffixInput(trackableInput));
        mrContext.addInput(trackableInput);
        return trackableInput;
    });
    inputNames.add(wrapped.getAlias());
}

Example 2 with LimitingInputFormatProvider

use of io.cdap.cdap.etl.batch.preview.LimitingInputFormatProvider in project cdap by caskdata.

the class SparkBatchSourceContext method setInput.

@Override
public void setInput(Input input) {
    Input trackableInput = input;
    // Wrap the input provider with tracking counter for metrics collection via MR counter.
    if (trackableInput instanceof Input.InputFormatProviderInput) {
        InputFormatProvider provider = ((Input.InputFormatProviderInput) trackableInput).getInputFormatProvider();
        Map<String, String> conf = new HashMap<>(provider.getInputFormatConfiguration());
        conf.put(TrackingInputFormat.DELEGATE_CLASS_NAME, provider.getInputFormatClassName());
        provider = new BasicInputFormatProvider(TrackingInputFormat.class.getName(), conf);
        trackableInput = Input.of(trackableInput.getName(), provider).alias(trackableInput.getAlias());
    }
    // Limit preview input by wrapping the input
    if (isPreviewEnabled && trackableInput instanceof Input.InputFormatProviderInput) {
        InputFormatProvider inputFormatProvider = ((Input.InputFormatProviderInput) trackableInput).getInputFormatProvider();
        LimitingInputFormatProvider wrapper = new LimitingInputFormatProvider(inputFormatProvider, getMaxPreviewRecords());
        trackableInput = Input.of(trackableInput.getName(), wrapper).alias(trackableInput.getAlias());
    }
    trackableInput = ExternalDatasets.makeTrackable(admin, suffixInput(trackableInput));
    sourceFactory.addInput(getStageName(), trackableInput);
}