use of org.apache.beam.runners.direct.DirectOptions in project components by Talend.
the class SimpleFileIODatasetRuntime method getSample.
@Override
public void getSample(int limit, Consumer<IndexedRecord> consumer) {
// Create an input runtime based on the properties.
SimpleFileIOInputRuntime inputRuntime = new SimpleFileIOInputRuntime();
SimpleFileIOInputProperties inputProperties = new SimpleFileIOInputProperties(null);
inputProperties.limit.setValue(limit);
inputProperties.init();
inputProperties.setDatasetProperties(properties);
inputRuntime.initialize(null, inputProperties);
// Create a pipeline using the input component to get records.
DirectOptions options = BeamLocalRunnerOption.getOptions();
final Pipeline p = Pipeline.create(options);
try (DirectConsumerCollector<IndexedRecord> collector = DirectConsumerCollector.of(consumer)) {
// Collect a sample of the input records.
//
p.apply(inputRuntime).apply(//
Sample.<IndexedRecord>any(limit)).apply(collector);
try {
p.run().waitUntilFinish();
} catch (Pipeline.PipelineExecutionException e) {
if (e.getCause() instanceof TalendRuntimeException)
throw (TalendRuntimeException) e.getCause();
throw e;
}
}
}
Aggregations