use of org.talend.components.simplefileio.input.SimpleFileIOInputProperties in project components by Talend.
the class SimpleFileIORoundTripRuntimeTest method testParquet.
/**
* Basic Parquet test.
*/
@Test
public void testParquet() throws IOException {
// The file that we will be creating.
RecordSet rs = getSimpleTestData(0);
String fileSpec = mini.getLocalFsNewFolder() + "output/";
// Configure the components.
SimpleFileIOOutputProperties outputProps = createOutputComponentProperties();
outputProps.getDatasetProperties().format.setValue(SimpleFileIOFormat.PARQUET);
outputProps.getDatasetProperties().path.setValue(fileSpec);
SimpleFileIOInputProperties inputProps = createInputComponentProperties();
inputProps.setDatasetProperties(outputProps.getDatasetProperties());
List<IndexedRecord> actual = runRoundTripPipelines(beam, rs.getAllData(), outputProps, inputProps);
// Generate the set of expected records. By default, CSV turns all columns into String and loses the original
// column name.
List<IndexedRecord> expected = rs.getAllData();
assertThat(actual, containsInAnyOrder(expected.toArray()));
// Verify that the file on the filesystem was correctly written.
// TODO(rskraba): verify independently from
// mini.assertReadFile(
// mini.getLocalFs(),
// fileSpec,
// rewriteRecordsAsCsvLines(expected, inputProps.getDatasetProperties().recordDelimiter.getValue(),
// inputProps.getDatasetProperties().fieldDelimiter.getValue()));
}
use of org.talend.components.simplefileio.input.SimpleFileIOInputProperties in project components by Talend.
the class SimpleFileIOInputRuntimeTest method testBasicAvro.
/**
* Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
*/
@Test
public void testBasicAvro() throws IOException, URISyntaxException {
RecordSet rs = getSimpleTestData(0);
writeRandomAvroFile(mini.getFs(), "/user/test/input.avro", rs);
String fileSpec = mini.getFs().getUri().resolve("/user/test/input.avro").toString();
// Configure the component.
SimpleFileIOInputProperties inputProps = createInputComponentProperties();
inputProps.getDatasetProperties().format.setValue(SimpleFileIOFormat.AVRO);
inputProps.getDatasetProperties().path.setValue(fileSpec);
// Create the runtime.
SimpleFileIOInputRuntime runtime = new SimpleFileIOInputRuntime();
runtime.initialize(null, inputProps);
// Use the runtime in a direct pipeline to test.
final Pipeline p = beam.createPipeline();
PCollection<IndexedRecord> readLines = p.apply(runtime);
// Check the expected values.
PAssert.that(readLines).containsInAnyOrder(rs.getAllData());
// And run the test.
p.run().waitUntilFinish();
}
use of org.talend.components.simplefileio.input.SimpleFileIOInputProperties in project components by Talend.
the class SimpleFileIOInputRuntimeTest method testBasicDefaults.
/**
* Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
*/
@Test
public void testBasicDefaults() throws IOException, URISyntaxException {
String inputFile = writeRandomCsvFile(mini.getFs(), "/user/test/input.csv", 0, 0, 10, 10, 6, ";", "\n");
String fileSpec = mini.getFs().getUri().resolve("/user/test/input.csv").toString();
// Configure the component.
SimpleFileIOInputProperties inputProps = createInputComponentProperties();
inputProps.getDatasetProperties().path.setValue(fileSpec);
// Create the runtime.
SimpleFileIOInputRuntime runtime = new SimpleFileIOInputRuntime();
runtime.initialize(null, inputProps);
// Use the runtime in a direct pipeline to test.
final Pipeline p = beam.createPipeline();
PCollection<IndexedRecord> readLines = p.apply(runtime);
// Check the expected values.
List<IndexedRecord> expected = new ArrayList<>();
for (String record : inputFile.split("\n")) {
expected.add(ConvertToIndexedRecord.convertToAvro(record.split(";")));
}
PAssert.that(readLines).containsInAnyOrder(expected);
// And run the test.
p.run().waitUntilFinish();
}
use of org.talend.components.simplefileio.input.SimpleFileIOInputProperties in project components by Talend.
the class SimpleFileIOInputRuntimeTest method createInputComponentProperties.
/**
* @return the properties for this component, fully initialized with the default values.
*/
public static SimpleFileIOInputProperties createInputComponentProperties() {
// Configure the component.
SimpleFileIOInputProperties inputProps = new SimpleFileIOInputProperties(null);
inputProps.init();
inputProps.setDatasetProperties(SimpleFileIODatasetRuntimeTest.createDatasetProperties());
return inputProps;
}
use of org.talend.components.simplefileio.input.SimpleFileIOInputProperties in project components by Talend.
the class SimpleFileIOInputRuntimeTest method testBasicCsvCustomDelimiters.
@Test
public void testBasicCsvCustomDelimiters() throws IOException, URISyntaxException {
String inputFile = writeRandomCsvFile(mini.getFs(), "/user/test/input.csv", 0, 0, 10, 10, 6, "|", "---");
String fileSpec = mini.getFs().getUri().resolve("/user/test/input.csv").toString();
// Configure the component.
SimpleFileIOInputProperties inputProps = createInputComponentProperties();
inputProps.getDatasetProperties().path.setValue(fileSpec);
inputProps.getDatasetProperties().recordDelimiter.setValue(RecordDelimiterType.OTHER);
inputProps.getDatasetProperties().specificRecordDelimiter.setValue("---");
inputProps.getDatasetProperties().fieldDelimiter.setValue(FieldDelimiterType.OTHER);
inputProps.getDatasetProperties().specificFieldDelimiter.setValue("|");
// Create the runtime.
SimpleFileIOInputRuntime runtime = new SimpleFileIOInputRuntime();
runtime.initialize(null, inputProps);
// Use the runtime in a direct pipeline to test.
// TODO(rskraba): This fails for certain values of targetParallelism! To fix.
final Pipeline p = beam.createPipeline(3);
PCollection<IndexedRecord> readLines = p.apply(runtime);
List<IndexedRecord> expected = new ArrayList<>();
for (String record : inputFile.split("---")) {
expected.add(ConvertToIndexedRecord.convertToAvro(record.split("\\Q|\\E")));
}
PAssert.that(readLines).containsInAnyOrder(expected);
p.run().waitUntilFinish();
}
Aggregations