Examples with SimpleFileIOOutputProperties org.talend.components.simplefileio.output.SimpleFileIOOutputProperties used on opensource projects

Example 11 with SimpleFileIOOutputProperties

use of org.talend.components.simplefileio.output.SimpleFileIOOutputProperties in project components by Talend.

the class SimpleFileIOOutputRuntimeTest method createOutputComponentProperties.

/**
 * @return the properties for this component, fully initialized with the default values.
 */
public static SimpleFileIOOutputProperties createOutputComponentProperties() {
    // Configure the component.
    SimpleFileIOOutputProperties outputProps = new SimpleFileIOOutputProperties(null);
    outputProps.init();
    outputProps.setDatasetProperties(SimpleFileIODatasetRuntimeTest.createDatasetProperties());
    return outputProps;
}

Example 12 with SimpleFileIOOutputProperties

use of org.talend.components.simplefileio.output.SimpleFileIOOutputProperties in project components by Talend.

the class GSRoundTripRuntimeTestIT method testCsv.

@Test
public void testCsv() {
    List<IndexedRecord> expected = new ArrayList<>();
    expected.add(ConvertToIndexedRecord.convertToAvro(new String[] { "1", "one" }));
    expected.add(ConvertToIndexedRecord.convertToAvro(new String[] { "2", "two" }));
    SimpleFileIOOutputProperties outputProps = createOutputProps();
    outputProps.getDatasetProperties().path.setValue(gsPath);
    SimpleFileIOOutputRuntime outputRuntime = new SimpleFileIOOutputRuntime();
    outputRuntime.initialize(null, outputProps);
    PCollection<IndexedRecord> input = writeP.apply(Create.of(expected));
    input.apply(outputRuntime);
    writeP.run(pipelineOptions).waitUntilFinish();
    SimpleFileIOInputProperties inputProps = createInputProps();
    inputProps.getDatasetProperties().path.setValue(gsPath + "*");
    SimpleFileIOInputRuntime inputRuntime = new SimpleFileIOInputRuntime();
    inputRuntime.initialize(null, inputProps);
    PCollection<IndexedRecord> readRecords = readP.apply(inputRuntime);
    PAssert.that(readRecords).containsInAnyOrder(expected);
    readP.run(pipelineOptions).waitUntilFinish();
}

Example 13 with SimpleFileIOOutputProperties

use of org.talend.components.simplefileio.output.SimpleFileIOOutputProperties in project components by Talend.

the class GSRoundTripRuntimeTestIT method createOutputProps.

private SimpleFileIOOutputProperties createOutputProps() {
    SimpleFileIOOutputProperties outputProps = new SimpleFileIOOutputProperties(null);
    outputProps.init();
    outputProps.setDatasetProperties(SimpleFileIODatasetRuntimeTest.createDatasetProperties());
    return outputProps;
}

Example 14 with SimpleFileIOOutputProperties

use of org.talend.components.simplefileio.output.SimpleFileIOOutputProperties in project components by Talend.

the class SimpleFileIOOutputErrorTest method testUnauthorizedAccess.

/**
 * Basic unit test using all default values (except for the path) on an in-memory DFS cluster.
 */
@Test
public void testUnauthorizedAccess() throws IOException, URISyntaxException {
    Path parent = new Path(mini.newFolder().toString());
    String fileSpec = mini.getLocalFs().getUri().resolve(new Path(parent, "output.csv").toUri()).toString();
    // Ensure that the parent is unwritable.
    FileUtil.chmod(parent.toUri().toString(), "000", true);
    // Requesting a wrong execution engine causes an exception.
    thrown.expect(TalendRuntimeException.class);
    thrown.expect(hasProperty("code", is(SimpleFileIOErrorCode.OUTPUT_NOT_AUTHORIZED)));
    thrown.expectMessage("Can not write to " + fileSpec + ". Please check user permissions or existence of base directory.");
    // Now try using the component.
    try {
        // Configure the component.
        SimpleFileIOOutputProperties props = SimpleFileIOOutputRuntimeTest.createOutputComponentProperties();
        props.getDatasetProperties().path.setValue(fileSpec);
        // Create the runtime.
        SimpleFileIOOutputRuntime runtime = new SimpleFileIOOutputRuntime();
        runtime.initialize(null, props);
        // Use the runtime in a direct pipeline to test.
        final Pipeline p = beam.createPipeline();
        PCollection<IndexedRecord> input = // 
        p.apply(// 
        Create.of(// 
        ConvertToIndexedRecord.convertToAvro(new String[] { "1", "one" }), // 
        ConvertToIndexedRecord.convertToAvro(new String[] { "2", "two" })));
        input.apply(runtime);
        // And run the test.
        p.run().waitUntilFinish();
    } catch (Pipeline.PipelineExecutionException e) {
        if (e.getCause() instanceof TalendRuntimeException)
            throw (TalendRuntimeException) e.getCause();
        throw e;
    }
    // Check the expected values.
    mini.assertReadFile(mini.getLocalFs(), fileSpec, "1;one", "2;two");
}

Example 15 with SimpleFileIOOutputProperties

use of org.talend.components.simplefileio.output.SimpleFileIOOutputProperties in project components by Talend.

the class SimpleFileIORoundTripRuntimeTest method testCsvWithDelimiters.

/**
 * Test CSV with custom delimiters.
 */
@Test
public void testCsvWithDelimiters() throws IOException {
    // The file that we will be creating.
    RecordSet rs = getSimpleTestData(0);
    String fileSpec = mini.getLocalFsNewFolder() + "output/";
    // Configure the components.
    SimpleFileIOOutputProperties outputProps = createOutputComponentProperties();
    outputProps.getDatasetProperties().format.setValue(SimpleFileIOFormat.CSV);
    outputProps.getDatasetProperties().path.setValue(fileSpec);
    outputProps.getDatasetProperties().recordDelimiter.setValue(RecordDelimiterType.OTHER);
    outputProps.getDatasetProperties().specificRecordDelimiter.setValue("---");
    outputProps.getDatasetProperties().fieldDelimiter.setValue(FieldDelimiterType.OTHER);
    outputProps.getDatasetProperties().specificFieldDelimiter.setValue("|");
    SimpleFileIOInputProperties inputProps = createInputComponentProperties();
    inputProps.setDatasetProperties(outputProps.getDatasetProperties());
    List<IndexedRecord> actual = runRoundTripPipelines(beam, rs.getAllData(), outputProps, inputProps);
    // Generate the set of expected records. By default, CSV turns all columns into String and loses the original
    // column name.
    List<IndexedRecord> expected = rewriteRecordsWithCsvSchema(rs.getAllData());
    assertThat(expected, containsInAnyOrder(actual.toArray()));
    // Verify that the file on the filesystem was correctly written.
    mini.assertReadFile("---", mini.getLocalFs(), fileSpec, rewriteRecordsAsCsvLines(expected, inputProps.getDatasetProperties().getRecordDelimiter(), inputProps.getDatasetProperties().getFieldDelimiter()));
}