Examples with S3InputProperties org.talend.components.simplefileio.s3.input.S3InputProperties used on opensource projects

Example 1 with S3InputProperties

use of org.talend.components.simplefileio.s3.input.S3InputProperties in project components by Talend.

the class S3DatasetRuntime method getSample.

@Override
public void getSample(int limit, Consumer<IndexedRecord> consumer) {
    // Create an input runtime based on the properties.
    S3InputRuntime inputRuntime = new S3InputRuntime();
    S3InputProperties inputProperties = new S3InputProperties(null);
    inputProperties.limit.setValue(limit);
    inputProperties.init();
    inputProperties.setDatasetProperties(properties);
    inputRuntime.initialize(null, inputProperties);
    // Create a pipeline using the input component to get records.
    DirectOptions options = BeamLocalRunnerOption.getOptions();
    final Pipeline p = Pipeline.create(options);
    try (DirectConsumerCollector<IndexedRecord> collector = DirectConsumerCollector.of(consumer)) {
        // Collect a sample of the input records.
        // 
        p.apply(inputRuntime).apply(// 
        Sample.<IndexedRecord>any(limit)).apply(collector);
        p.run().waitUntilFinish();
    }
}

Example 2 with S3InputProperties

use of org.talend.components.simplefileio.s3.input.S3InputProperties in project components by Talend.

the class S3SparkRuntimeTestIT method test_noEncryption.

public void test_noEncryption(S3DatasetProperties datasetProps) throws IOException {
    // The file that we will be creating.
    RecordSet rs = getSimpleTestData(0);
    // Configure the components.
    S3OutputProperties outputProps = new S3OutputProperties("out");
    outputProps.setDatasetProperties(datasetProps);
    S3InputProperties inputProps = new S3InputProperties("in");
    inputProps.setDatasetProperties(datasetProps);
    List<IndexedRecord> actual = runRoundTripPipelines(rs.getAllData(), outputProps, inputProps);
    List<IndexedRecord> expected = rs.getAllData();
    assertThat(actual, containsInAnyOrder(expected.toArray()));
    List<IndexedRecord> samples = getSample(datasetProps);
    assertThat(samples, containsInAnyOrder(expected.toArray()));
    Schema schema = getSchema(datasetProps);
    assertEquals(expected.get(0).getSchema(), schema);
}

Example 3 with S3InputProperties

use of org.talend.components.simplefileio.s3.input.S3InputProperties in project components by Talend.

the class S3RoundTripRuntimeTestIT method test_noEncryption.

public void test_noEncryption(S3DatasetProperties datasetProps) throws IOException {
    // The file that we will be creating.
    RecordSet rs = getSimpleTestData(0);
    // Configure the components.
    S3OutputProperties outputProps = new S3OutputProperties("out");
    outputProps.setDatasetProperties(datasetProps);
    S3InputProperties inputProps = new S3InputProperties("in");
    inputProps.setDatasetProperties(datasetProps);
    List<IndexedRecord> actual = runRoundTripPipelines(rs.getAllData(), outputProps, inputProps);
    List<IndexedRecord> expected = rs.getAllData();
    assertThat(actual, containsInAnyOrder(expected.toArray()));
    List<IndexedRecord> samples = getSample(datasetProps);
    assertThat(samples, containsInAnyOrder(expected.toArray()));
    Schema schema = getSchema(datasetProps);
    assertEquals(expected.get(0).getSchema(), schema);
}