Examples with S3DatasetProperties org.talend.components.simplefileio.s3.S3DatasetProperties used on opensource projects

Example 21 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3RoundTripRuntimeTestIT method testCsv_noEncryption.

/**
 * Basic Csv test.
 */
@Test
@Ignore("columns name different, can't editable")
public void testCsv_noEncryption() throws IOException {
    S3DatasetProperties datasetProps = s3.createS3DatasetProperties();
    datasetProps.format.setValue(SimpleFileIOFormat.CSV);
    datasetProps.recordDelimiter.setValue(SimpleFileIODatasetProperties.RecordDelimiterType.LF);
    datasetProps.fieldDelimiter.setValue(SimpleFileIODatasetProperties.FieldDelimiterType.SEMICOLON);
    test_noEncryption(datasetProps);
}

Example 22 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3RoundTripRuntimeTestIT method test_noEncryption.

public void test_noEncryption(S3DatasetProperties datasetProps) throws IOException {
    // The file that we will be creating.
    RecordSet rs = getSimpleTestData(0);
    // Configure the components.
    S3OutputProperties outputProps = new S3OutputProperties("out");
    outputProps.setDatasetProperties(datasetProps);
    S3InputProperties inputProps = new S3InputProperties("in");
    inputProps.setDatasetProperties(datasetProps);
    List<IndexedRecord> actual = runRoundTripPipelines(rs.getAllData(), outputProps, inputProps);
    List<IndexedRecord> expected = rs.getAllData();
    assertThat(actual, containsInAnyOrder(expected.toArray()));
    List<IndexedRecord> samples = getSample(datasetProps);
    assertThat(samples, containsInAnyOrder(expected.toArray()));
    Schema schema = getSchema(datasetProps);
    assertEquals(expected.get(0).getSchema(), schema);
}

Example 23 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3TestResource method createS3DatasetProperties.

/**
 * Return an S3DatasetProperties potentially configured for encryption.
 *
 * @param sseKms Whether server-side encryption is used. The KMS key is taken from the system environment.
 * @param sseKms Whether client-side encryption is used. The KMS key is taken from the system environment.
 * @return An S3DatasetProperties with credentials in the datastore, configured for the specified encryption. The
 * region are bucket are taken from the environment, and a unique "object" property is created for this unit test.
 */
public S3DatasetProperties createS3DatasetProperties(boolean sseKms, boolean cseKms) {
    S3DatasetProperties properties = new S3DatasetProperties(null);
    properties.init();
    properties.region.setValue(S3Region.valueOf(System.getProperty("s3.region")));
    properties.bucket.setValue(bucketName);
    properties.object.setValue(getPath());
    properties.setDatastoreProperties(createS3DatastoreProperties());
    if (sseKms) {
        properties.encryptDataAtRest.setValue(true);
        properties.kmsForDataAtRest.setValue(System.getProperty("s3.ssekmskey"));
    }
    if (cseKms) {
        properties.encryptDataInMotion.setValue(true);
        properties.kmsForDataInMotion.setValue(System.getProperty("s3.csekmskey"));
    }
    return properties;
}

Example 24 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3TestResource method after.

/**
 * Remove any resources created on the S3 bucket.
 */
@Override
protected void after() {
    try {
        S3DatasetProperties datasetProps = createS3DatasetProperties();
        S3AFileSystem fs = S3Connection.createFileSystem(datasetProps);
        Path path = new Path(S3Connection.getUriPath(datasetProps));
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }
}