Examples with S3DatasetProperties org.talend.components.simplefileio.s3.S3DatasetProperties used on opensource projects

Example 6 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3RoundTripRuntimeTestIT method testAvro_noEncryption.

/**
 * Basic Avro test.
 */
@Test
public void testAvro_noEncryption() throws IOException {
    S3DatasetProperties datasetProps = s3.createS3DatasetProperties();
    datasetProps.format.setValue(SimpleFileIOFormat.AVRO);
    test_noEncryption(datasetProps);
    // Get some object metadata from the results.
    ObjectMetadata md = s3.getObjectMetadata(datasetProps);
    assertThat(md.getSSEAlgorithm(), nullValue());
    assertThat(md.getSSEAwsKmsKeyId(), nullValue());
}

Example 7 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3RoundTripRuntimeTestIT method testAvro_sseKmsEncryption.

/**
 * Basic Avro test with sseKmsEncryption.
 */
@Test
public void testAvro_sseKmsEncryption() throws IOException {
    S3DatasetProperties datasetProps = s3.createS3DatasetProperties(true, false);
    datasetProps.format.setValue(SimpleFileIOFormat.AVRO);
    test_noEncryption(datasetProps);
    // Get some object metadata from the results.
    ObjectMetadata md = s3.getObjectMetadata(datasetProps);
    assertThat(md.getSSEAlgorithm(), is("aws:kms"));
    assertThat(md.getSSEAwsKmsKeyId(), is(datasetProps.kmsForDataAtRest.getValue()));
}

Example 8 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3Connection method createClient.

public static AmazonS3 createClient(S3OutputProperties properties) {
    S3DatasetProperties data_set = properties.getDatasetProperties();
    S3DatastoreProperties data_store = properties.getDatasetProperties().getDatastoreProperties();
    com.amazonaws.auth.AWSCredentials credentials = new com.amazonaws.auth.BasicAWSCredentials(data_store.accessKey.getValue(), data_store.secretKey.getValue());
    Region region = RegionUtils.getRegion(data_set.region.getValue().getValue());
    Boolean clientSideEnc = data_set.encryptDataInMotion.getValue();
    AmazonS3 conn = null;
    if (clientSideEnc != null && clientSideEnc) {
        String kms_cmk = data_set.kmsForDataInMotion.getValue();
        KMSEncryptionMaterialsProvider encryptionMaterialsProvider = new KMSEncryptionMaterialsProvider(kms_cmk);
        conn = new AmazonS3EncryptionClient(credentials, encryptionMaterialsProvider, new CryptoConfiguration().withAwsKmsRegion(region));
    } else {
        AWSCredentialsProvider basicCredentialsProvider = new StaticCredentialsProvider(credentials);
        conn = new AmazonS3Client(basicCredentialsProvider);
    }
    conn.setRegion(region);
    return conn;
}

Example 9 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class PropertiesPreparer method createS3OtuputProperties.

public static S3OutputProperties createS3OtuputProperties() {
    S3OutputProperties properties = new S3OutputProperties("s3output");
    S3DatasetProperties dataset = new S3DatasetProperties("dataset");
    S3DatastoreProperties datastore = new S3DatastoreProperties("datastore");
    properties.setDatasetProperties(dataset);
    dataset.setDatastoreProperties(datastore);
    datastore.accessKey.setValue(accessKey);
    datastore.secretKey.setValue(secretkey);
    dataset.region.setValue(S3Region.valueOf(region));
    dataset.bucket.setValue(bucket);
    dataset.kmsForDataInMotion.setValue(ssekmskey);
    dataset.kmsForDataAtRest.setValue(csekmskey);
    dataset.object.setValue(objectkey);
    return properties;
}

Example 10 with S3DatasetProperties

use of org.talend.components.simplefileio.s3.S3DatasetProperties in project components by Talend.

the class S3OutputWriter method close.

/**
 * not sure the method is called one or two times, it depend on the platform
 */
@Override
public Result close() throws IOException {
    if (closed) {
        return result;
    }
    closed = true;
    try {
        if (writer != null) {
            writer.flush();
            writer.close();
        }
        S3DatasetProperties data_set = properties.getDatasetProperties();
        PutObjectRequest request = new PutObjectRequest(data_set.bucket.getValue(), data_set.object.getValue(), data_file);
        Boolean serverSideEnc = data_set.encryptDataAtRest.getValue();
        if (serverSideEnc != null && serverSideEnc) {
            request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(data_set.kmsForDataAtRest.getValue()));
        }
        s3_client.putObject(request);
    } finally {
        writer = null;
        data_file.delete();
        if (s3_client != null) {
            s3_client.shutdown();
            s3_client = null;
        }
    }
    result.successCount = result.totalCount;
    return result;
}