Examples with AwsProperties org.apache.iceberg.aws.AwsProperties used on opensource projects

Example 1 with AwsProperties

use of org.apache.iceberg.aws.AwsProperties in project iceberg by apache.

the class S3FileIO method initialize.

@Override
public void initialize(Map<String, String> properties) {
    this.awsProperties = new AwsProperties(properties);
    // Do not override s3 client if it was provided
    if (s3 == null) {
        this.s3 = AwsClientFactories.from(properties)::s3;
    }
    // Report Hadoop metrics if Hadoop is available
    try {
        DynConstructors.Ctor<MetricsContext> ctor = DynConstructors.builder(MetricsContext.class).hiddenImpl(DEFAULT_METRICS_IMPL, String.class).buildChecked();
        this.metrics = ctor.newInstance("s3");
        metrics.initialize(properties);
    } catch (NoSuchMethodException | ClassCastException e) {
        LOG.warn("Unable to load metrics class: '{}', falling back to null metrics", DEFAULT_METRICS_IMPL, e);
    }
}

Example 2 with AwsProperties

use of org.apache.iceberg.aws.AwsProperties in project iceberg by apache.

the class TestGlueCatalog method testConstructorEmptyWarehousePath.

@Test
public void testConstructorEmptyWarehousePath() {
    AssertHelpers.assertThrows("warehouse path cannot be null", IllegalArgumentException.class, "Cannot initialize GlueCatalog because warehousePath must not be null", () -> {
        GlueCatalog catalog = new GlueCatalog();
        catalog.initialize(CATALOG_NAME, null, new AwsProperties(), glue, LockManagers.defaultLockManager(), null);
    });
}

Example 3 with AwsProperties

use of org.apache.iceberg.aws.AwsProperties in project iceberg by apache.

the class TestS3FileIOIntegration method testServerSideS3Encryption.

@Test
public void testServerSideS3Encryption() throws Exception {
    AwsProperties properties = new AwsProperties();
    properties.setS3FileIoSseType(AwsProperties.S3FILEIO_SSE_TYPE_S3);
    S3FileIO s3FileIO = new S3FileIO(clientFactory::s3, properties);
    write(s3FileIO);
    validateRead(s3FileIO);
    GetObjectResponse response = s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build()).response();
    Assert.assertEquals(ServerSideEncryption.AES256, response.serverSideEncryption());
}

Example 4 with AwsProperties

use of org.apache.iceberg.aws.AwsProperties in project iceberg by apache.

the class TestS3FileIOIntegration method testServerSideCustomEncryption.

@Test
public void testServerSideCustomEncryption() throws Exception {
    // generate key
    KeyGenerator keyGenerator = KeyGenerator.getInstance("AES");
    keyGenerator.init(256, new SecureRandom());
    SecretKey secretKey = keyGenerator.generateKey();
    Base64.Encoder encoder = Base64.getEncoder();
    String encodedKey = new String(encoder.encode(secretKey.getEncoded()), StandardCharsets.UTF_8);
    // generate md5
    MessageDigest digest = MessageDigest.getInstance("MD5");
    String md5 = new String(encoder.encode(digest.digest(secretKey.getEncoded())), StandardCharsets.UTF_8);
    AwsProperties properties = new AwsProperties();
    properties.setS3FileIoSseType(AwsProperties.S3FILEIO_SSE_TYPE_CUSTOM);
    properties.setS3FileIoSseKey(encodedKey);
    properties.setS3FileIoSseMd5(md5);
    S3FileIO s3FileIO = new S3FileIO(clientFactory::s3, properties);
    write(s3FileIO);
    validateRead(s3FileIO);
    GetObjectResponse response = s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).sseCustomerAlgorithm(ServerSideEncryption.AES256.name()).sseCustomerKey(encodedKey).sseCustomerKeyMD5(md5).build()).response();
    Assert.assertNull(response.serverSideEncryption());
    Assert.assertEquals(ServerSideEncryption.AES256.name(), response.sseCustomerAlgorithm());
    Assert.assertEquals(md5, response.sseCustomerKeyMD5());
}

Example 5 with AwsProperties

use of org.apache.iceberg.aws.AwsProperties in project iceberg by apache.

the class TestS3OutputStream method testStagingDirectoryCreation.

@Test
public void testStagingDirectoryCreation() throws IOException {
    AwsProperties newStagingDirectoryAwsProperties = new AwsProperties(ImmutableMap.of(AwsProperties.S3FILEIO_STAGING_DIRECTORY, newTmpDirectory));
    S3OutputStream stream = new S3OutputStream(s3, randomURI(), newStagingDirectoryAwsProperties, nullMetrics());
    stream.close();
}