Examples with EndpointConfiguration com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration used on opensource projects

Example 11 with EndpointConfiguration

use of com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration in project presto by prestodb.

the class PrestoS3ClientFactory method getS3Client.

synchronized AmazonS3 getS3Client(Configuration config, HiveClientConfig clientConfig) {
    if (s3Client != null) {
        return s3Client;
    }
    HiveS3Config defaults = new HiveS3Config();
    String userAgentPrefix = config.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
    int maxErrorRetries = config.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
    boolean sslEnabled = config.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
    Duration connectTimeout = Duration.valueOf(config.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
    Duration socketTimeout = Duration.valueOf(config.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
    int maxConnections = config.getInt(S3_SELECT_PUSHDOWN_MAX_CONNECTIONS, clientConfig.getS3SelectPushdownMaxConnections());
    if (clientConfig.isS3SelectPushdownEnabled()) {
        s3UserAgentSuffix = "presto-select";
    }
    ClientConfiguration clientConfiguration = new ClientConfiguration().withMaxErrorRetry(maxErrorRetries).withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP).withConnectionTimeout(toIntExact(connectTimeout.toMillis())).withSocketTimeout(toIntExact(socketTimeout.toMillis())).withMaxConnections(maxConnections).withUserAgentPrefix(userAgentPrefix).withUserAgentSuffix(s3UserAgentSuffix);
    AWSCredentialsProvider awsCredentialsProvider = getAwsCredentialsProvider(config, defaults);
    AmazonS3Builder<? extends AmazonS3Builder, ? extends AmazonS3> clientBuilder = AmazonS3Client.builder().withCredentials(awsCredentialsProvider).withClientConfiguration(clientConfiguration).withMetricsCollector(new PrestoS3FileSystemMetricCollector(PrestoS3FileSystem.getFileSystemStats())).enablePathStyleAccess();
    boolean regionOrEndpointSet = false;
    String endpoint = config.get(S3_ENDPOINT);
    boolean pinS3ClientToCurrentRegion = config.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion());
    verify(!pinS3ClientToCurrentRegion || endpoint == null, "Invalid configuration: either endpoint can be set or S3 client can be pinned to the current region");
    // use local region when running inside of EC2
    if (pinS3ClientToCurrentRegion) {
        Region region = Regions.getCurrentRegion();
        if (region != null) {
            clientBuilder.withRegion(region.getName());
            regionOrEndpointSet = true;
        }
    }
    if (!isNullOrEmpty(endpoint)) {
        clientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, null));
        regionOrEndpointSet = true;
    }
    if (!regionOrEndpointSet) {
        clientBuilder.withRegion(US_EAST_1);
        clientBuilder.setForceGlobalBucketAccessEnabled(true);
    }
    s3Client = clientBuilder.build();
    return s3Client;
}

Example 12 with EndpointConfiguration

use of com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration in project presto by prestodb.

the class PrestoS3FileSystem method createAmazonS3Client.

private AmazonS3 createAmazonS3Client(Configuration hadoopConfig, ClientConfiguration clientConfig) {
    Optional<EncryptionMaterialsProvider> encryptionMaterialsProvider = createEncryptionMaterialsProvider(hadoopConfig);
    AmazonS3Builder<? extends AmazonS3Builder, ? extends AmazonS3> clientBuilder;
    String signerType = hadoopConfig.get(S3_SIGNER_TYPE);
    if (signerType != null) {
        clientConfig.withSignerOverride(signerType);
    }
    if (encryptionMaterialsProvider.isPresent()) {
        clientBuilder = AmazonS3EncryptionClient.encryptionBuilder().withCredentials(credentialsProvider).withEncryptionMaterials(encryptionMaterialsProvider.get()).withClientConfiguration(clientConfig).withMetricsCollector(METRIC_COLLECTOR);
    } else {
        clientBuilder = AmazonS3Client.builder().withCredentials(credentialsProvider).withClientConfiguration(clientConfig).withMetricsCollector(METRIC_COLLECTOR);
    }
    boolean regionOrEndpointSet = false;
    // use local region when running inside of EC2
    if (pinS3ClientToCurrentRegion) {
        Region region = Regions.getCurrentRegion();
        if (region != null) {
            clientBuilder = clientBuilder.withRegion(region.getName());
            regionOrEndpointSet = true;
        }
    }
    String endpoint = hadoopConfig.get(S3_ENDPOINT);
    if (endpoint != null) {
        clientBuilder = clientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, null));
        regionOrEndpointSet = true;
    }
    if (isPathStyleAccess) {
        clientBuilder = clientBuilder.enablePathStyleAccess();
    }
    if (!regionOrEndpointSet) {
        clientBuilder = clientBuilder.withRegion(US_EAST_1);
        clientBuilder.setForceGlobalBucketAccessEnabled(true);
    }
    return clientBuilder.build();
}

Example 13 with EndpointConfiguration

use of com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration in project OsmAnd-tools by osmandapp.

the class StorageService method getStorageProviderById.

private StorageType getStorageProviderById(String id) {
    id = id.trim();
    StorageType st = storageProviders.get(id);
    if (st != null) {
        return st;
    }
    if (id.equals(LOCAL_STORAGE)) {
        st = new StorageType();
        st.local = true;
    } else {
        String prefix = "storage.datasource." + id + ".";
        String endpointUrl = env.getProperty(prefix + "endpoint");
        checkNotNull(endpointUrl, "endpoint", id);
        String region = env.getProperty(prefix + "region");
        checkNotNull(region, "endpoint", id);
        String bucket = env.getProperty(prefix + "bucket");
        checkNotNull(region, "bucket", id);
        String accessKey = env.getProperty(prefix + "accesskey");
        String secretKey = env.getProperty(prefix + "secretkey");
        AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withEndpointConfiguration(new EndpointConfiguration(endpointUrl, region));
        if (!Algorithms.isEmpty(accessKey)) {
            builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
        }
        AmazonS3 s3 = builder.build();
        st = new StorageType();
        st.bucket = bucket;
        st.s3Conn = s3;
    }
    storageProviders.put(id, st);
    return st;
}