Examples with S3ClientOptions com.amazonaws.services.s3.S3ClientOptions used on opensource projects

Example 1 with S3ClientOptions

use of com.amazonaws.services.s3.S3ClientOptions in project elasticsearch by elastic.

the class InternalAwsS3Service method client.

@Override
public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess) {
    String clientName = CLIENT_NAME.get(repositorySettings);
    String foundEndpoint = findEndpoint(logger, repositorySettings, settings, clientName);
    AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings, clientName);
    Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
    AmazonS3Client client = clients.get(clientDescriptor);
    if (client != null) {
        return client;
    }
    client = new AmazonS3Client(credentials, buildConfiguration(logger, repositorySettings, settings, clientName, maxRetries, foundEndpoint, useThrottleRetries));
    if (pathStyleAccess != null) {
        client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
    }
    if (!foundEndpoint.isEmpty()) {
        client.setEndpoint(foundEndpoint);
    }
    clients.put(clientDescriptor, client);
    return client;
}

Example 2 with S3ClientOptions

use of com.amazonaws.services.s3.S3ClientOptions in project alluxio by Alluxio.

the class S3AUnderFileSystem method createInstance.

/**
   * Constructs a new instance of {@link S3AUnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @return the created {@link S3AUnderFileSystem} instance
   */
public static S3AUnderFileSystem createInstance(AlluxioURI uri) {
    String bucketName = uri.getHost();
    // Set the aws credential system properties based on Alluxio properties, if they are set
    if (Configuration.containsKey(PropertyKey.S3A_ACCESS_KEY)) {
        System.setProperty(SDKGlobalConfiguration.ACCESS_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_ACCESS_KEY));
    }
    if (Configuration.containsKey(PropertyKey.S3A_SECRET_KEY)) {
        System.setProperty(SDKGlobalConfiguration.SECRET_KEY_SYSTEM_PROPERTY, Configuration.get(PropertyKey.S3A_SECRET_KEY));
    }
    // Checks, in order, env variables, system properties, profile file, and instance profile
    AWSCredentialsProvider credentials = new AWSCredentialsProviderChain(new DefaultAWSCredentialsProviderChain());
    // Set the client configuration based on Alluxio configuration values
    ClientConfiguration clientConf = new ClientConfiguration();
    // Socket timeout
    clientConf.setSocketTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS));
    // HTTP protocol
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED)) {
        clientConf.setProtocol(Protocol.HTTPS);
    } else {
        clientConf.setProtocol(Protocol.HTTP);
    }
    // Proxy host
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        clientConf.setProxyHost(Configuration.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }
    // Proxy port
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
        clientConf.setProxyPort(Configuration.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }
    int numAdminThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX);
    int numTransferThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX);
    int numThreads = Configuration.getInt(PropertyKey.UNDERFS_S3_THREADS_MAX);
    if (numThreads < numAdminThreads + numTransferThreads) {
        LOG.warn("Configured s3 max threads: {} is less than # admin threads: {} plus transfer " + "threads {}. Using admin threads + transfer threads as max threads instead.");
        numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);
    // Set client request timeout for all requests since multipart copy is used, and copy parts can
    // only be set with the client configuration.
    clientConf.setRequestTimeout(Configuration.getInt(PropertyKey.UNDERFS_S3A_REQUEST_TIMEOUT));
    AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, clientConf);
    // Set a custom endpoint.
    if (Configuration.containsKey(PropertyKey.UNDERFS_S3_ENDPOINT)) {
        amazonS3Client.setEndpoint(Configuration.get(PropertyKey.UNDERFS_S3_ENDPOINT));
    }
    // Disable DNS style buckets, this enables path style requests.
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
        S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
        amazonS3Client.setS3ClientOptions(clientOptions);
    }
    ExecutorService service = ExecutorServiceFactories.fixedThreadPoolExecutorServiceFactory("alluxio-s3-transfer-manager-worker", numTransferThreads).create();
    TransferManager transferManager = new TransferManager(amazonS3Client, service);
    TransferManagerConfiguration transferConf = new TransferManagerConfiguration();
    transferConf.setMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD);
    transferManager.setConfiguration(transferConf);
    // Default to readable and writable by the user.
    short bucketMode = (short) 700;
    // There is no known account owner by default.
    String accountOwner = "";
    // if ACL enabled inherit bucket acl for all the objects.
    if (Configuration.getBoolean(PropertyKey.UNDERFS_S3A_INHERIT_ACL)) {
        String accountOwnerId = amazonS3Client.getS3AccountOwner().getId();
        // Gets the owner from user-defined static mapping from S3 canonical user
        // id to Alluxio user name.
        String owner = CommonUtils.getValueFromStaticMapping(Configuration.get(PropertyKey.UNDERFS_S3_OWNER_ID_TO_USERNAME_MAPPING), accountOwnerId);
        // If there is no user-defined mapping, use the display name.
        if (owner == null) {
            owner = amazonS3Client.getS3AccountOwner().getDisplayName();
        }
        accountOwner = owner == null ? accountOwnerId : owner;
        AccessControlList acl = amazonS3Client.getBucketAcl(bucketName);
        bucketMode = S3AUtils.translateBucketAcl(acl, accountOwnerId);
    }
    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName, bucketMode, accountOwner, transferManager);
}

Example 3 with S3ClientOptions

use of com.amazonaws.services.s3.S3ClientOptions in project camel by apache.

the class S3Endpoint method createS3Client.

/**
     * Provide the possibility to override this method for an mock implementation
     */
AmazonS3 createS3Client() {
    AmazonS3Client client = null;
    ClientConfiguration clientConfiguration = null;
    boolean isClientConfigFound = false;
    if (configuration.hasProxyConfiguration()) {
        clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProxyHost(configuration.getProxyHost());
        clientConfiguration.setProxyPort(configuration.getProxyPort());
        isClientConfigFound = true;
    }
    if (configuration.getAccessKey() != null && configuration.getSecretKey() != null) {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        if (isClientConfigFound) {
            client = new AmazonS3Client(credentials, clientConfiguration);
        } else {
            client = new AmazonS3Client(credentials);
        }
    } else {
        if (isClientConfigFound) {
            client = new AmazonS3Client();
        } else {
            client = new AmazonS3Client(clientConfiguration);
        }
    }
    S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(configuration.isPathStyleAccess()).build();
    client.setS3ClientOptions(clientOptions);
    return client;
}

Example 4 with S3ClientOptions

use of com.amazonaws.services.s3.S3ClientOptions in project acs-aem-commons by Adobe-Consulting-Services.

the class S3AssetIngestorTest method setup.

@Before
public void setup() throws PersistenceException {
    context.registerAdapter(ResourceResolver.class, AssetManager.class, new Function<ResourceResolver, AssetManager>() {

        @Nullable
        @Override
        public AssetManager apply(@Nullable ResourceResolver input) {
            return assetManager;
        }
    });
    context.create().resource("/content/dam", JcrConstants.JCR_PRIMARYTYPE, "sling:Folder");
    context.resourceResolver().commit();
    ingestor = new S3AssetIngestor(context.getService(MimeTypeService.class));
    ingestor.jcrBasePath = "/content/dam";
    ingestor.ignoreFileList = Collections.emptyList();
    ingestor.ignoreExtensionList = Collections.emptyList();
    ingestor.ignoreFolderList = Arrays.asList(".ds_store");
    ingestor.existingAssetAction = AssetIngestor.AssetAction.skip;
    int port = FreePortFinder.findFreeLocalPort();
    s3Mock = new S3Mock.Builder().withPort(port).withInMemoryBackend().build();
    s3Mock.start();
    S3ClientOptions options = S3ClientOptions.builder().setPathStyleAccess(true).build();
    s3Client = new AmazonS3Client(new AnonymousAWSCredentials());
    s3Client.setS3ClientOptions(options);
    s3Client.setEndpoint("http://localhost:" + port);
    ingestor.s3Client = s3Client;
    ingestor.bucket = TEST_BUCKET;
    s3Client.createBucket(TEST_BUCKET);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            CheckedConsumer<ResourceResolver> method = (CheckedConsumer<ResourceResolver>) invocation.getArguments()[0];
            method.accept(context.resourceResolver());
            return null;
        }
    }).when(actionManager).deferredWithResolver(any(CheckedConsumer.class));
}

Example 5 with S3ClientOptions

use of com.amazonaws.services.s3.S3ClientOptions in project presto by prestodb.

the class TestPrestoS3FileSystem method testPathStyleAccess.

@Test
public void testPathStyleAccess() throws Exception {
    Configuration config = new Configuration();
    config.setBoolean(S3_PATH_STYLE_ACCESS, true);
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        fs.initialize(new URI("s3n://test-bucket/"), config);
        S3ClientOptions clientOptions = getFieldValue(fs.getS3Client(), AmazonS3Client.class, "clientOptions", S3ClientOptions.class);
        assertTrue(clientOptions.isPathStyleAccess());
    }
}