Examples with TransferManager com.amazonaws.services.s3.transfer.TransferManager used on opensource projects

Example 41 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project tutorials by eugenp.

the class MultipartUpload method main.

public static void main(String[] args) throws Exception {
    String existingBucketName = "baeldung-bucket";
    String keyName = "my-picture.jpg";
    String filePath = "documents/my-picture.jpg";
    AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard().withCredentials(new DefaultAWSCredentialsProviderChain()).withRegion(Regions.DEFAULT_REGION).build();
    int maxUploadThreads = 5;
    TransferManager tm = TransferManagerBuilder.standard().withS3Client(amazonS3).withMultipartUploadThreshold((long) (5 * 1024 * 1024)).withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads)).build();
    ProgressListener progressListener = progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred());
    PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath));
    request.setGeneralProgressListener(progressListener);
    Upload upload = tm.upload(request);
    try {
        upload.waitForCompletion();
        System.out.println("Upload complete.");
    } catch (AmazonClientException e) {
        System.out.println("Error occurred while uploading file");
        e.printStackTrace();
    }
}

Example 42 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project qpp-conversion-tool by CMSgov.

the class StorageServiceImpl method asynchronousAction.

/**
 * Uses the {@link TransferManager} to upload a file.
 *
 * @param objectToActOn The put request.
 * @return The object key in the bucket.
 */
@Override
protected String asynchronousAction(Supplier<PutObjectRequest> objectToActOn) {
    String returnValue;
    PutObjectRequest request = objectToActOn.get();
    try {
        Upload upload = s3TransferManager.upload(request);
        returnValue = upload.waitForUploadResult().getKey();
    } catch (InterruptedException exception) {
        Thread.currentThread().interrupt();
        throw new UncheckedInterruptedException(exception);
    }
    API_LOG.info("Successfully wrote object {} to S3 bucket {}", returnValue, request.getBucketName());
    return returnValue;
}

Example 43 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project alluxio by Alluxio.

the class S3AUnderFileSystem method createInstance.

/**
 * Constructs a new instance of {@link S3AUnderFileSystem}.
 *
 * @param uri the {@link AlluxioURI} for this UFS
 * @param conf the configuration for this UFS
 * @return the created {@link S3AUnderFileSystem} instance
 */
public static S3AUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf) {
    AWSCredentialsProvider credentials = createAwsCredentialsProvider(conf);
    String bucketName = UnderFileSystemUtils.getBucketName(uri);
    // Set the client configuration based on Alluxio configuration values.
    ClientConfiguration clientConf = new ClientConfiguration();
    // Max error retry
    if (conf.isSet(PropertyKey.UNDERFS_S3_MAX_ERROR_RETRY)) {
        clientConf.setMaxErrorRetry(conf.getInt(PropertyKey.UNDERFS_S3_MAX_ERROR_RETRY));
    }
    clientConf.setConnectionTTL(conf.getMs(PropertyKey.UNDERFS_S3_CONNECT_TTL));
    // Socket timeout
    clientConf.setSocketTimeout((int) conf.getMs(PropertyKey.UNDERFS_S3_SOCKET_TIMEOUT));
    // HTTP protocol
    if (conf.getBoolean(PropertyKey.UNDERFS_S3_SECURE_HTTP_ENABLED) || conf.getBoolean(PropertyKey.UNDERFS_S3_SERVER_SIDE_ENCRYPTION_ENABLED)) {
        clientConf.setProtocol(Protocol.HTTPS);
    } else {
        clientConf.setProtocol(Protocol.HTTP);
    }
    // Proxy host
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        clientConf.setProxyHost(conf.getString(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }
    // Proxy port
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
        clientConf.setProxyPort(conf.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }
    // Number of metadata and I/O threads to S3.
    int numAdminThreads = conf.getInt(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX);
    int numTransferThreads = conf.getInt(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX);
    int numThreads = conf.getInt(PropertyKey.UNDERFS_S3_THREADS_MAX);
    if (numThreads < numAdminThreads + numTransferThreads) {
        LOG.warn("Configured s3 max threads ({}) is less than # admin threads ({}) plus transfer " + "threads ({}). Using admin threads + transfer threads as max threads instead.", numThreads, numAdminThreads, numTransferThreads);
        numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);
    // Set client request timeout for all requests since multipart copy is used,
    // and copy parts can only be set with the client configuration.
    clientConf.setRequestTimeout((int) conf.getMs(PropertyKey.UNDERFS_S3_REQUEST_TIMEOUT));
    boolean streamingUploadEnabled = conf.getBoolean(PropertyKey.UNDERFS_S3_STREAMING_UPLOAD_ENABLED);
    // Signer algorithm
    if (conf.isSet(PropertyKey.UNDERFS_S3_SIGNER_ALGORITHM)) {
        clientConf.setSignerOverride(conf.getString(PropertyKey.UNDERFS_S3_SIGNER_ALGORITHM));
    }
    AwsClientBuilder.EndpointConfiguration endpointConfiguration = createEndpointConfiguration(conf, clientConf);
    AmazonS3 amazonS3Client = createAmazonS3(credentials, clientConf, endpointConfiguration, conf);
    ExecutorService service = ExecutorServiceFactories.fixedThreadPool("alluxio-s3-transfer-manager-worker", numTransferThreads).create();
    TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(createAmazonS3(credentials, clientConf, endpointConfiguration, conf)).withExecutorFactory(() -> service).withMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD).build();
    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName, service, transferManager, conf, streamingUploadEnabled);
}

Example 44 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project cloudstack by apache.

the class S3Utils method getTransferManager.

public static TransferManager getTransferManager(final ClientOptions clientOptions) {
    if (TRANSFERMANAGER_ACCESSKEY_MAP.containsKey(clientOptions.getAccessKey())) {
        return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
    }
    final AWSCredentials basicAWSCredentials = new BasicAWSCredentials(clientOptions.getAccessKey(), clientOptions.getSecretKey());
    final ClientConfiguration configuration = new ClientConfiguration();
    if (clientOptions.isHttps() != null) {
        configuration.setProtocol(clientOptions.isHttps() ? HTTPS : HTTP);
    }
    if (clientOptions.getConnectionTimeout() != null) {
        configuration.setConnectionTimeout(clientOptions.getConnectionTimeout());
    }
    if (clientOptions.getMaxErrorRetry() != null) {
        configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
    }
    if (clientOptions.getSocketTimeout() != null) {
        configuration.setSocketTimeout(clientOptions.getSocketTimeout());
    }
    if (clientOptions.getUseTCPKeepAlive() != null) {
        configuration.setUseTcpKeepAlive(clientOptions.getUseTCPKeepAlive());
    }
    if (clientOptions.getConnectionTtl() != null) {
        configuration.setConnectionTTL(clientOptions.getConnectionTtl());
    }
    if (clientOptions.getSigner() != null) {
        configuration.setSignerOverride(clientOptions.getSigner());
    }
    LOGGER.debug(format("Creating S3 client with configuration: [protocol: %1$s, signer: %2$s, connectionTimeOut: %3$s, maxErrorRetry: %4$s, socketTimeout: %5$s, useTCPKeepAlive: %6$s, connectionTtl: %7$s]", configuration.getProtocol(), configuration.getSignerOverride(), configuration.getConnectionTimeout(), configuration.getMaxErrorRetry(), configuration.getSocketTimeout(), clientOptions.getUseTCPKeepAlive(), clientOptions.getConnectionTtl()));
    final AmazonS3Client client = new AmazonS3Client(basicAWSCredentials, configuration);
    if (StringUtils.isNotBlank(clientOptions.getEndPoint())) {
        LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));
        client.setEndpoint(clientOptions.getEndPoint());
    }
    TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));
    return TRANSFERMANAGER_ACCESSKEY_MAP.get(clientOptions.getAccessKey());
}

Example 45 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project pipeline-aws-plugin by jenkinsci.

the class S3UploadStepTransferManagerIntegrationTest method setupSdk.

@Before
public void setupSdk() throws Exception {
    PowerMockito.mockStatic(AWSClientFactory.class);
    AmazonS3 amazonS3 = PowerMockito.mock(AmazonS3.class);
    PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(StepContext.class))).thenReturn(amazonS3);
    PowerMockito.mockStatic(TransferManagerBuilder.class);
    transferManager = Mockito.mock(TransferManager.class);
    TransferManagerBuilder transferManagerBuilder = PowerMockito.mock(TransferManagerBuilder.class);
    PowerMockito.when(TransferManagerBuilder.standard()).thenReturn(transferManagerBuilder);
    PowerMockito.when(transferManagerBuilder.withS3Client(Mockito.any(AmazonS3.class))).thenReturn(transferManagerBuilder);
    PowerMockito.when(transferManagerBuilder.build()).thenReturn(transferManager);
}