Examples with Transfer com.amazonaws.services.s3.transfer.Transfer used on opensource projects

Example 51 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project spring-integration-aws by spring-projects.

the class S3MessageHandler method copy.

private Transfer copy(Message<?> requestMessage) {
    String sourceBucketName = obtainBucket(requestMessage);
    String sourceKey = null;
    if (this.keyExpression != null) {
        sourceKey = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class);
    }
    Assert.state(sourceKey != null, "The 'keyExpression' must not be null for 'copy' operation " + "and 'keyExpression' can't evaluate to null. " + "Root object is: " + requestMessage);
    String destinationBucketName = null;
    if (this.destinationBucketExpression != null) {
        destinationBucketName = this.destinationBucketExpression.getValue(this.evaluationContext, requestMessage, String.class);
    }
    if (this.resourceIdResolver != null) {
        destinationBucketName = this.resourceIdResolver.resolveToPhysicalResourceId(destinationBucketName);
    }
    Assert.state(destinationBucketName != null, "The 'destinationBucketExpression' must not be null for 'copy' operation and can't evaluate to null. " + "Root object is: " + requestMessage);
    String destinationKey = null;
    if (this.destinationKeyExpression != null) {
        destinationKey = this.destinationKeyExpression.getValue(this.evaluationContext, requestMessage, String.class);
    }
    Assert.state(destinationKey != null, "The 'destinationKeyExpression' must not be null for 'copy' operation and can't evaluate to null. " + "Root object is: " + requestMessage);
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey);
    return this.transferManager.copy(copyObjectRequest);
}

Example 52 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project alluxio by Alluxio.

the class S3AUnderFileSystem method createInstance.

/**
 * Constructs a new instance of {@link S3AUnderFileSystem}.
 *
 * @param uri the {@link AlluxioURI} for this UFS
 * @param conf the configuration for this UFS
 * @return the created {@link S3AUnderFileSystem} instance
 */
public static S3AUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf) {
    AWSCredentialsProvider credentials = createAwsCredentialsProvider(conf);
    String bucketName = UnderFileSystemUtils.getBucketName(uri);
    // Set the client configuration based on Alluxio configuration values.
    ClientConfiguration clientConf = new ClientConfiguration();
    // Max error retry
    if (conf.isSet(PropertyKey.UNDERFS_S3_MAX_ERROR_RETRY)) {
        clientConf.setMaxErrorRetry(conf.getInt(PropertyKey.UNDERFS_S3_MAX_ERROR_RETRY));
    }
    clientConf.setConnectionTTL(conf.getMs(PropertyKey.UNDERFS_S3_CONNECT_TTL));
    // Socket timeout
    clientConf.setSocketTimeout((int) conf.getMs(PropertyKey.UNDERFS_S3_SOCKET_TIMEOUT));
    // HTTP protocol
    if (conf.getBoolean(PropertyKey.UNDERFS_S3_SECURE_HTTP_ENABLED) || conf.getBoolean(PropertyKey.UNDERFS_S3_SERVER_SIDE_ENCRYPTION_ENABLED)) {
        clientConf.setProtocol(Protocol.HTTPS);
    } else {
        clientConf.setProtocol(Protocol.HTTP);
    }
    // Proxy host
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
        clientConf.setProxyHost(conf.getString(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }
    // Proxy port
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
        clientConf.setProxyPort(conf.getInt(PropertyKey.UNDERFS_S3_PROXY_PORT));
    }
    // Number of metadata and I/O threads to S3.
    int numAdminThreads = conf.getInt(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX);
    int numTransferThreads = conf.getInt(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX);
    int numThreads = conf.getInt(PropertyKey.UNDERFS_S3_THREADS_MAX);
    if (numThreads < numAdminThreads + numTransferThreads) {
        LOG.warn("Configured s3 max threads ({}) is less than # admin threads ({}) plus transfer " + "threads ({}). Using admin threads + transfer threads as max threads instead.", numThreads, numAdminThreads, numTransferThreads);
        numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);
    // Set client request timeout for all requests since multipart copy is used,
    // and copy parts can only be set with the client configuration.
    clientConf.setRequestTimeout((int) conf.getMs(PropertyKey.UNDERFS_S3_REQUEST_TIMEOUT));
    boolean streamingUploadEnabled = conf.getBoolean(PropertyKey.UNDERFS_S3_STREAMING_UPLOAD_ENABLED);
    // Signer algorithm
    if (conf.isSet(PropertyKey.UNDERFS_S3_SIGNER_ALGORITHM)) {
        clientConf.setSignerOverride(conf.getString(PropertyKey.UNDERFS_S3_SIGNER_ALGORITHM));
    }
    AwsClientBuilder.EndpointConfiguration endpointConfiguration = createEndpointConfiguration(conf, clientConf);
    AmazonS3 amazonS3Client = createAmazonS3(credentials, clientConf, endpointConfiguration, conf);
    ExecutorService service = ExecutorServiceFactories.fixedThreadPool("alluxio-s3-transfer-manager-worker", numTransferThreads).create();
    TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(createAmazonS3(credentials, clientConf, endpointConfiguration, conf)).withExecutorFactory(() -> service).withMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD).build();
    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName, service, transferManager, conf, streamingUploadEnabled);
}

Example 53 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project carina by qaprosoft.

the class AmazonS3Manager method download.

/**
 * Method to download file from s3 to local file system
 *
 * @param bucketName AWS S3 bucket name
 * @param key (example: android/apkFolder/ApkName.apk)
 * @param file (local file name)
 * @param pollingInterval (polling interval in sec for S3 download status determination)
 */
public void download(final String bucketName, final String key, final File file, long pollingInterval) {
    LOGGER.info("App will be downloaded from s3.");
    LOGGER.info(String.format("[Bucket name: %s] [Key: %s] [File: %s]", bucketName, key, file.getAbsolutePath()));
    DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
    TransferManager tx = new TransferManager(credentialProviderChain.getCredentials());
    Download appDownload = tx.download(bucketName, key, file);
    try {
        LOGGER.info("Transfer: " + appDownload.getDescription());
        LOGGER.info("	State: " + appDownload.getState());
        LOGGER.info("	Progress: ");
        // You can poll your transfer's status to check its progress
        while (!appDownload.isDone()) {
            LOGGER.info("		transferred: " + (int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%");
            CommonUtils.pause(pollingInterval);
        }
        LOGGER.info("	State: " + appDownload.getState());
    // appDownload.waitForCompletion();
    } catch (AmazonClientException e) {
        throw new RuntimeException("File wasn't downloaded from s3. See log: ".concat(e.getMessage()));
    }
// tx.shutdownNow();
}