Examples with Transfer com.amazonaws.services.s3.transfer.Transfer used on opensource projects

Example 31 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project carina by qaprosoft.

the class AmazonS3Manager method download.

/**
 * Method to download file from s3 to local file system
 *
 * @param bucketName AWS S3 bucket name
 * @param key (example: android/apkFolder/ApkName.apk)
 * @param file (local file name)
 * @param pollingInterval (polling interval in sec for S3 download status determination)
 */
public void download(final String bucketName, final String key, final File file, long pollingInterval) {
    LOGGER.info("App will be downloaded from s3.");
    LOGGER.info(String.format("[Bucket name: %s] [Key: %s] [File: %s]", bucketName, key, file.getAbsolutePath()));
    DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
    TransferManager tx = new TransferManager(credentialProviderChain.getCredentials());
    Download appDownload = tx.download(bucketName, key, file);
    try {
        LOGGER.info("Transfer: " + appDownload.getDescription());
        LOGGER.info("	State: " + appDownload.getState());
        LOGGER.info("	Progress: ");
        // You can poll your transfer's status to check its progress
        while (!appDownload.isDone()) {
            LOGGER.info("		transferred: " + (int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%");
            CommonUtils.pause(pollingInterval);
        }
        LOGGER.info("	State: " + appDownload.getState());
    // appDownload.waitForCompletion();
    } catch (AmazonClientException e) {
        throw new RuntimeException("File wasn't downloaded from s3. See log: ".concat(e.getMessage()));
    }
// tx.shutdownNow();
}

Example 32 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project herd by FINRAOS.

the class MockS3OperationsImpl method copyFile.

/**
 * {@inheritDoc} <p/> <p> This implementation simulates a copyFile operation. </p> <p> This method copies files in-memory. </p> <p> The result {@link Copy}
 * has the following properties: <dl> <p/> <dt>description</dt> <dd>"MockTransfer"</dd> <p/> <dt>state</dt> <dd>{@link TransferState#Completed}</dd> <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt> <dd>1024</dd> <p/> <dt>transferProgress.updateProgress</dt> <dd>1024</dd> <p/> </dl> <p/> All other
 * properties are set as default. </p> <p> This operation takes the following hints when suffixed in copyObjectRequest.sourceKey: <dl> <p/>
 * <dt>MOCK_S3_FILE_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> </dl> </p>
 */
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager) {
    LOGGER.debug("copyFile(): copyObjectRequest.getSourceBucketName() = " + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = " + copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = " + copyObjectRequest.getDestinationBucketName() + ", copyObjectRequest.getDestinationKey() = " + copyObjectRequest.getDestinationKey());
    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_SERVICE_EXCEPTION)) {
        throw new AmazonServiceException(null);
    }
    String sourceBucketName = copyObjectRequest.getSourceBucketName();
    String sourceKey = copyObjectRequest.getSourceKey();
    MockS3Bucket mockSourceS3Bucket = getOrCreateBucket(sourceBucketName);
    MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);
    if (mockSourceS3Object == null) {
        AmazonServiceException amazonServiceException = new AmazonServiceException(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        amazonServiceException.setErrorCode(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        throw amazonServiceException;
    }
    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(mockSourceS3Object.getObjectMetadata().getContentLength());
    transferProgress.updateProgress(mockSourceS3Object.getObjectMetadata().getContentLength());
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);
    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    if (copyObjectRequest.getSSEAwsKeyManagementParams() != null) {
        final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
        if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER)) {
            copy.setState(TransferState.Failed);
            copy.setMonitor(new TransferMonitor() {

                @Override
                public Future<?> getFuture() {
                    if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION)) {
                        throw new AmazonServiceException("Key '" + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId() + "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                    }
                    // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                    BasicFuture<?> future = new BasicFuture<Void>(null);
                    future.completed(null);
                    return future;
                }

                @Override
                public boolean isDone() {
                    return true;
                }
            });
        } else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER)) {
            // If the KMS indicates a cancelled transfer, just update the state to canceled.
            copy.setState(TransferState.Canceled);
        }
    }
    // If copy operation is marked as completed, perform the actual file copy in memory.
    if (copy.getState().equals(TransferState.Completed)) {
        String destinationBucketName = copyObjectRequest.getDestinationBucketName();
        String destinationObjectKey = copyObjectRequest.getDestinationKey();
        String destinationObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(destinationBucketName) ? UUID.randomUUID().toString() : null;
        String destinationObjectKeyVersion = destinationObjectKey + (destinationObjectVersion != null ? destinationObjectVersion : "");
        ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
        MockS3Object mockDestinationS3Object = new MockS3Object();
        mockDestinationS3Object.setKey(destinationObjectKey);
        mockDestinationS3Object.setVersion(destinationObjectVersion);
        mockDestinationS3Object.setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
        mockDestinationS3Object.setObjectMetadata(objectMetadata);
        MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
        mockDestinationS3Bucket.getObjects().put(destinationObjectKey, mockDestinationS3Object);
        mockDestinationS3Bucket.getVersions().put(destinationObjectKeyVersion, mockDestinationS3Object);
    }
    return copy;
}

Example 33 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project herd by FINRAOS.

the class S3DaoImplTest method testTagObjectsS3FilesListEmpty.

@Test
public void testTagObjectsS3FilesListEmpty() {
    // Create an S3 file transfer request parameters DTO to access S3 objects without specifying S3 files.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setFiles(new ArrayList<>());
    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
    // Call the method under test.
    s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag);
    // Verify the external calls.
    verifyNoMoreInteractionsHelper();
}

Example 34 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project herd by FINRAOS.

the class S3DaoImplTest method testDeleteDirectoryNoS3VersionsExist.

@Test
public void testDeleteDirectoryNoS3VersionsExist() {
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);
    // Create a retry policy.
    RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);
    // Create an empty version listing.
    VersionListing versionListing = new VersionListing();
    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
    // Call the method under test.
    s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
    // Verify the external calls.
    verify(retryPolicyFactory).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}

Example 35 with Transfer

use of com.amazonaws.services.s3.transfer.Transfer in project herd by FINRAOS.

the class S3DaoImplTest method testTagObjects.

@Test
public void testTagObjects() {
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setFiles(Arrays.asList(new File(S3_KEY_PREFIX + "/" + LOCAL_FILE)));
    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);
    // Create a retry policy.
    RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);
    // Create a get object tagging result.
    GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null);
    // Create a set object tagging result.
    SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult();
    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult);
    when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult);
    // Call the method under test.
    s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, tag);
    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}