Examples with S3FileTransferResultsDto org.finra.herd.model.dto.S3FileTransferResultsDto used on opensource projects

Example 11 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3ServiceTest method testDownloadDirectory.

@Test
public void testDownloadDirectory() throws InterruptedException {
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer result DTO.
    S3FileTransferResultsDto s3FileTransferResultsDto = new S3FileTransferResultsDto();
    // Mock the external calls.
    when(s3Dao.downloadDirectory(s3FileTransferRequestParamsDto)).thenReturn(s3FileTransferResultsDto);
    // Call the method under test.
    S3FileTransferResultsDto result = s3Service.downloadDirectory(s3FileTransferRequestParamsDto);
    // Verify the external calls.
    verify(s3Dao).downloadDirectory(s3FileTransferRequestParamsDto);
    verifyNoMoreInteractions(s3Dao);
    // Validate the returned object.
    assertEquals(s3FileTransferResultsDto, result);
}

Example 12 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoImpl method downloadFile.

@Override
public S3FileTransferResultsDto downloadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException {
    LOGGER.info("Downloading S3 file... s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath());
    // Perform the transfer.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer() {

        @Override
        public Transfer performTransfer(TransferManager transferManager) {
            return s3Operations.download(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), transferManager);
        }
    });
    LOGGER.info("Downloaded S3 file to the local system. s3Key=\"{}\" s3BucketName=\"{}\" localPath=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getS3KeyPrefix(), params.getS3BucketName(), params.getLocalPath(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
    logOverallTransferRate(results);
    return results;
}

Example 13 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoImpl method uploadFileList.

@Override
public S3FileTransferResultsDto uploadFileList(final S3FileTransferRequestParamsDto params) throws InterruptedException {
    LOGGER.info("Uploading a list of files from the local directory to S3... localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={}", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), params.getFiles().size());
    if (LOGGER.isInfoEnabled()) {
        for (File file : params.getFiles()) {
            LOGGER.info("s3Key=\"{}\"", file.getPath());
        }
    }
    // Perform the transfer.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer() {

        @Override
        public Transfer performTransfer(TransferManager transferManager) {
            return s3Operations.uploadFileList(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.getFiles(), new ObjectMetadataProvider() {

                @Override
                public void provideObjectMetadata(File file, ObjectMetadata metadata) {
                    prepareMetadata(params, metadata);
                }
            }, transferManager);
        }
    });
    LOGGER.info("Uploaded list of files from the local directory to S3. " + "localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
    logOverallTransferRate(results);
    return results;
}

Example 14 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoImpl method uploadDirectory.

@Override
public S3FileTransferResultsDto uploadDirectory(final S3FileTransferRequestParamsDto params) throws InterruptedException {
    LOGGER.info("Uploading local directory to S3... localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName());
    // Perform the transfer.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer() {

        @Override
        public Transfer performTransfer(TransferManager transferManager) {
            return s3Operations.uploadDirectory(params.getS3BucketName(), params.getS3KeyPrefix(), new File(params.getLocalPath()), params.isRecursive(), new ObjectMetadataProvider() {

                @Override
                public void provideObjectMetadata(File file, ObjectMetadata metadata) {
                    prepareMetadata(params, metadata);
                }
            }, transferManager);
        }
    });
    LOGGER.info("Uploaded local directory to S3. " + "localDirectory=\"{}\" s3KeyPrefix=\"{}\" s3BucketName=\"{}\" s3KeyCount={} totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalFilesTransferred(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
    logOverallTransferRate(results);
    return results;
}

Example 15 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoImpl method uploadFile.

@Override
public S3FileTransferResultsDto uploadFile(final S3FileTransferRequestParamsDto params) throws InterruptedException {
    LOGGER.info("Uploading local file to S3... localPath=\"{}\" s3Key=\"{}\" s3BucketName=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName());
    // Perform the transfer.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer() {

        @Override
        public Transfer performTransfer(TransferManager transferManager) {
            // Get a handle to the local file.
            File localFile = new File(params.getLocalPath());
            // Create and prepare the metadata.
            ObjectMetadata metadata = new ObjectMetadata();
            prepareMetadata(params, metadata);
            // Create a put request and a transfer manager with the parameters and the metadata.
            PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), params.getS3KeyPrefix(), localFile);
            putObjectRequest.setMetadata(metadata);
            return s3Operations.upload(putObjectRequest, transferManager);
        }
    });
    LOGGER.info("Uploaded local file to the S3. localPath=\"{}\" s3Key=\"{}\" s3BucketName=\"{}\" totalBytesTransferred={} transferDuration=\"{}\"", params.getLocalPath(), params.getS3KeyPrefix(), params.getS3BucketName(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
    logOverallTransferRate(results);
    return results;
}