Examples with S3FileTransferResultsDto org.finra.herd.model.dto.S3FileTransferResultsDto used on opensource projects

Example 21 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoTest method testDeleteDirectoryBucketVersioningEnabled.

/**
 * Test that we are able to perform the deleteDirectory S3Dao operation on S3 with enabled versioning using our DAO tier.
 */
@Test
public void testDeleteDirectoryBucketVersioningEnabled() throws IOException, InterruptedException {
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = null;
    try {
        // Create local test files.
        for (String file : LOCAL_FILES) {
            createLocalFile(localTempPath.toString(), file, FILE_SIZE_1_KB);
        }
        // Upload twice the same set of test files to an S3 bucket with enabled versioning.
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
        s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
        s3FileTransferRequestParamsDto.setRecursive(true);
        for (int i = 0; i < 2; i++) {
            S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
            Assert.assertEquals(Long.valueOf(LOCAL_FILES.size()), results.getTotalFilesTransferred());
        }
        // Validate the existence of keys and key versions for the uploaded test files in S3.
        Assert.assertEquals(LOCAL_FILES.size(), s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
        Assert.assertEquals(LOCAL_FILES.size() * 2, s3Dao.listVersions(s3FileTransferRequestParamsDto).size());
        // Delete directory from S3 using s3Dao.
        s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        // Validate that S3 directory got deleted.
        Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
        Assert.assertEquals(0, s3Dao.listVersions(s3FileTransferRequestParamsDto).size());
    } catch (Exception e) {
        // Clean up the S3 on failure.
        if (s3FileTransferRequestParamsDto != null) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
    }
}

Example 22 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadDirectoryZeroBytes.

/**
 * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier.
 */
@Test
public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException {
    // Create a zero size local file.
    File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE);
    Assert.assertTrue(targetFile.isFile());
    // Upload empty folder to s3Dao.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX);
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == 1L);
    // Validate the zero bytes upload.
    Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
}

Example 23 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadDirectoryLargeNumberOfFiles.

/**
 * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with more than 1000 files using our DAO tier. This test is needed to
 * make AmazonS3.listObjects() to start truncating ObjectListing that it returns.
 */
@Test
public void testUploadDirectoryLargeNumberOfFiles() throws IOException, InterruptedException {
    final int NUM_OF_FILES = 1001;
    // Create local files.
    for (int i = 0; i < NUM_OF_FILES; i++) {
        File targetFile = createLocalFile(localTempPath.toString(), String.format("%04d_%s", i, LOCAL_FILE), FILE_SIZE_0_BYTE);
        Assert.assertTrue(targetFile.isFile());
    }
    // Upload empty folder to s3Dao.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX);
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == NUM_OF_FILES);
    // Validate the empty folder upload.
    Assert.assertEquals(NUM_OF_FILES, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
}

Example 24 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class S3DaoTest method testCopyFileNoKmsId.

/**
 * Test S3 file copy without a KMS ID specified.
 */
@Test
public void testCopyFileNoKmsId() throws InterruptedException {
    // Put a 1 byte file in S3.
    s3Operations.putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null);
    // Perform an S3 file copy operation when KMS ID value is not specified.
    for (String kmsId : Arrays.asList(BLANK_TEXT, null)) {
        S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto();
        transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName());
        transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName());
        transferDto.setSourceObjectKey(TARGET_S3_KEY);
        transferDto.setTargetObjectKey(TARGET_S3_KEY);
        transferDto.setKmsKeyId(kmsId);
        S3FileTransferResultsDto resultsDto = s3Dao.copyFile(transferDto);
        assertEquals(Long.valueOf(1L), resultsDto.getTotalFilesTransferred());
    }
}

Example 25 with S3FileTransferResultsDto

use of org.finra.herd.model.dto.S3FileTransferResultsDto in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method prepareTestS3Files.

/**
 * Creates specified list of files in the local temporary directory and uploads them to the test S3 bucket. This method also creates 0 byte S3 directory
 * markers relative to the s3 key prefix.
 *
 * @param bucketName the bucket name in S3 to place the files.
 * @param s3KeyPrefix the destination S3 key prefix
 * @param localTempPath the local temporary directory
 * @param localFilePaths the list of local files that might include sub-directories
 * @param directoryPaths the list of directory paths to be created in S3 relative to the S3 key prefix
 *
 * @throws Exception
 */
public void prepareTestS3Files(String bucketName, String s3KeyPrefix, Path localTempPath, List<String> localFilePaths, List<String> directoryPaths) throws Exception {
    // Create local test files.
    for (String file : localFilePaths) {
        AbstractServiceTest.createLocalFile(localTempPath.toString(), file, AbstractServiceTest.FILE_SIZE_1_KB);
    }
    // Upload test file to S3.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    if (bucketName != null) {
        s3FileTransferRequestParamsDto.setS3BucketName(bucketName);
    }
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Service.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate the transfer result.
    assertEquals(Long.valueOf(localFilePaths.size()), results.getTotalFilesTransferred());
    // Create 0 byte S3 directory markers.
    for (String directoryPath : directoryPaths) {
        // Create 0 byte directory marker.
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + "/" + directoryPath);
        s3Service.createDirectory(s3FileTransferRequestParamsDto);
    }
    // Validate the uploaded S3 files and created directory markers, if any.
    s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);
    assertEquals(localFilePaths.size() + directoryPaths.size(), s3Service.listDirectory(s3FileTransferRequestParamsDto).size());
}