Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 6 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceImplTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT), new StorageFile(S3_KEY_2, FILE_SIZE_2, ROW_COUNT_2));
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, STORAGE_UNIT_STATUS_2, STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    // Create an initial instance of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an updated version of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    // Create a mock S3 object summary for S3 object that belongs to Glacier storage class.
    S3ObjectSummary glacierS3ObjectSummary = mock(S3ObjectSummary.class);
    when(glacierS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary standardS3ObjectSummary = mock(S3ObjectSummary.class);
    when(standardS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
    // Create a list of S3 files.
    List<S3ObjectSummary> s3Files = Arrays.asList(glacierS3ObjectSummary, standardS3ObjectSummary);
    // Create a list of S3 objects that belong to Glacier storage class.
    List<S3ObjectSummary> glacierS3Files = Arrays.asList(glacierS3ObjectSummary);
    // Create a list of storage files that represent S3 objects of Glacier storage class.
    List<StorageFile> glacierStorageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a list of files.
    List<File> files = Arrays.asList(new File(S3_KEY));
    // Create a final version of DTO for business object data restore parameters.
    S3FileTransferRequestParamsDto finalS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    finalS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    finalS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    finalS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    finalS3FileTransferRequestParamsDto.setFiles(files);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles);
    when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(files);
    // Call the method under test.
    businessObjectDataFinalizeRestoreHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true));
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
    verify(storageFileHelper).getFiles(glacierStorageFiles);
    verify(s3Service).validateGlacierS3FilesRestored(finalS3FileTransferRequestParamsDto);
    verifyNoMoreInteractionsHelper();
}

Example 7 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataInitiateDestroyHelperServiceImplTest method runExecuteS3SpecificStepsTest.

private void runExecuteS3SpecificStepsTest() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage file path.
    String storageFilePath = TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create a business object data destroy parameters DTO.
    BusinessObjectDataDestroyDto businessObjectDataDestroyDto = new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create a list of all S3 files matching the S3 key prefix form the S3 bucket.
    List<S3ObjectSummary> actualS3Files = Arrays.asList(new S3ObjectSummary());
    // Create a list of storage files selected for S3 object tagging.
    List<StorageFile> storageFilesSelectedForTagging = Arrays.asList(new StorageFile());
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForTagging = Arrays.asList(new File(storageFilePath));
    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    updatedS3FileTransferRequestParamsDto.setFiles(filesSelectedForTagging);
    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME)).thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(actualS3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesSelectedForTagging);
    when(storageFileHelper.getFiles(storageFilesSelectedForTagging)).thenReturn(filesSelectedForTagging);
    // Call the method under test.
    businessObjectDataInitiateDestroyHelperServiceImpl.executeS3SpecificSteps(businessObjectDataDestroyDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, false);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
    verify(storageFileHelper).getFiles(storageFilesSelectedForTagging);
    verify(s3Service).tagObjects(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS), businessObjectDataDestroyDto);
}

Example 8 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class DownloaderController method createDownloaderOutputManifestDto.

/**
 * Creates a downloader output manifest instance from the business object data.
 *
 * @param businessObjectData the business object data that we need to create the manifest for
 * @param storageUnit the S3 storage unit for this business object data
 * @param s3KeyPrefix the S3 key prefix for this business object data
 *
 * @return the created downloader output manifest instance
 */
protected DownloaderOutputManifestDto createDownloaderOutputManifestDto(BusinessObjectData businessObjectData, StorageUnit storageUnit, String s3KeyPrefix) {
    DownloaderOutputManifestDto downloaderOutputManifestDto = new DownloaderOutputManifestDto();
    // Populate basic fields.
    downloaderOutputManifestDto.setNamespace(businessObjectData.getNamespace());
    downloaderOutputManifestDto.setBusinessObjectDefinitionName(businessObjectData.getBusinessObjectDefinitionName());
    downloaderOutputManifestDto.setBusinessObjectFormatUsage(businessObjectData.getBusinessObjectFormatUsage());
    downloaderOutputManifestDto.setBusinessObjectFormatFileType(businessObjectData.getBusinessObjectFormatFileType());
    downloaderOutputManifestDto.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion()));
    downloaderOutputManifestDto.setPartitionKey(businessObjectData.getPartitionKey());
    downloaderOutputManifestDto.setPartitionValue(businessObjectData.getPartitionValue());
    downloaderOutputManifestDto.setSubPartitionValues(businessObjectData.getSubPartitionValues());
    downloaderOutputManifestDto.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion()));
    downloaderOutputManifestDto.setStorageName(storageUnit.getStorage().getName());
    // Build a list of manifest files with paths relative to the S3 key prefix.
    List<ManifestFile> manifestFiles = new ArrayList<>();
    downloaderOutputManifestDto.setManifestFiles(manifestFiles);
    if (!CollectionUtils.isEmpty(storageUnit.getStorageFiles())) {
        for (StorageFile storageFile : storageUnit.getStorageFiles()) {
            ManifestFile manifestFile = new ManifestFile();
            manifestFiles.add(manifestFile);
            manifestFile.setFileName(storageFile.getFilePath().replace(s3KeyPrefix, ""));
            manifestFile.setFileSizeBytes(storageFile.getFileSizeBytes());
            manifestFile.setRowCount(storageFile.getRowCount());
        }
    }
    // Populate the attributes.
    HashMap<String, String> attributes = new HashMap<>();
    if (!CollectionUtils.isEmpty(businessObjectData.getAttributes())) {
        for (Attribute attribute : businessObjectData.getAttributes()) {
            attributes.put(attribute.getName(), attribute.getValue());
        }
    }
    downloaderOutputManifestDto.setAttributes(attributes);
    // Populate the business object data parents and children.
    downloaderOutputManifestDto.setBusinessObjectDataParents(businessObjectData.getBusinessObjectDataParents());
    downloaderOutputManifestDto.setBusinessObjectDataChildren(businessObjectData.getBusinessObjectDataChildren());
    return downloaderOutputManifestDto;
}

Example 9 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3FilesUnexpectedS3FileFound.

@Test
public void testValidateRegisteredS3FilesUnexpectedS3FileFound() throws IOException {
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Try to validate S3 files when unexpected S3 file exists.
    try {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating registered S3 files. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}

Example 10 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3FilesActualFileSizeMismatch.

@Test
public void testValidateRegisteredS3FilesActualFileSizeMismatch() throws IOException {
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));
    // Try to validate S3 files when expected expected file size does not match actual file size.
    try {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Specified file size of %d bytes for registered \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.", FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}