Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 11 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3FilesActualFileNoExists.

@Test
public void testValidateRegisteredS3FilesActualFileNoExists() throws IOException {
    // Create two lists of expected and actual storage files, with one expected file not being added to the list of actual files.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, ROW_COUNT_1000));
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    // Try to validate S3 files when expected S3 file does not exist.
    try {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an ObjectNotFoundException when the registered S3 file does not exist.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", TARGET_S3_KEY, STORAGE_NAME), e.getMessage());
    }
}

Example 12 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class ExpireRestoredBusinessObjectDataHelperServiceImplTest method testCompleteStorageUnitExpiration.

@Test
public void testCompleteStorageUnitExpiration() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT)), NO_EXCEPTION);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    // Create a new storage unit status entity.
    StorageUnitStatusEntity newStorageUnitStatusEntity = new StorageUnitStatusEntity();
    newStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.ARCHIVED);
    // Create an old storage unit status entity.
    StorageUnitStatusEntity oldStorageUnitStatusEntity = new StorageUnitStatusEntity();
    oldStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.EXPIRING);
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStatus(oldStorageUnitStatusEntity);
    // Mock the external calls.
    when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity);
    when(storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity)).thenReturn(storageUnitEntity);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the storage unit entity.
            StorageUnitEntity storageUnitEntity = (StorageUnitEntity) invocation.getArguments()[0];
            // Update the storage unit status.
            storageUnitEntity.setStatus(newStorageUnitStatusEntity);
            return null;
        }
    }).when(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVED, StorageUnitStatusEntity.ARCHIVED);
    // Call the method under test.
    expireRestoredBusinessObjectDataHelperServiceImpl.completeStorageUnitExpiration(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
    verify(storageUnitDaoHelper).getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVED, StorageUnitStatusEntity.ARCHIVED);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(StorageUnitStatusEntity.ARCHIVED, businessObjectDataRestoreDto.getNewStorageUnitStatus());
    assertEquals(StorageUnitStatusEntity.EXPIRING, businessObjectDataRestoreDto.getOldStorageUnitStatus());
}

Example 13 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class ExpireRestoredBusinessObjectDataHelperServiceImplTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT), new StorageFile(S3_KEY_2, FILE_SIZE_2, ROW_COUNT_2));
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, STORAGE_UNIT_STATUS_2, STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    // Create an initial instance of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an updated version of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    // Create a mock S3 object summary for S3 object that belongs to Glacier storage class.
    S3ObjectSummary glacierS3ObjectSummary = mock(S3ObjectSummary.class);
    when(glacierS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary standardS3ObjectSummary = mock(S3ObjectSummary.class);
    when(standardS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
    // Create a list of S3 files.
    List<S3ObjectSummary> s3Files = Arrays.asList(glacierS3ObjectSummary, standardS3ObjectSummary);
    // Create a list of S3 objects that belong to Glacier storage class.
    List<S3ObjectSummary> glacierS3Files = Arrays.asList(glacierS3ObjectSummary);
    // Create a list of storage files that represent S3 objects of Glacier storage class.
    List<StorageFile> glacierStorageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a list of files.
    List<File> files = Arrays.asList(new File(S3_KEY));
    // Create a final version of DTO for business object data restore parameters.
    S3FileTransferRequestParamsDto finalS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    finalS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    finalS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    finalS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    finalS3FileTransferRequestParamsDto.setFiles(files);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles);
    when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(files);
    // Call the method under test.
    expireRestoredBusinessObjectDataHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true));
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
    verify(storageFileHelper).getFiles(glacierStorageFiles);
    verify(s3Service).restoreObjects(finalS3FileTransferRequestParamsDto, 1);
    verifyNoMoreInteractionsHelper();
}

Example 14 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class ExpireRestoredBusinessObjectDataHelperServiceImplTest method testPrepareToExpireStorageUnit.

@Test
public void testPrepareToExpireStorageUnit() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = new BusinessObjectDataStorageUnitKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME);
    // Create a storage entity.
    StorageEntity storageEntity = new StorageEntity();
    storageEntity.setName(STORAGE_NAME);
    // Create a business object format entity.
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    businessObjectDataEntity.setBusinessObjectFormat(businessObjectFormatEntity);
    // Create a new storage unit status entity.
    StorageUnitStatusEntity newStorageUnitStatusEntity = new StorageUnitStatusEntity();
    newStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.EXPIRING);
    // Create an old storage unit status entity.
    StorageUnitStatusEntity oldStorageUnitStatusEntity = new StorageUnitStatusEntity();
    oldStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.RESTORED);
    // Create a list of storage file entities.
    List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity());
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitEntity.setStatus(oldStorageUnitStatusEntity);
    storageUnitEntity.setStorageFiles(storageFileEntities);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Mock the external calls.
    when(businessObjectDataHelper.createBusinessObjectDataKeyFromStorageUnitKey(storageUnitKey)).thenReturn(businessObjectDataKey);
    when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity);
    when(storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity)).thenReturn(storageUnitEntity);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME)).thenReturn(S3_ATTRIBUTE_NAME_BUCKET_NAME);
    when(storageHelper.getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true)).thenReturn(S3_BUCKET_NAME);
    when(s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey)).thenReturn(S3_KEY_PREFIX);
    when(storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey)).thenReturn(storageFiles);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the storage unit entity.
            StorageUnitEntity storageUnitEntity = (StorageUnitEntity) invocation.getArguments()[0];
            // Update the storage unit status.
            storageUnitEntity.setStatus(newStorageUnitStatusEntity);
            return null;
        }
    }).when(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.EXPIRING, StorageUnitStatusEntity.EXPIRING);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT)).thenReturn(S3_ENDPOINT);
    // Call the method under test.
    BusinessObjectDataRestoreDto result = expireRestoredBusinessObjectDataHelperServiceImpl.prepareToExpireStorageUnit(storageUnitKey);
    // Verify the external calls.
    verify(businessObjectDataHelper).createBusinessObjectDataKeyFromStorageUnitKey(storageUnitKey);
    verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
    verify(storageUnitDaoHelper).getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME);
    verify(storageHelper).getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true);
    verify(s3KeyPrefixHelper).buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey);
    verify(storageFileHelper).getAndValidateStorageFiles(storageUnitEntity, S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileDaoHelper).validateStorageFilesCount(STORAGE_NAME, businessObjectDataKey, S3_KEY_PREFIX, storageFiles.size());
    verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.EXPIRING, StorageUnitStatusEntity.EXPIRING);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ENDPOINT);
    verifyNoMoreInteractionsHelper();
    // Validate the result.
    assertEquals(new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, StorageUnitStatusEntity.EXPIRING, StorageUnitStatusEntity.RESTORED, storageFiles, NO_EXCEPTION), result);
}

Example 15 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImplTest method testExecuteStoragePolicyTransitionImpl.

@Test
public void testExecuteStoragePolicyTransitionImpl() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage file path.
    String storageFilePath = TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create a list of storage files to be passed as an input.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a storage policy transition parameters DTO.
    StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto = new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create a list of S3 object summaries selected without zero byte directory markers.
    List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = Arrays.asList(new S3ObjectSummary());
    // Create a list of all S3 files matching the S3 key prefix form the S3 bucket.
    List<S3ObjectSummary> actualS3Files = Arrays.asList(new S3ObjectSummary());
    // Create a list of storage files selected for S3 object tagging.
    List<StorageFile> storageFilesSelectedForTagging = Arrays.asList(new StorageFile());
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForTagging = Arrays.asList(new File(storageFilePath));
    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    updatedS3FileTransferRequestParamsDto.setFiles(filesSelectedForTagging);
    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME)).thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(actualS3FilesWithoutZeroByteDirectoryMarkers);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(actualS3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesSelectedForTagging);
    when(storageFileHelper.getFiles(storageFilesSelectedForTagging)).thenReturn(filesSelectedForTagging);
    // Call the method under test.
    storagePolicyProcessorHelperServiceImpl.executeStoragePolicyTransitionImpl(storagePolicyTransitionParamsDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, actualS3FilesWithoutZeroByteDirectoryMarkers, STORAGE_NAME, businessObjectDataKey);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, true);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, false);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
    verify(storageFileHelper).getFiles(storageFilesSelectedForTagging);
    verify(s3Service).tagObjects(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME), storagePolicyTransitionParamsDto);
}