Examples with StorageFile org.finra.herd.model.api.xml.StorageFile used on opensource projects

Example 71 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreServiceImplTest method testFinalizeRestore.

@Test
public void testFinalizeRestore() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage unit key.
    BusinessObjectDataStorageUnitKey storageUnitKey = new BusinessObjectDataStorageUnitKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, STORAGE_NAME);
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT)), NO_EXCEPTION);
    // Mock the external calls.
    when(businessObjectDataFinalizeRestoreHelperService.prepareToFinalizeRestore(storageUnitKey)).thenReturn(businessObjectDataRestoreDto);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the parameters DTO for the business object data restore.
            BusinessObjectDataRestoreDto businessObjectDataRestoreDto = (BusinessObjectDataRestoreDto) invocation.getArguments()[0];
            // Set the value for the new and old storage unit statuses.
            businessObjectDataRestoreDto.setNewStorageUnitStatus(StorageUnitStatusEntity.RESTORED);
            businessObjectDataRestoreDto.setOldStorageUnitStatus(StorageUnitStatusEntity.RESTORING);
            return null;
        }
    }).when(businessObjectDataFinalizeRestoreHelperService).completeFinalizeRestore(businessObjectDataRestoreDto);
    // Call the method under test.
    businessObjectDataFinalizeRestoreServiceImpl.finalizeRestore(storageUnitKey);
    // Verify the external calls.
    verify(businessObjectDataFinalizeRestoreHelperService).prepareToFinalizeRestore(storageUnitKey);
    verify(businessObjectDataFinalizeRestoreHelperService).executeS3SpecificSteps(businessObjectDataRestoreDto);
    verify(businessObjectDataFinalizeRestoreHelperService).completeFinalizeRestore(businessObjectDataRestoreDto);
    verify(notificationEventService).processStorageUnitNotificationEventAsync(NotificationEventTypeEntity.EventTypesStorageUnit.STRGE_UNIT_STTS_CHG, businessObjectDataKey, STORAGE_NAME, StorageUnitStatusEntity.RESTORED, StorageUnitStatusEntity.RESTORING);
    verifyNoMoreInteractionsHelper();
}

Example 72 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceImplTest method testCompleteFinalizeRestore.

@Test
public void testCompleteFinalizeRestore() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT)), NO_EXCEPTION);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    // Create a new storage unit status entity.
    StorageUnitStatusEntity newStorageUnitStatusEntity = new StorageUnitStatusEntity();
    newStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.RESTORED);
    // Create an old storage unit status entity.
    StorageUnitStatusEntity oldStorageUnitStatusEntity = new StorageUnitStatusEntity();
    oldStorageUnitStatusEntity.setCode(StorageUnitStatusEntity.RESTORING);
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStatus(oldStorageUnitStatusEntity);
    // Mock the external calls.
    when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity);
    when(storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity)).thenReturn(storageUnitEntity);
    when(storageUnitStatusDaoHelper.getStorageUnitStatusEntity(StorageUnitStatusEntity.RESTORED)).thenReturn(newStorageUnitStatusEntity);
    // Call the method under test.
    businessObjectDataFinalizeRestoreHelperServiceImpl.completeFinalizeRestore(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
    verify(storageUnitDaoHelper).getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    verify(storageUnitStatusDaoHelper).getStorageUnitStatusEntity(StorageUnitStatusEntity.RESTORED);
    verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, newStorageUnitStatusEntity, StorageUnitStatusEntity.RESTORED);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(StorageUnitStatusEntity.RESTORED, businessObjectDataRestoreDto.getNewStorageUnitStatus());
    assertEquals(StorageUnitStatusEntity.RESTORING, businessObjectDataRestoreDto.getOldStorageUnitStatus());
}

Example 73 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class BusinessObjectDataInitiateDestroyHelperServiceImplTest method testPrepareToInitiateDestroy.

@Test
public void testPrepareToInitiateDestroy() {
    // Create an empty business object data destroy parameters DTO.
    BusinessObjectDataDestroyDto businessObjectDataDestroyDto = new BusinessObjectDataDestroyDto();
    // Create a version-less business object format key.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, NO_FORMAT_VERSION);
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a retention type entity.
    RetentionTypeEntity retentionTypeEntity = new RetentionTypeEntity();
    retentionTypeEntity.setCode(RetentionTypeEntity.PARTITION_VALUE);
    // Create a business object format entity.
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    businessObjectFormatEntity.setLatestVersion(true);
    businessObjectFormatEntity.setRetentionType(retentionTypeEntity);
    businessObjectFormatEntity.setRetentionPeriodInDays(RETENTION_PERIOD_DAYS);
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.VALID);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    businessObjectDataEntity.setBusinessObjectFormat(businessObjectFormatEntity);
    businessObjectDataEntity.setPartitionValue(PARTITION_VALUE);
    businessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a storage platform entity.
    StoragePlatformEntity storagePlatformEntity = new StoragePlatformEntity();
    storagePlatformEntity.setName(StoragePlatformEntity.S3);
    // Create a storage entity.
    StorageEntity storageEntity = new StorageEntity();
    storageEntity.setStoragePlatform(storagePlatformEntity);
    storageEntity.setName(STORAGE_NAME);
    // Create a list of storage file entities.
    List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity());
    // Create a storage unit status entity.
    StorageUnitStatusEntity storageUnitStatusEntity = new StorageUnitStatusEntity();
    storageUnitStatusEntity.setCode(StorageUnitStatusEntity.ENABLED);
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitEntity.setStorageFiles(storageFileEntities);
    storageUnitEntity.setStatus(storageUnitStatusEntity);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Create a current timestamp.
    Timestamp currentTimestamp = new Timestamp(new Date().getTime());
    // Create a date representing the primary partition value that satisfies the retention threshold check.
    Calendar calendar = Calendar.getInstance();
    calendar.add(Calendar.DATE, -(RETENTION_PERIOD_DAYS + 1));
    Date primaryPartitionValueDate = calendar.getTime();
    // Mock the external calls.
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_KEY)).thenReturn(S3_OBJECT_TAG_KEY);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_VALUE)).thenReturn(S3_OBJECT_TAG_VALUE);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_ARN)).thenReturn(S3_OBJECT_TAGGER_ROLE_ARN);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_SESSION_NAME)).thenReturn(S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    when(herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.BDATA_FINAL_DESTROY_DELAY_IN_DAYS)).thenReturn(BDATA_FINAL_DESTROY_DELAY_IN_DAYS);
    when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity);
    when(businessObjectDataHelper.getDateFromString(PARTITION_VALUE)).thenReturn(primaryPartitionValueDate);
    when(herdDao.getCurrentTimestamp()).thenReturn(currentTimestamp);
    when(storageUnitDao.getStorageUnitsByStoragePlatformAndBusinessObjectData(StoragePlatformEntity.S3, businessObjectDataEntity)).thenReturn(Arrays.asList(storageUnitEntity));
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX)).thenReturn(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX);
    when(storageHelper.getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX, storageEntity, false, true)).thenReturn(true);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME)).thenReturn(S3_ATTRIBUTE_NAME_BUCKET_NAME);
    when(storageHelper.getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true)).thenReturn(S3_BUCKET_NAME);
    when(s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey)).thenReturn(TEST_S3_KEY_PREFIX);
    when(storageFileHelper.getAndValidateStorageFilesIfPresent(storageUnitEntity, TEST_S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey)).thenReturn(storageFiles);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the new storage unit status.
            String storageUnitStatus = (String) invocation.getArguments()[1];
            // Create a storage unit status entity for the new storage unit status.
            StorageUnitStatusEntity storageUnitStatusEntity = new StorageUnitStatusEntity();
            storageUnitStatusEntity.setCode(storageUnitStatus);
            // Update the storage unit with the new status.
            StorageUnitEntity storageUnitEntity = (StorageUnitEntity) invocation.getArguments()[0];
            storageUnitEntity.setStatus(storageUnitStatusEntity);
            return null;
        }
    }).when(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.DISABLING);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the new storage unit status.
            String businessObjectDataStatus = (String) invocation.getArguments()[1];
            // Create a business object data status entity for the new business object data status.
            BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
            businessObjectDataStatusEntity.setCode(businessObjectDataStatus);
            // Update the business object data entity with the new status.
            BusinessObjectDataEntity businessObjectDataEntity = (BusinessObjectDataEntity) invocation.getArguments()[0];
            businessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
            return null;
        }
    }).when(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(businessObjectDataEntity, BusinessObjectDataStatusEntity.DELETED);
    when(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)).thenReturn(businessObjectDataKey);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT)).thenReturn(S3_ENDPOINT);
    // Call the method under test.
    businessObjectDataInitiateDestroyHelperServiceImpl.prepareToInitiateDestroy(businessObjectDataDestroyDto, businessObjectDataKey);
    // Verify the external calls.
    verify(businessObjectDataHelper).validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_KEY);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_TAG_VALUE);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_ARN);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_OBJECT_DELETE_ROLE_SESSION_NAME);
    verify(herdStringHelper).getConfigurationValueAsInteger(ConfigurationValue.BDATA_FINAL_DESTROY_DELAY_IN_DAYS);
    verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
    verify(businessObjectDataHelper).getDateFromString(PARTITION_VALUE);
    verify(herdDao).getCurrentTimestamp();
    verify(storageUnitDao).getStorageUnitsByStoragePlatformAndBusinessObjectData(StoragePlatformEntity.S3, businessObjectDataEntity);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX);
    verify(storageHelper).getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX, storageEntity, false, true);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME);
    verify(storageHelper).getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true);
    verify(s3KeyPrefixHelper).buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey);
    verify(storageFileHelper).getAndValidateStorageFilesIfPresent(storageUnitEntity, TEST_S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileDaoHelper).validateStorageFilesCount(STORAGE_NAME, businessObjectDataKey, TEST_S3_KEY_PREFIX, storageFileEntities.size());
    verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.DISABLING);
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(businessObjectDataEntity, BusinessObjectDataStatusEntity.DELETED);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(businessObjectDataEntity);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ENDPOINT);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS), businessObjectDataDestroyDto);
}

Example 74 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method testValidateRegisteredS3Files.

@Test
public void testValidateRegisteredS3Files() throws IOException {
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES) {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }
    // Validate the files.
    storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION));
}

Example 75 with StorageFile

use of org.finra.herd.model.api.xml.StorageFile in project herd by FINRAOS.

the class StorageFileHelperTest method createStorageUnit.

/**
 * Creates a storage unit with the specified list of files.
 *
 * @param s3KeyPrefix the S3 key prefix.
 * @param files the list of files.
 * @param fileSizeBytes the file size in bytes.
 *
 * @return the storage unit with the list of file paths
 */
private StorageUnit createStorageUnit(String s3KeyPrefix, List<String> files, Long fileSizeBytes) {
    StorageUnit storageUnit = new StorageUnit();
    Storage storage = new Storage();
    storageUnit.setStorage(storage);
    storage.setName("TEST_STORAGE");
    List<StorageFile> storageFiles = new ArrayList<>();
    storageUnit.setStorageFiles(storageFiles);
    if (!CollectionUtils.isEmpty(files)) {
        for (String file : files) {
            StorageFile storageFile = new StorageFile();
            storageFiles.add(storageFile);
            storageFile.setFilePath(String.format("%s/%s", s3KeyPrefix, file));
            storageFile.setFileSizeBytes(fileSizeBytes);
        }
    }
    storageUnit.setStorageUnitStatus(StorageUnitStatusEntity.ENABLED);
    return storageUnit;
}