Examples with StoragePolicyEntity org.finra.herd.model.jpa.StoragePolicyEntity used on opensource projects

Example 31 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicyServiceTest method testUpdateStoragePolicyTrimParameters.

@Test
public void testUpdateStoragePolicyTrimParameters() {
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist the relative database entities.
    storagePolicyServiceTestHelper.createDatabaseEntitiesForStoragePolicyTesting();
    // Create and persist a storage policy entity.
    StoragePolicyEntity storagePolicyEntity = storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, STORAGE_POLICY_RULE_TYPE_2, STORAGE_POLICY_RULE_VALUE_2, BDEF_NAMESPACE_2, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, STORAGE_NAME_2, STORAGE_POLICY_TRANSITION_TYPE_2, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Update a storage policy using input parameters with leading and trailing empty spaces.
    StoragePolicy resultStoragePolicy = storagePolicyService.updateStoragePolicy(new StoragePolicyKey(addWhitespace(STORAGE_POLICY_NAMESPACE_CD), addWhitespace(STORAGE_POLICY_NAME)), storagePolicyServiceTestHelper.createStoragePolicyUpdateRequest(addWhitespace(STORAGE_POLICY_RULE_TYPE), STORAGE_POLICY_RULE_VALUE, addWhitespace(BDEF_NAMESPACE), addWhitespace(BDEF_NAME), addWhitespace(FORMAT_USAGE_CODE), addWhitespace(FORMAT_FILE_TYPE_CODE), addWhitespace(STORAGE_NAME), addWhitespace(STORAGE_POLICY_TRANSITION_TYPE), addWhitespace(StoragePolicyStatusEntity.DISABLED)));
    // Validate the returned object.
    assertEquals(new StoragePolicy(resultStoragePolicy.getId(), new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME), new StoragePolicyRule(STORAGE_POLICY_RULE_TYPE, STORAGE_POLICY_RULE_VALUE), new StoragePolicyFilter(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME), new StoragePolicyTransition(STORAGE_POLICY_TRANSITION_TYPE), StoragePolicyStatusEntity.DISABLED), resultStoragePolicy);
    assertTrue(resultStoragePolicy.getId() > storagePolicyEntity.getId());
}

Example 32 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicySelectorServiceImpl method execute.

@Override
public List<StoragePolicySelection> execute(String sqsQueueName, int maxResult) {
    // Create a result list.
    List<StoragePolicySelection> storagePolicySelections = new ArrayList<>();
    // Get the current timestamp from the database.
    Timestamp currentTimestamp = herdDao.getCurrentTimestamp();
    // Get the threshold in days since business object data registration update for business object data to be selectable
    // by a storage policy with DAYS_SINCE_BDATA_PRIMARY_PARTITION_VALUE storage policy rule type.
    int updatedOnThresholdInDays = herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS);
    // Get the maximum number of failed storage policy transition attempts before the relative storage unit gets excluded from
    // being selected per storage policies by the storage policy selector system job. 0 means the maximum is not set.
    int maxAllowedTransitionAttempts = herdStringHelper.getConfigurationValueAsInteger(ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS);
    LOGGER.info("{}={} {}={}", ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS.getKey(), updatedOnThresholdInDays, ConfigurationValue.STORAGE_POLICY_TRANSITION_MAX_ALLOWED_ATTEMPTS.getKey(), maxAllowedTransitionAttempts);
    // Compute business object data "updated on" threshold timestamp based on
    // the current database timestamp and the threshold value configured in the system.
    Timestamp updatedOnThresholdTimestamp = HerdDateUtils.addDays(currentTimestamp, -updatedOnThresholdInDays);
    // Keep track of all business object data entities selected per storage policies. This is need to avoid a lower priority selection policy
    // to be executed ahead of a higher priority one.
    Set<BusinessObjectDataEntity> selectedBusinessObjectDataEntities = new LinkedHashSet<>();
    // policies will be listed earlier in the final result map.
    for (StoragePolicyPriorityLevel storagePolicyPriorityLevel : STORAGE_POLICY_PRIORITY_LEVELS) {
        // Until we reach maximum number of results or run out of entities to select, retrieve and process business object data entities mapped to their
        // corresponding storage policy entities, where the business object data status is supported by the storage policy feature and the business object
        // data alternate key values match storage policy's filter and transition (not taking into account storage policy rules).
        int startPosition = 0;
        while (true) {
            Map<BusinessObjectDataEntity, StoragePolicyEntity> map = businessObjectDataDao.getBusinessObjectDataEntitiesMatchingStoragePolicies(storagePolicyPriorityLevel, SUPPORTED_BUSINESS_OBJECT_DATA_STATUSES, maxAllowedTransitionAttempts, startPosition, maxResult);
            for (Map.Entry<BusinessObjectDataEntity, StoragePolicyEntity> entry : map.entrySet()) {
                BusinessObjectDataEntity businessObjectDataEntity = entry.getKey();
                // Process this storage policy selection, only if this business object data has not been selected earlier.
                if (!selectedBusinessObjectDataEntities.contains(businessObjectDataEntity)) {
                    boolean createStoragePolicySelection = false;
                    // Remember that we got this business object data entity as matching to a storage policy.
                    // This is done so we would not try to select this business object data again later by a lower level storage policy.
                    selectedBusinessObjectDataEntities.add(businessObjectDataEntity);
                    // Get the storage policy entity, so we can validate the storage policy rule against this business object data.
                    StoragePolicyEntity storagePolicyEntity = entry.getValue();
                    // Get a storage policy rule type and value.
                    String storagePolicyRuleType = storagePolicyEntity.getStoragePolicyRuleType().getCode();
                    Integer storagePolicyRuleValue = storagePolicyEntity.getStoragePolicyRuleValue();
                    // For DAYS_SINCE_BDATA_REGISTERED storage policy rule type, select business object data based on it's "created on" timestamp.
                    if (StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED.equals(storagePolicyRuleType)) {
                        // Compute "created on" threshold timestamp based on the current timestamp and storage policy rule value.
                        Timestamp createdOnThresholdTimestamp = HerdDateUtils.addDays(currentTimestamp, -storagePolicyRuleValue);
                        // Select this business object data if it has "created on" timestamp before or equal to the threshold timestamp.
                        createStoragePolicySelection = (businessObjectDataEntity.getCreatedOn().compareTo(createdOnThresholdTimestamp) <= 0);
                    } else // partition value compared against storage policy rule value and "updated on" timestamp being below the threshold.
                    if (StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_PRIMARY_PARTITION_VALUE.equals(storagePolicyRuleType)) {
                        // For this storage policy rule, we ignore this business object data if it was updated earlier than the threshold value of days ago.
                        if (businessObjectDataEntity.getUpdatedOn().compareTo(updatedOnThresholdTimestamp) <= 0) {
                            // Try to convert business object data primary partition value to a timestamp.
                            // If it is not a date, the storage policy rule is not matching this business object data.
                            Date primaryPartitionValue = businessObjectDataHelper.getDateFromString(businessObjectDataEntity.getPartitionValue());
                            // For this storage policy rule, we ignore this business data if primary partition value is not a date.
                            if (primaryPartitionValue != null) {
                                // Compute the relative primary partition value threshold date based on the current timestamp and storage policy rule value.
                                Date primaryPartitionValueThreshold = new Date(HerdDateUtils.addDays(currentTimestamp, -storagePolicyRuleValue).getTime());
                                // Select this business object data if it has it's primary partition value before or equal to the threshold date.
                                createStoragePolicySelection = (primaryPartitionValue.compareTo(primaryPartitionValueThreshold) <= 0);
                            }
                        }
                    } else // Fail on an un-supported storage policy rule type.
                    {
                        throw new IllegalStateException(String.format("Storage policy type \"%s\" is not supported.", storagePolicyRuleType));
                    }
                    // If this business object data got selected, create a storage policy selection and add it to the result list.
                    if (createStoragePolicySelection) {
                        // Create business object data key and storage policy key per selected entities.
                        BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity);
                        StoragePolicyKey storagePolicyKey = new StoragePolicyKey(storagePolicyEntity.getNamespace().getCode(), storagePolicyEntity.getName());
                        // Create and add a storage policy selection to the result list.
                        storagePolicySelections.add(new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, storagePolicyEntity.getVersion()));
                        LOGGER.info("Selected business object data for storage policy processing: " + "businessObjectDataKey={} storagePolicyKey={} storagePolicyVersion={}", jsonHelper.objectToJson(businessObjectDataKey), jsonHelper.objectToJson(storagePolicyKey), storagePolicyEntity.getVersion());
                        // Stop adding storage policy selections to the result list if we reached the maximum results limit.
                        if (storagePolicySelections.size() >= maxResult) {
                            break;
                        }
                    }
                }
            }
            // Stop processing storage policies if we reached the max result limit or there are no more business object data to select.
            if (storagePolicySelections.size() >= maxResult || map.isEmpty()) {
                break;
            }
            // Increment start position for the next select.
            startPosition += maxResult;
        }
        // Stop processing storage policies if we reached the max result limit.
        if (storagePolicySelections.size() >= maxResult) {
            break;
        }
    }
    // Send all storage policy selections to the specified SQS queue.
    sendStoragePolicySelectionToSqsQueue(sqsQueueName, storagePolicySelections);
    return storagePolicySelections;
}

Example 33 with StoragePolicyEntity

use of org.finra.herd.model.jpa.StoragePolicyEntity in project herd by FINRAOS.

the class StoragePolicyProcessorHelperServiceImplTest method testInitiateStoragePolicyTransitionImpl.

@Test
public void testInitiateStoragePolicyTransitionImpl() {
    // Create an empty storage policy transition parameters DTO.
    StoragePolicyTransitionParamsDto storagePolicyTransitionParamsDto = new StoragePolicyTransitionParamsDto();
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a business object format entity.
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.VALID);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    businessObjectDataEntity.setBusinessObjectFormat(businessObjectFormatEntity);
    businessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create a storage platform entity.
    StoragePlatformEntity storagePlatformEntity = new StoragePlatformEntity();
    storagePlatformEntity.setName(StoragePlatformEntity.S3);
    // Create a storage entity.
    StorageEntity storageEntity = new StorageEntity();
    storageEntity.setStoragePlatform(storagePlatformEntity);
    storageEntity.setName(STORAGE_NAME);
    // Create a storage policy entity.
    StoragePolicyTransitionTypeEntity storagePolicyTransitionTypeEntity = new StoragePolicyTransitionTypeEntity();
    storagePolicyTransitionTypeEntity.setCode(StoragePolicyTransitionTypeEntity.GLACIER);
    // Create a storage policy entity.
    StoragePolicyEntity storagePolicyEntity = new StoragePolicyEntity();
    storagePolicyEntity.setStorage(storageEntity);
    storagePolicyEntity.setStoragePolicyTransitionType(storagePolicyTransitionTypeEntity);
    // Create a list of storage file entities.
    List<StorageFileEntity> storageFileEntities = Arrays.asList(new StorageFileEntity());
    // Create a storage unit status entity.
    StorageUnitStatusEntity storageUnitStatusEntity = new StorageUnitStatusEntity();
    storageUnitStatusEntity.setCode(StorageUnitStatusEntity.ENABLED);
    // Create a storage unit entity.
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitEntity.setStorageFiles(storageFileEntities);
    storageUnitEntity.setStatus(storageUnitStatusEntity);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE_1_KB, ROW_COUNT_1000));
    // Mock the external calls.
    when(businessObjectDataDaoHelper.getBusinessObjectDataEntity(businessObjectDataKey)).thenReturn(businessObjectDataEntity);
    when(storagePolicyDaoHelper.getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, STORAGE_POLICY_VERSION)).thenReturn(storagePolicyEntity);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX)).thenReturn(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX);
    when(storageHelper.getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX, storageEntity, false, true)).thenReturn(true);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE)).thenReturn(S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE);
    when(storageHelper.getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE, storageEntity, false, true)).thenReturn(true);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME)).thenReturn(S3_ATTRIBUTE_NAME_BUCKET_NAME);
    when(storageHelper.getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true)).thenReturn(S3_BUCKET_NAME);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY)).thenReturn(S3_OBJECT_TAG_KEY);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE)).thenReturn(S3_OBJECT_TAG_VALUE);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN)).thenReturn(S3_OBJECT_TAGGER_ROLE_ARN);
    when(configurationHelper.getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME)).thenReturn(S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    when(storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity)).thenReturn(storageUnitEntity);
    when(s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey)).thenReturn(S3_KEY_PREFIX);
    when(storageFileHelper.getAndValidateStorageFiles(storageUnitEntity, S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey)).thenReturn(storageFiles);
    doAnswer(new Answer<Void>() {

        public Void answer(InvocationOnMock invocation) {
            // Get the new storage unit status.
            String storageUnitStatus = (String) invocation.getArguments()[1];
            // Create a storage unit status entity for the new storage unit status.
            StorageUnitStatusEntity storageUnitStatusEntity = new StorageUnitStatusEntity();
            storageUnitStatusEntity.setCode(storageUnitStatus);
            // Update the storage unit with the new status.
            StorageUnitEntity storageUnitEntity = (StorageUnitEntity) invocation.getArguments()[0];
            storageUnitEntity.setStatus(storageUnitStatusEntity);
            return null;
        }
    }).when(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ARCHIVING);
    when(configurationHelper.getProperty(ConfigurationValue.S3_ENDPOINT)).thenReturn(S3_ENDPOINT);
    // Call the method under test.
    storagePolicyProcessorHelperServiceImpl.initiateStoragePolicyTransitionImpl(storagePolicyTransitionParamsDto, new StoragePolicySelection(businessObjectDataKey, storagePolicyKey, STORAGE_POLICY_VERSION));
    // Verify the external calls.
    verify(businessObjectDataHelper).validateBusinessObjectDataKey(businessObjectDataKey, true, true);
    verify(storagePolicyHelper).validateStoragePolicyKey(storagePolicyKey);
    verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey);
    verify(businessObjectDataHelper, times(2)).businessObjectDataKeyToString(businessObjectDataKey);
    verify(storagePolicyDaoHelper).getStoragePolicyEntityByKeyAndVersion(storagePolicyKey, STORAGE_POLICY_VERSION);
    verify(storagePolicyHelper, times(2)).storagePolicyKeyAndVersionToString(storagePolicyKey, STORAGE_POLICY_VERSION);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX);
    verify(storageHelper).getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_PATH_PREFIX, storageEntity, false, true);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE);
    verify(storageHelper).getBooleanStorageAttributeValueByName(S3_ATTRIBUTE_NAME_VALIDATE_FILE_EXISTENCE, storageEntity, false, true);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME);
    verify(storageHelper).getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_KEY);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_TAG_VALUE);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_ARN);
    verify(configurationHelper).getRequiredProperty(ConfigurationValue.S3_ARCHIVE_TO_GLACIER_ROLE_SESSION_NAME);
    verify(storageUnitDaoHelper).getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
    verify(s3KeyPrefixHelper).buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey);
    verify(storageFileHelper).getAndValidateStorageFiles(storageUnitEntity, S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileDaoHelper).validateStorageFilesCount(STORAGE_NAME, businessObjectDataKey, S3_KEY_PREFIX, storageFileEntities.size());
    verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ARCHIVING);
    verify(configurationHelper).getProperty(ConfigurationValue.S3_ENDPOINT);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new StoragePolicyTransitionParamsDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, StorageUnitStatusEntity.ARCHIVING, StorageUnitStatusEntity.ENABLED, storageFiles, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME), storagePolicyTransitionParamsDto);
}