Examples with StorageUnitEntity org.finra.herd.model.jpa.StorageUnitEntity used on opensource projects

Example 71 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitionsDirectoryPathMismatchS3KeyPrefix.

@Test
public void testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitionsDirectoryPathMismatchS3KeyPrefix() {
    // Create one VALID sub-partition with an "available" storage unit a non-Glacier storage.
    List<StorageUnitEntity> storageUnitEntities = businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1)));
    // Save the original S3 key prefix (directory path) and update the storage unit with a directory path that does not match the S3 key prefix.
    String originalS3KeyPrefix = storageUnitEntities.get(0).getDirectoryPath();
    storageUnitEntities.get(0).setDirectoryPath(BLANK_TEXT);
    // sub-partitions when its storage unit directory path does not match the expected S3 key prefix.
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage directory path \"%s\" registered with business object data {%s} in \"%s\" storage does not match " + "the expected S3 key prefix \"%s\".", BLANK_TEXT, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_1), DATA_VERSION)), STORAGE_NAME, originalS3KeyPrefix), e.getMessage());
    }
}

Example 72 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class AbstractHerdDao method getMaximumBusinessObjectDataVersionSubQuery.

/**
 * TODO This method may be bdata specific. Consider creating new abstract class to group all bdata related DAO. Builds a sub-query to select the maximum
 * business object data version.
 *
 * @param builder the criteria builder
 * @param criteria the criteria query
 * @param businessObjectDataEntity the business object data entity that appears in the from clause of the main query
 * @param businessObjectFormatEntity the business object format entity that appears in the from clause of the main query
 * @param businessObjectDataStatus the business object data status
 * @param storageNames the list of storage names where the business object data storage units should be looked for (case-insensitive)
 * @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
 * @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
 * the storage platform type is specified
 * @param selectOnlyAvailableStorageUnits specifies if only available storage units will be selected or any storage units regardless of their status
 *
 * @return the sub-query to select the maximum business object data version
 */
protected Subquery<Integer> getMaximumBusinessObjectDataVersionSubQuery(CriteriaBuilder builder, CriteriaQuery<?> criteria, From<?, BusinessObjectDataEntity> businessObjectDataEntity, From<?, BusinessObjectFormatEntity> businessObjectFormatEntity, String businessObjectDataStatus, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, boolean selectOnlyAvailableStorageUnits) {
    // Business object data version is not specified, so get the latest one in the specified storage.
    Subquery<Integer> subQuery = criteria.subquery(Integer.class);
    // The criteria root is the business object data.
    Root<BusinessObjectDataEntity> subBusinessObjectDataEntity = subQuery.from(BusinessObjectDataEntity.class);
    // Join to the other tables we can filter on.
    Join<BusinessObjectDataEntity, StorageUnitEntity> subStorageUnitEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.storageUnits);
    Join<StorageUnitEntity, StorageEntity> subStorageEntity = subStorageUnitEntity.join(StorageUnitEntity_.storage);
    Join<StorageEntity, StoragePlatformEntity> subStoragePlatformEntity = subStorageEntity.join(StorageEntity_.storagePlatform);
    Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> subBusinessObjectFormatEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
    Join<StorageUnitEntity, StorageUnitStatusEntity> subStorageUnitStatusEntity = subStorageUnitEntity.join(StorageUnitEntity_.status);
    // Add a standard restriction on business object format.
    Predicate subQueryRestriction = builder.equal(subBusinessObjectFormatEntity, businessObjectFormatEntity);
    // Create and add standard restrictions on primary and sub-partition values.
    subQueryRestriction = builder.and(subQueryRestriction, getQueryRestrictionOnPartitionValues(builder, subBusinessObjectDataEntity, businessObjectDataEntity));
    // If specified, create and add a standard restriction on business object data status.
    if (businessObjectDataStatus != null) {
        Join<BusinessObjectDataEntity, BusinessObjectDataStatusEntity> subBusinessObjectDataStatusEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.status);
        subQueryRestriction = builder.and(subQueryRestriction, builder.equal(builder.upper(subBusinessObjectDataStatusEntity.get(BusinessObjectDataStatusEntity_.code)), businessObjectDataStatus.toUpperCase()));
    }
    // Create and add a standard restriction on storage.
    subQueryRestriction = builder.and(subQueryRestriction, getQueryRestrictionOnStorage(builder, subStorageEntity, subStoragePlatformEntity, storageNames, storagePlatformType, excludedStoragePlatformType));
    // If specified, add a restriction on storage unit status availability flag.
    if (selectOnlyAvailableStorageUnits) {
        subQueryRestriction = builder.and(subQueryRestriction, builder.isTrue(subStorageUnitStatusEntity.get(StorageUnitStatusEntity_.available)));
    }
    subQuery.select(builder.max(subBusinessObjectDataEntity.get(BusinessObjectDataEntity_.version))).where(subQueryRestriction);
    return subQuery;
}

Example 73 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StoragePolicySelectorServiceTest method testExecuteBusinessObjectDataNotSelectedDueToHigherPriorityLevelStoragePolicy.

@Test
public void testExecuteBusinessObjectDataNotSelectedDueToHigherPriorityLevelStoragePolicy() {
    // Storage a storage policy with a filter that has no fields specified.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD_2, STORAGE_POLICY_NAME_2), StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED, BDATA_AGE_IN_DAYS, NO_BDEF_NAMESPACE, NO_BDEF_NAME, NO_FORMAT_USAGE_CODE, NO_FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Create and persist a storage unit in the storage policy filter storage.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Get the business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    // Apply the offset in days to business object data "created on" value, so it would match to the storage policy.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(businessObjectDataEntity, BDATA_AGE_IN_DAYS + 1);
    // Execute the storage policy selection and validate the results. The business object data is expected to be selected.
    assertEquals(1, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Storage a storage policy with a filter that has only usage and file type specified
    // and with the age restriction greater than the current business object data entity age.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD_2, STORAGE_POLICY_NAME), StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED, BDATA_AGE_IN_DAYS + 2, NO_BDEF_NAMESPACE, NO_BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Execute the storage policy selection and validate the results. The business object data is not expected to be selected.
    assertEquals(0, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Apply the offset in days to business object data "created on" value, so it would match to the last added storage policy.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(businessObjectDataEntity, 2);
    // Execute the storage policy selection and validate the results. The business object data is expected to be selected.
    assertEquals(1, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Storage a storage policy with a filter that has only business object definition
    // specified and with the age restriction greater than the current business object data entity age.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME_2), StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED, BDATA_AGE_IN_DAYS + 4, BDEF_NAMESPACE, BDEF_NAME, NO_FORMAT_USAGE_CODE, NO_FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Execute the storage policy selection and validate the results. The business object data is not expected to be selected.
    assertEquals(0, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Apply the offset in days to business object data "created on" value, so it would match to the last added storage policy.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(businessObjectDataEntity, 2);
    // Execute the storage policy selection and validate the results. The business object data is expected to be selected.
    assertEquals(1, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Storage a storage policy with a filter that has business object definition, usage, and file type
    // specified and with the age restriction greater than the current business object data entity age.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME), StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED, BDATA_AGE_IN_DAYS + 6, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Execute the storage policy selection and validate the results. The business object data is not expected to be selected.
    assertEquals(0, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    // Apply the offset in days to business object data "created on" value, so it would match to the last added storage policy.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(businessObjectDataEntity, 2);
    // Execute the storage policy selection and validate the results. The business object data is expected to be selected.
    assertEquals(1, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
}

Example 74 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StoragePolicySelectorServiceTest method testExecuteTestingMaxResult.

@Test
public void testExecuteTestingMaxResult() {
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_REGISTERED, BDATA_AGE_IN_DAYS, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Create and persist a storage unit in the storage policy filter storage.
    StorageUnitEntity storageUnitEntity1 = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Apply the offset in days to business object data "created on" value.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(storageUnitEntity1.getBusinessObjectData(), BDATA_AGE_IN_DAYS + 1);
    // Create and persist a second storage unit in the storage policy filter storage.
    StorageUnitEntity storageUnitEntity2 = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_2, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Also apply an offset to business object data "created on" value, but make this business object data older than the first.
    businessObjectDataDaoTestHelper.ageBusinessObjectData(storageUnitEntity2.getBusinessObjectData(), BDATA_AGE_IN_DAYS + 2);
    // Try to retrieve both business object data instances as matching to the storage policy, but with max result limit set to 1.
    List<StoragePolicySelection> resultStoragePolicySelections = storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, 1);
    // Validate the results. Only the oldest business object data should get selected.
    assertEquals(Arrays.asList(new StoragePolicySelection(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_2, SUBPARTITION_VALUES, DATA_VERSION), storagePolicyKey, INITIAL_VERSION)), resultStoragePolicySelections);
}

Example 75 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StoragePolicySelectorServiceTest method testExecutePrimaryPartitionValueStoragePolicyRuleTypePrimaryPartitionValueNotOldEnough.

@Test
public void testExecutePrimaryPartitionValueStoragePolicyRuleTypePrimaryPartitionValueNotOldEnough() throws Exception {
    // Create a storage policy key.
    StoragePolicyKey storagePolicyKey = new StoragePolicyKey(STORAGE_POLICY_NAMESPACE_CD, STORAGE_POLICY_NAME);
    // Create and persist a storage policy entity.
    storagePolicyDaoTestHelper.createStoragePolicyEntity(storagePolicyKey, StoragePolicyRuleTypeEntity.DAYS_SINCE_BDATA_PRIMARY_PARTITION_VALUE, BDATA_PARTITION_VALUE_AGE_IN_DAYS, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, STORAGE_NAME, StoragePolicyTransitionTypeEntity.GLACIER, StoragePolicyStatusEntity.ENABLED, INITIAL_VERSION, LATEST_VERSION_FLAG_SET);
    // Create a partition value that would satisfy the primary partition value age check as per the storage policy rule.
    String primaryPartitionValue = getTestPrimaryPartitionValue(BDATA_PARTITION_VALUE_AGE_IN_DAYS + 1);
    // Create and persist a storage unit with ENABLED status in the storage policy filter storage.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Overwrite the "updated on" threshold for a newly created business object data to be selectable by the storage policy.
    Map<String, Object> overrideMap = new HashMap<>();
    overrideMap.put(ConfigurationValue.STORAGE_POLICY_PROCESSOR_BDATA_UPDATED_ON_THRESHOLD_DAYS.getKey(), -1);
    modifyPropertySourceInEnvironment(overrideMap);
    try {
        // Execute the storage policy selection and validate the results. One business object data matching to storage policy should get selected.
        assertEquals(Arrays.asList(new StoragePolicySelection(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, SUBPARTITION_VALUES, DATA_VERSION), storagePolicyKey, INITIAL_VERSION)), storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT));
        // Update this business object data, so it's primary partition value is now after the storage policy threshold date.
        storageUnitEntity.getBusinessObjectData().setPartitionValue(getTestPrimaryPartitionValue(BDATA_PARTITION_VALUE_AGE_IN_DAYS - 1));
        businessObjectDataDao.saveAndRefresh(storageUnitEntity.getBusinessObjectData());
        // Execute the storage policy selection and validate the results. No business object data matching to storage policy should get selected.
        assertEquals(0, storagePolicySelectorService.execute(AWS_SQS_QUEUE_NAME, MAX_RESULT).size());
    } finally {
        restorePropertySourceInEnvironment();
    }
}