use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class BusinessObjectDataDaoTest method testGetBusinessObjectDataFromStorageWithThreshold.
/**
* Validates that we correctly select business object data entities per specified storage name, threshold minutes and excluded business object status
* values.
*/
@Test
public void testGetBusinessObjectDataFromStorageWithThreshold() {
// Create the database entities required for testing.
List<String> storageNames = Arrays.asList(STORAGE_NAME, STORAGE_NAME_2);
List<String> businessObjectDataStatuses = Arrays.asList(BDATA_STATUS, BDATA_STATUS_2);
List<Integer> createdOnTimestampMinutesOffsets = Arrays.asList(5, 15, 20);
StorageUnitStatusEntity storageUnitStatusEntity = storageUnitStatusDaoTestHelper.createStorageUnitStatusEntity(STORAGE_UNIT_STATUS);
int counter = 0;
for (String storageName : storageNames) {
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(storageName);
for (String businessObjectDataStatus : businessObjectDataStatuses) {
for (Integer offset : createdOnTimestampMinutesOffsets) {
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, String.format("%s-%d", PARTITION_VALUE, counter++), SUBPARTITION_VALUES, DATA_VERSION, true, businessObjectDataStatus);
// Apply the offset in minutes to createdOn value.
businessObjectDataEntity.setCreatedOn(new Timestamp(businessObjectDataEntity.getCreatedOn().getTime() - offset * 60 * 1000));
storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, storageUnitStatusEntity, NO_STORAGE_DIRECTORY_PATH);
herdDao.saveAndRefresh(businessObjectDataEntity);
}
}
}
// Select a subset of test business object entities.
List<BusinessObjectDataEntity> resultBusinessObjectDataEntities = businessObjectDataDao.getBusinessObjectDataFromStorageOlderThan(STORAGE_NAME, 10, Collections.singletonList(BDATA_STATUS_2));
// Validate the results.
assertNotNull(resultBusinessObjectDataEntities);
assertEquals(2, resultBusinessObjectDataEntities.size());
for (BusinessObjectDataEntity businessObjectDataEntity : resultBusinessObjectDataEntities) {
assertEquals(1, businessObjectDataEntity.getStorageUnits().size());
assertEquals(STORAGE_NAME, IterableUtils.get(businessObjectDataEntity.getStorageUnits(), 0).getStorage().getName());
assertEquals(BDATA_STATUS, businessObjectDataEntity.getStatus().getCode());
}
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class BusinessObjectDataDaoTest method testGetBusinessObjectDataPartitionValueStoragePlatformTypeSpecified.
@Test
public void testGetBusinessObjectDataPartitionValueStoragePlatformTypeSpecified() {
// Create database entities required for testing.
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, STORAGE_PLATFORM_CODE);
storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity.getName(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, STORAGE_UNIT_STATUS, NO_STORAGE_DIRECTORY_PATH);
// Create a business object format key.
BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION);
// Validate that we can retrieve maximum and minimum partition values without specifying storage, storage platform, or excluded storage platform type.
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, null, null, null, null));
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, null, null));
// Validate that we can still retrieve maximum and minimum partition values when we specify correct storage platform type.
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE, null, null, null));
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE, null));
// Validate that we fail to retrieve maximum and minimum partition values when we specify an invalid storage platform type.
assertNull(businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE_2, null, null, null));
assertNull(businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE_2, null));
// Validate that specifying storage forces to ignore an invalid storage platform type.
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, Collections.singletonList(STORAGE_NAME), STORAGE_PLATFORM_CODE_2, null, null, null));
assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, Collections.singletonList(STORAGE_NAME), STORAGE_PLATFORM_CODE_2, null));
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class StorageUnitDaoImpl method getStorageUnitsByStoragePlatformAndBusinessObjectData.
@Override
public List<StorageUnitEntity> getStorageUnitsByStoragePlatformAndBusinessObjectData(String storagePlatform, BusinessObjectDataEntity businessObjectDataEntity) {
// Create the criteria builder and the criteria.
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
CriteriaQuery<StorageUnitEntity> criteria = builder.createQuery(StorageUnitEntity.class);
// The criteria root is the storage unit.
Root<StorageUnitEntity> storageUnitEntity = criteria.from(StorageUnitEntity.class);
// Join to the other tables we can filter on.
Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
Join<StorageEntity, StoragePlatformEntity> storagePlatformEntity = storageEntity.join(StorageEntity_.storagePlatform);
// Create the standard restrictions (i.e. the standard where clauses).
List<Predicate> predicates = new ArrayList<>();
predicates.add(builder.equal(builder.upper(storagePlatformEntity.get(StoragePlatformEntity_.name)), storagePlatform.toUpperCase()));
predicates.add(builder.equal(storageUnitEntity.get(StorageUnitEntity_.businessObjectData), businessObjectDataEntity));
// Order by storage name.
Order orderBy = builder.asc(storageEntity.get(StorageEntity_.name));
// Add the clauses for the query.
criteria.select(storageUnitEntity).where(builder.and(predicates.toArray(new Predicate[predicates.size()]))).orderBy(orderBy);
// Execute the query and return the results.
return entityManager.createQuery(criteria).getResultList();
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class StorageUnitDaoImpl method getStorageUnitsByPartitionFiltersAndStorages.
/**
* Retrieves a list of storage unit entities per specified parameters. This method processes a sublist of partition filters specified by
* partitionFilterSubListFromIndex and partitionFilterSubListSize parameters.
*
* @param businessObjectFormatKey the business object format key (case-insensitive). If a business object format version isn't specified, the latest
* available format version for each partition value will be used
* @param partitionFilters the list of partition filter to be used to select business object data instances. Each partition filter contains a list of
* primary and sub-partition values in the right order up to the maximum partition levels allowed by business object data registration - with partition
* values for the relative partitions not to be used for selection passed as nulls
* @param businessObjectDataVersion the business object data version. If a business object data version isn't specified, the latest data version based on
* the specified business object data status is returned
* @param businessObjectDataStatus the business object data status. This parameter is ignored when the business object data version is specified. When
* business object data version and business object data status both are not specified, the latest data version for each set of partition values will be
* used regardless of the status
* @param storageNames the list of storage names where the business object data storage units should be looked for (case-insensitive)
* @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
* @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
* the storage platform type is specified
* @param partitionFilterSubListFromIndex the index of the first element in the partition filter sublist
* @param partitionFilterSubListSize the size of the partition filter sublist
* @param selectOnlyAvailableStorageUnits specifies if only available storage units will be selected or any storage units regardless of their status
*
* @return the list of storage unit entities sorted by partition values
*/
private List<StorageUnitEntity> getStorageUnitsByPartitionFiltersAndStorages(BusinessObjectFormatKey businessObjectFormatKey, List<List<String>> partitionFilters, Integer businessObjectDataVersion, String businessObjectDataStatus, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, boolean selectOnlyAvailableStorageUnits, int partitionFilterSubListFromIndex, int partitionFilterSubListSize) {
// Create the criteria builder and the criteria.
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
CriteriaQuery<Tuple> criteria = builder.createTupleQuery();
// The criteria root is the storage unit.
Root<StorageUnitEntity> storageUnitEntity = criteria.from(StorageUnitEntity.class);
// Join to the other tables we can filter on.
Join<StorageUnitEntity, BusinessObjectDataEntity> businessObjectDataEntity = storageUnitEntity.join(StorageUnitEntity_.businessObjectData);
Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
Join<StorageEntity, StoragePlatformEntity> storagePlatformEntity = storageEntity.join(StorageEntity_.storagePlatform);
Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> businessObjectFormatEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
Join<BusinessObjectFormatEntity, FileTypeEntity> fileTypeEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.fileType);
Join<BusinessObjectFormatEntity, BusinessObjectDefinitionEntity> businessObjectDefinitionEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.businessObjectDefinition);
Join<StorageUnitEntity, StorageUnitStatusEntity> storageUnitStatusEntity = storageUnitEntity.join(StorageUnitEntity_.status);
// Create the standard restrictions (i.e. the standard where clauses).
// Create a standard restriction based on the business object format key values.
// Please note that we specify not to ignore the business object format version.
Predicate mainQueryRestriction = getQueryRestriction(builder, businessObjectFormatEntity, fileTypeEntity, businessObjectDefinitionEntity, businessObjectFormatKey, false);
// If a format version was not specified, use the latest available for this set of partition values.
if (businessObjectFormatKey.getBusinessObjectFormatVersion() == null) {
// Get the latest available format version for this set of partition values and per other restrictions.
Subquery<Integer> subQuery = getMaximumBusinessObjectFormatVersionSubQuery(builder, criteria, businessObjectDefinitionEntity, businessObjectFormatEntity, fileTypeEntity, businessObjectDataEntity, businessObjectDataVersion, businessObjectDataStatus, storageNames, storagePlatformType, excludedStoragePlatformType, selectOnlyAvailableStorageUnits);
mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectFormatEntity.get(BusinessObjectFormatEntity_.businessObjectFormatVersion)).value(subQuery));
}
// Add restriction as per specified primary and/or sub-partition values.
mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnPartitionValues(builder, businessObjectDataEntity, partitionFilters.subList(partitionFilterSubListFromIndex, partitionFilterSubListFromIndex + partitionFilterSubListSize)));
// If a data version was specified, use it. Otherwise, use the latest one as per specified business object data status.
if (businessObjectDataVersion != null) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectDataEntity.get(BusinessObjectDataEntity_.version), businessObjectDataVersion));
} else {
// Business object data version is not specified, so get the latest one as per specified business object data status, if any.
// Meaning, when both business object data version and business object data status are not specified, we just return
// the latest business object data version in the specified storage.
Subquery<Integer> subQuery = getMaximumBusinessObjectDataVersionSubQuery(builder, criteria, businessObjectDataEntity, businessObjectFormatEntity, businessObjectDataStatus, storageNames, storagePlatformType, excludedStoragePlatformType, selectOnlyAvailableStorageUnits);
mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectDataEntity.get(BusinessObjectDataEntity_.version)).value(subQuery));
}
// If specified, add restriction on storage.
mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnStorage(builder, storageEntity, storagePlatformEntity, storageNames, storagePlatformType, excludedStoragePlatformType));
// If specified, add a restriction on storage unit status availability flag.
if (selectOnlyAvailableStorageUnits) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.isTrue(storageUnitStatusEntity.get(StorageUnitStatusEntity_.available)));
}
// Order by partitions and storage names.
List<Order> orderBy = new ArrayList<>();
for (SingularAttribute<BusinessObjectDataEntity, String> businessObjectDataPartition : BUSINESS_OBJECT_DATA_PARTITIONS) {
orderBy.add(builder.asc(businessObjectDataEntity.get(businessObjectDataPartition)));
}
orderBy.add(builder.asc(storageEntity.get(StorageEntity_.name)));
// Add the clauses for the query.
// Please note that we use multiselect here in order to eliminate the Hibernate N+1 SELECT's problem,
// happening when we select storage unit entities and access their relative business object data entities.
// This is an alternative approach, since adding @Fetch(FetchMode.JOIN) failed to address the issue.
criteria.multiselect(storageUnitEntity, storageUnitStatusEntity, storageEntity, storagePlatformEntity, businessObjectDataEntity, businessObjectFormatEntity).where(mainQueryRestriction).orderBy(orderBy);
// Run the query to get a list of tuples back.
List<Tuple> tuples = entityManager.createQuery(criteria).getResultList();
// Build a list of storage unit entities to return.
List<StorageUnitEntity> storageUnitEntities = new ArrayList<>();
for (Tuple tuple : tuples) {
storageUnitEntities.add(tuple.get(storageUnitEntity));
}
return storageUnitEntities;
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class StorageDaoImpl method getStorageByName.
@Override
public StorageEntity getStorageByName(String storageName) {
// Create the criteria builder and the criteria.
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
CriteriaQuery<StorageEntity> criteria = builder.createQuery(StorageEntity.class);
// The criteria root is the namespace.
Root<StorageEntity> storageEntity = criteria.from(StorageEntity.class);
// Create the standard restrictions (i.e. the standard where clauses).
Predicate queryRestriction = builder.equal(builder.upper(storageEntity.get(StorageEntity_.name)), storageName.toUpperCase());
criteria.select(storageEntity).where(queryRestriction);
return executeSingleResultQuery(criteria, String.format("Found more than one storage with \"%s\" name.", storageName));
}
Aggregations