Search in sources :

Example 76 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataDaoTest method testGetBusinessObjectDataFromStorageWithThreshold.

/**
 * Validates that we correctly select business object data entities per specified storage name, threshold minutes and excluded business object status
 * values.
 */
@Test
public void testGetBusinessObjectDataFromStorageWithThreshold() {
    // Create the database entities required for testing.
    List<String> storageNames = Arrays.asList(STORAGE_NAME, STORAGE_NAME_2);
    List<String> businessObjectDataStatuses = Arrays.asList(BDATA_STATUS, BDATA_STATUS_2);
    List<Integer> createdOnTimestampMinutesOffsets = Arrays.asList(5, 15, 20);
    StorageUnitStatusEntity storageUnitStatusEntity = storageUnitStatusDaoTestHelper.createStorageUnitStatusEntity(STORAGE_UNIT_STATUS);
    int counter = 0;
    for (String storageName : storageNames) {
        StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(storageName);
        for (String businessObjectDataStatus : businessObjectDataStatuses) {
            for (Integer offset : createdOnTimestampMinutesOffsets) {
                BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, String.format("%s-%d", PARTITION_VALUE, counter++), SUBPARTITION_VALUES, DATA_VERSION, true, businessObjectDataStatus);
                // Apply the offset in minutes to createdOn value.
                businessObjectDataEntity.setCreatedOn(new Timestamp(businessObjectDataEntity.getCreatedOn().getTime() - offset * 60 * 1000));
                storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, storageUnitStatusEntity, NO_STORAGE_DIRECTORY_PATH);
                herdDao.saveAndRefresh(businessObjectDataEntity);
            }
        }
    }
    // Select a subset of test business object entities.
    List<BusinessObjectDataEntity> resultBusinessObjectDataEntities = businessObjectDataDao.getBusinessObjectDataFromStorageOlderThan(STORAGE_NAME, 10, Collections.singletonList(BDATA_STATUS_2));
    // Validate the results.
    assertNotNull(resultBusinessObjectDataEntities);
    assertEquals(2, resultBusinessObjectDataEntities.size());
    for (BusinessObjectDataEntity businessObjectDataEntity : resultBusinessObjectDataEntities) {
        assertEquals(1, businessObjectDataEntity.getStorageUnits().size());
        assertEquals(STORAGE_NAME, IterableUtils.get(businessObjectDataEntity.getStorageUnits(), 0).getStorage().getName());
        assertEquals(BDATA_STATUS, businessObjectDataEntity.getStatus().getCode());
    }
}
Also used : StorageUnitStatusEntity(org.finra.herd.model.jpa.StorageUnitStatusEntity) StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity) Timestamp(java.sql.Timestamp) Test(org.junit.Test)

Example 77 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataDaoTest method testGetBusinessObjectDataPartitionValueStoragePlatformTypeSpecified.

@Test
public void testGetBusinessObjectDataPartitionValueStoragePlatformTypeSpecified() {
    // Create database entities required for testing.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, STORAGE_PLATFORM_CODE);
    storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity.getName(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, STORAGE_UNIT_STATUS, NO_STORAGE_DIRECTORY_PATH);
    // Create a business object format key.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION);
    // Validate that we can retrieve maximum and minimum partition values without specifying storage, storage platform, or excluded storage platform type.
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, null, null, null, null));
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, null, null));
    // Validate that we can still retrieve maximum and minimum partition values when we specify correct storage platform type.
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE, null, null, null));
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE, null));
    // Validate that we fail to retrieve maximum and minimum partition values when we specify an invalid storage platform type.
    assertNull(businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE_2, null, null, null));
    assertNull(businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, NO_STORAGE_NAMES, STORAGE_PLATFORM_CODE_2, null));
    // Validate that specifying storage forces to ignore an invalid storage platform type.
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMaxPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, Collections.singletonList(STORAGE_NAME), STORAGE_PLATFORM_CODE_2, null, null, null));
    assertEquals(PARTITION_VALUE, businessObjectDataDao.getBusinessObjectDataMinPartitionValue(BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, businessObjectFormatKey, NO_DATA_VERSION, NO_BDATA_STATUS, Collections.singletonList(STORAGE_NAME), STORAGE_PLATFORM_CODE_2, null));
}
Also used : BusinessObjectFormatKey(org.finra.herd.model.api.xml.BusinessObjectFormatKey) StorageEntity(org.finra.herd.model.jpa.StorageEntity) Test(org.junit.Test)

Example 78 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageUnitDaoImpl method getStorageUnitsByStoragePlatformAndBusinessObjectData.

@Override
public List<StorageUnitEntity> getStorageUnitsByStoragePlatformAndBusinessObjectData(String storagePlatform, BusinessObjectDataEntity businessObjectDataEntity) {
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<StorageUnitEntity> criteria = builder.createQuery(StorageUnitEntity.class);
    // The criteria root is the storage unit.
    Root<StorageUnitEntity> storageUnitEntity = criteria.from(StorageUnitEntity.class);
    // Join to the other tables we can filter on.
    Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
    Join<StorageEntity, StoragePlatformEntity> storagePlatformEntity = storageEntity.join(StorageEntity_.storagePlatform);
    // Create the standard restrictions (i.e. the standard where clauses).
    List<Predicate> predicates = new ArrayList<>();
    predicates.add(builder.equal(builder.upper(storagePlatformEntity.get(StoragePlatformEntity_.name)), storagePlatform.toUpperCase()));
    predicates.add(builder.equal(storageUnitEntity.get(StorageUnitEntity_.businessObjectData), businessObjectDataEntity));
    // Order by storage name.
    Order orderBy = builder.asc(storageEntity.get(StorageEntity_.name));
    // Add the clauses for the query.
    criteria.select(storageUnitEntity).where(builder.and(predicates.toArray(new Predicate[predicates.size()]))).orderBy(orderBy);
    // Execute the query and return the results.
    return entityManager.createQuery(criteria).getResultList();
}
Also used : CriteriaBuilder(javax.persistence.criteria.CriteriaBuilder) Order(javax.persistence.criteria.Order) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) StoragePlatformEntity(org.finra.herd.model.jpa.StoragePlatformEntity) ArrayList(java.util.ArrayList) StorageEntity(org.finra.herd.model.jpa.StorageEntity) Predicate(javax.persistence.criteria.Predicate)

Example 79 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageUnitDaoImpl method getStorageUnitsByPartitionFiltersAndStorages.

/**
 * Retrieves a list of storage unit entities per specified parameters. This method processes a sublist of partition filters specified by
 * partitionFilterSubListFromIndex and partitionFilterSubListSize parameters.
 *
 * @param businessObjectFormatKey the business object format key (case-insensitive). If a business object format version isn't specified, the latest
 * available format version for each partition value will be used
 * @param partitionFilters the list of partition filter to be used to select business object data instances. Each partition filter contains a list of
 * primary and sub-partition values in the right order up to the maximum partition levels allowed by business object data registration - with partition
 * values for the relative partitions not to be used for selection passed as nulls
 * @param businessObjectDataVersion the business object data version. If a business object data version isn't specified, the latest data version based on
 * the specified business object data status is returned
 * @param businessObjectDataStatus the business object data status. This parameter is ignored when the business object data version is specified. When
 * business object data version and business object data status both are not specified, the latest data version for each set of partition values will be
 * used regardless of the status
 * @param storageNames the list of storage names where the business object data storage units should be looked for (case-insensitive)
 * @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
 * @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
 * the storage platform type is specified
 * @param partitionFilterSubListFromIndex the index of the first element in the partition filter sublist
 * @param partitionFilterSubListSize the size of the partition filter sublist
 * @param selectOnlyAvailableStorageUnits specifies if only available storage units will be selected or any storage units regardless of their status
 *
 * @return the list of storage unit entities sorted by partition values
 */
private List<StorageUnitEntity> getStorageUnitsByPartitionFiltersAndStorages(BusinessObjectFormatKey businessObjectFormatKey, List<List<String>> partitionFilters, Integer businessObjectDataVersion, String businessObjectDataStatus, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, boolean selectOnlyAvailableStorageUnits, int partitionFilterSubListFromIndex, int partitionFilterSubListSize) {
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<Tuple> criteria = builder.createTupleQuery();
    // The criteria root is the storage unit.
    Root<StorageUnitEntity> storageUnitEntity = criteria.from(StorageUnitEntity.class);
    // Join to the other tables we can filter on.
    Join<StorageUnitEntity, BusinessObjectDataEntity> businessObjectDataEntity = storageUnitEntity.join(StorageUnitEntity_.businessObjectData);
    Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
    Join<StorageEntity, StoragePlatformEntity> storagePlatformEntity = storageEntity.join(StorageEntity_.storagePlatform);
    Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> businessObjectFormatEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
    Join<BusinessObjectFormatEntity, FileTypeEntity> fileTypeEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.fileType);
    Join<BusinessObjectFormatEntity, BusinessObjectDefinitionEntity> businessObjectDefinitionEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.businessObjectDefinition);
    Join<StorageUnitEntity, StorageUnitStatusEntity> storageUnitStatusEntity = storageUnitEntity.join(StorageUnitEntity_.status);
    // Create the standard restrictions (i.e. the standard where clauses).
    // Create a standard restriction based on the business object format key values.
    // Please note that we specify not to ignore the business object format version.
    Predicate mainQueryRestriction = getQueryRestriction(builder, businessObjectFormatEntity, fileTypeEntity, businessObjectDefinitionEntity, businessObjectFormatKey, false);
    // If a format version was not specified, use the latest available for this set of partition values.
    if (businessObjectFormatKey.getBusinessObjectFormatVersion() == null) {
        // Get the latest available format version for this set of partition values and per other restrictions.
        Subquery<Integer> subQuery = getMaximumBusinessObjectFormatVersionSubQuery(builder, criteria, businessObjectDefinitionEntity, businessObjectFormatEntity, fileTypeEntity, businessObjectDataEntity, businessObjectDataVersion, businessObjectDataStatus, storageNames, storagePlatformType, excludedStoragePlatformType, selectOnlyAvailableStorageUnits);
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectFormatEntity.get(BusinessObjectFormatEntity_.businessObjectFormatVersion)).value(subQuery));
    }
    // Add restriction as per specified primary and/or sub-partition values.
    mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnPartitionValues(builder, businessObjectDataEntity, partitionFilters.subList(partitionFilterSubListFromIndex, partitionFilterSubListFromIndex + partitionFilterSubListSize)));
    // If a data version was specified, use it. Otherwise, use the latest one as per specified business object data status.
    if (businessObjectDataVersion != null) {
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectDataEntity.get(BusinessObjectDataEntity_.version), businessObjectDataVersion));
    } else {
        // Business object data version is not specified, so get the latest one as per specified business object data status, if any.
        // Meaning, when both business object data version and business object data status are not specified, we just return
        // the latest business object data version in the specified storage.
        Subquery<Integer> subQuery = getMaximumBusinessObjectDataVersionSubQuery(builder, criteria, businessObjectDataEntity, businessObjectFormatEntity, businessObjectDataStatus, storageNames, storagePlatformType, excludedStoragePlatformType, selectOnlyAvailableStorageUnits);
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectDataEntity.get(BusinessObjectDataEntity_.version)).value(subQuery));
    }
    // If specified, add restriction on storage.
    mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnStorage(builder, storageEntity, storagePlatformEntity, storageNames, storagePlatformType, excludedStoragePlatformType));
    // If specified, add a restriction on storage unit status availability flag.
    if (selectOnlyAvailableStorageUnits) {
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.isTrue(storageUnitStatusEntity.get(StorageUnitStatusEntity_.available)));
    }
    // Order by partitions and storage names.
    List<Order> orderBy = new ArrayList<>();
    for (SingularAttribute<BusinessObjectDataEntity, String> businessObjectDataPartition : BUSINESS_OBJECT_DATA_PARTITIONS) {
        orderBy.add(builder.asc(businessObjectDataEntity.get(businessObjectDataPartition)));
    }
    orderBy.add(builder.asc(storageEntity.get(StorageEntity_.name)));
    // Add the clauses for the query.
    // Please note that we use multiselect here in order to eliminate the Hibernate N+1 SELECT's problem,
    // happening when we select storage unit entities and access their relative business object data entities.
    // This is an alternative approach, since adding @Fetch(FetchMode.JOIN) failed to address the issue.
    criteria.multiselect(storageUnitEntity, storageUnitStatusEntity, storageEntity, storagePlatformEntity, businessObjectDataEntity, businessObjectFormatEntity).where(mainQueryRestriction).orderBy(orderBy);
    // Run the query to get a list of tuples back.
    List<Tuple> tuples = entityManager.createQuery(criteria).getResultList();
    // Build a list of storage unit entities to return.
    List<StorageUnitEntity> storageUnitEntities = new ArrayList<>();
    for (Tuple tuple : tuples) {
        storageUnitEntities.add(tuple.get(storageUnitEntity));
    }
    return storageUnitEntities;
}
Also used : ArrayList(java.util.ArrayList) BusinessObjectFormatEntity(org.finra.herd.model.jpa.BusinessObjectFormatEntity) Predicate(javax.persistence.criteria.Predicate) StoragePlatformEntity(org.finra.herd.model.jpa.StoragePlatformEntity) StorageUnitStatusEntity(org.finra.herd.model.jpa.StorageUnitStatusEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity) CriteriaBuilder(javax.persistence.criteria.CriteriaBuilder) Order(javax.persistence.criteria.Order) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) FileTypeEntity(org.finra.herd.model.jpa.FileTypeEntity) StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDefinitionEntity(org.finra.herd.model.jpa.BusinessObjectDefinitionEntity) Tuple(javax.persistence.Tuple)

Example 80 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class StorageDaoImpl method getStorageByName.

@Override
public StorageEntity getStorageByName(String storageName) {
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<StorageEntity> criteria = builder.createQuery(StorageEntity.class);
    // The criteria root is the namespace.
    Root<StorageEntity> storageEntity = criteria.from(StorageEntity.class);
    // Create the standard restrictions (i.e. the standard where clauses).
    Predicate queryRestriction = builder.equal(builder.upper(storageEntity.get(StorageEntity_.name)), storageName.toUpperCase());
    criteria.select(storageEntity).where(queryRestriction);
    return executeSingleResultQuery(criteria, String.format("Found more than one storage with \"%s\" name.", storageName));
}
Also used : CriteriaBuilder(javax.persistence.criteria.CriteriaBuilder) StorageEntity(org.finra.herd.model.jpa.StorageEntity) Predicate(javax.persistence.criteria.Predicate)

Aggregations

StorageEntity (org.finra.herd.model.jpa.StorageEntity)141 BusinessObjectDataEntity (org.finra.herd.model.jpa.BusinessObjectDataEntity)67 Test (org.junit.Test)63 StorageUnitEntity (org.finra.herd.model.jpa.StorageUnitEntity)57 ArrayList (java.util.ArrayList)42 BusinessObjectFormatEntity (org.finra.herd.model.jpa.BusinessObjectFormatEntity)38 AbstractServiceTest (org.finra.herd.service.AbstractServiceTest)34 StorageUnitStatusEntity (org.finra.herd.model.jpa.StorageUnitStatusEntity)24 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)23 BusinessObjectDefinitionEntity (org.finra.herd.model.jpa.BusinessObjectDefinitionEntity)23 Predicate (javax.persistence.criteria.Predicate)18 Attribute (org.finra.herd.model.api.xml.Attribute)18 BusinessObjectDataStatusEntity (org.finra.herd.model.jpa.BusinessObjectDataStatusEntity)18 CriteriaBuilder (javax.persistence.criteria.CriteriaBuilder)17 FileTypeEntity (org.finra.herd.model.jpa.FileTypeEntity)17 StorageFileEntity (org.finra.herd.model.jpa.StorageFileEntity)16 BusinessObjectFormatKey (org.finra.herd.model.api.xml.BusinessObjectFormatKey)15 BusinessObjectDefinitionKey (org.finra.herd.model.api.xml.BusinessObjectDefinitionKey)14 StoragePlatformEntity (org.finra.herd.model.jpa.StoragePlatformEntity)14 NamespaceEntity (org.finra.herd.model.jpa.NamespaceEntity)13