Examples with StorageUnitEntity org.finra.herd.model.jpa.StorageUnitEntity used on opensource projects

Example 46 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceImpl method getPartitionFilters.

/**
 * Gets a list of matched partition filters per specified list of storage unit entities and a sample partition filter.
 *
 * @param storageUnitEntities the list of storage unit entities
 * @param samplePartitionFilter the sample partition filter
 *
 * @return the list of partition filters
 */
private List<List<String>> getPartitionFilters(List<StorageUnitEntity> storageUnitEntities, List<String> samplePartitionFilter) {
    List<List<String>> partitionFilters = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData());
        partitionFilters.add(businessObjectDataHelper.getPartitionFilter(businessObjectDataKey, samplePartitionFilter));
    }
    return partitionFilters;
}

Example 47 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class BusinessObjectDataDaoImpl method getBusinessObjectDataEntities.

/**
 * Retrieves a list of business object data entities per specified parameters. This method processes a sublist of partition filters specified by
 * partitionFilterSubListFromIndex and partitionFilterSubListSize parameters.
 *
 * @param businessObjectFormatKey the business object format key (case-insensitive). If a business object format version isn't specified, the latest
 * available format version for each partition value will be used.
 * @param partitionFilters the list of partition filter to be used to select business object data instances. Each partition filter contains a list of
 * primary and sub-partition values in the right order up to the maximum partition levels allowed by business object data registration - with partition
 * values for the relative partitions not to be used for selection passed as nulls.
 * @param businessObjectDataVersion the business object data version. If a business object data version isn't specified, the latest data version based on
 * the specified business object data status is returned.
 * @param businessObjectDataStatus the business object data status. This parameter is ignored when the business object data version is specified. When
 * business object data version and business object data status both are not specified, the latest data version for each set of partition values will be
 * used regardless of the status.
 * @param storageName the name of the storage where the business object data storage unit is located (case-insensitive)
 * @param partitionFilterSubListFromIndex the index of the first element in the partition filter sublist
 * @param partitionFilterSubListSize the size of the partition filter sublist
 *
 * @return the list of business object data entities sorted by partition values
 */
private List<BusinessObjectDataEntity> getBusinessObjectDataEntities(BusinessObjectFormatKey businessObjectFormatKey, List<List<String>> partitionFilters, Integer businessObjectDataVersion, String businessObjectDataStatus, String storageName, int partitionFilterSubListFromIndex, int partitionFilterSubListSize) {
    // Create the criteria builder and the criteria.
    CriteriaBuilder builder = entityManager.getCriteriaBuilder();
    CriteriaQuery<BusinessObjectDataEntity> criteria = builder.createQuery(BusinessObjectDataEntity.class);
    // The criteria root is the business object data.
    Root<BusinessObjectDataEntity> businessObjectDataEntity = criteria.from(BusinessObjectDataEntity.class);
    // Join to the other tables we can filter on.
    Join<BusinessObjectDataEntity, StorageUnitEntity> storageUnitEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.storageUnits);
    Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
    Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> businessObjectFormatEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
    Join<BusinessObjectFormatEntity, FileTypeEntity> fileTypeEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.fileType);
    Join<BusinessObjectFormatEntity, BusinessObjectDefinitionEntity> businessObjectDefinitionEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.businessObjectDefinition);
    // Create the standard restrictions (i.e. the standard where clauses).
    // Create a standard restriction based on the business object format key values.
    // Please note that we specify not to ignore the business object format version.
    Predicate mainQueryRestriction = getQueryRestriction(builder, businessObjectFormatEntity, fileTypeEntity, businessObjectDefinitionEntity, businessObjectFormatKey, false);
    // If a format version was not specified, use the latest available for this set of partition values.
    if (businessObjectFormatKey.getBusinessObjectFormatVersion() == null) {
        // Business object format version is not specified, so just use the latest available for this set of partition values.
        Subquery<Integer> subQuery = criteria.subquery(Integer.class);
        // The criteria root is the business object data.
        Root<BusinessObjectDataEntity> subBusinessObjectDataEntity = subQuery.from(BusinessObjectDataEntity.class);
        // Join to the other tables we can filter on.
        Join<BusinessObjectDataEntity, StorageUnitEntity> subStorageUnitEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.storageUnits);
        Join<StorageUnitEntity, StorageEntity> subStorageEntity = subStorageUnitEntity.join(StorageUnitEntity_.storage);
        Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> subBusinessObjectFormatEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.businessObjectFormat);
        Join<BusinessObjectFormatEntity, BusinessObjectDefinitionEntity> subBusinessObjectDefinitionEntity = subBusinessObjectFormatEntity.join(BusinessObjectFormatEntity_.businessObjectDefinition);
        Join<BusinessObjectFormatEntity, FileTypeEntity> subBusinessObjectFormatFileTypeEntity = subBusinessObjectFormatEntity.join(BusinessObjectFormatEntity_.fileType);
        Join<BusinessObjectDataEntity, BusinessObjectDataStatusEntity> subBusinessObjectDataStatusEntity = subBusinessObjectDataEntity.join(BusinessObjectDataEntity_.status);
        // Create the standard restrictions (i.e. the standard where clauses).
        Predicate subQueryRestriction = builder.equal(subBusinessObjectDefinitionEntity, businessObjectDefinitionEntity);
        subQueryRestriction = builder.and(subQueryRestriction, builder.equal(subBusinessObjectFormatEntity.get(BusinessObjectFormatEntity_.usage), businessObjectFormatEntity.get(BusinessObjectFormatEntity_.usage)));
        subQueryRestriction = builder.and(subQueryRestriction, builder.equal(subBusinessObjectFormatFileTypeEntity, fileTypeEntity));
        // Create and add standard restrictions on primary and sub-partition values.
        subQueryRestriction = builder.and(subQueryRestriction, getQueryRestrictionOnPartitionValues(builder, subBusinessObjectDataEntity, businessObjectDataEntity));
        // Add restrictions on business object data version and business object data status.
        Predicate subQueryRestrictionOnBusinessObjectDataVersionAndStatus = getQueryRestrictionOnBusinessObjectDataVersionAndStatus(builder, subBusinessObjectDataEntity, subBusinessObjectDataStatusEntity, businessObjectDataVersion, businessObjectDataStatus);
        if (subQueryRestrictionOnBusinessObjectDataVersionAndStatus != null) {
            subQueryRestriction = builder.and(subQueryRestriction, subQueryRestrictionOnBusinessObjectDataVersionAndStatus);
        }
        // Create and add a standard restriction on storage.
        subQueryRestriction = builder.and(subQueryRestriction, builder.equal(subStorageEntity, storageEntity));
        subQuery.select(builder.max(subBusinessObjectFormatEntity.get(BusinessObjectFormatEntity_.businessObjectFormatVersion))).where(subQueryRestriction);
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectFormatEntity.get(BusinessObjectFormatEntity_.businessObjectFormatVersion)).value(subQuery));
    }
    // Add restriction as per specified primary and/or sub-partition values.
    mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnPartitionValues(builder, businessObjectDataEntity, partitionFilters.subList(partitionFilterSubListFromIndex, partitionFilterSubListFromIndex + partitionFilterSubListSize)));
    // If a data version was specified, use it. Otherwise, use the latest one as per specified business object data status.
    if (businessObjectDataVersion != null) {
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectDataEntity.get(BusinessObjectDataEntity_.version), businessObjectDataVersion));
    } else {
        // Business object data version is not specified, so get the latest one as per specified business object data status, if any.
        // Meaning, when both business object data version and business object data status are not specified, we just return
        // the latest business object data version in the specified storage.
        Subquery<Integer> subQuery = getMaximumBusinessObjectDataVersionSubQuery(builder, criteria, businessObjectDataEntity, businessObjectFormatEntity, businessObjectDataStatus, Collections.singletonList(storageName), null, null, false);
        mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectDataEntity.get(BusinessObjectDataEntity_.version)).value(subQuery));
    }
    // Add a storage name restriction to the main query where clause.
    mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(builder.upper(storageEntity.get(StorageEntity_.name)), storageName.toUpperCase()));
    // Add the clauses for the query.
    criteria.select(businessObjectDataEntity).where(mainQueryRestriction);
    // Order by partitions.
    List<Order> orderBy = new ArrayList<>();
    for (SingularAttribute<BusinessObjectDataEntity, String> businessObjectDataPartition : BUSINESS_OBJECT_DATA_PARTITIONS) {
        orderBy.add(builder.asc(businessObjectDataEntity.get(businessObjectDataPartition)));
    }
    criteria.orderBy(orderBy);
    return entityManager.createQuery(criteria).getResultList();
}

Example 48 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StorageUnitDaoTestHelper method createStorageUnitEntity.

/**
 * Creates and persists a new storage unit entity.
 *
 * @param storageEntity the storage entity
 * @param businessObjectDataEntity the business object data entity
 * @param storageUnitStatusEntity the storage unit status entity
 * @param directoryPath the storage directory path
 *
 * @return the newly created storage unit entity.
 */
public StorageUnitEntity createStorageUnitEntity(StorageEntity storageEntity, BusinessObjectDataEntity businessObjectDataEntity, StorageUnitStatusEntity storageUnitStatusEntity, String directoryPath) {
    StorageUnitEntity storageUnitEntity = new StorageUnitEntity();
    storageUnitEntity.setStorage(storageEntity);
    storageUnitEntity.setBusinessObjectData(businessObjectDataEntity);
    storageUnitEntity.setDirectoryPath(directoryPath);
    storageUnitEntity.setStatus(storageUnitStatusEntity);
    return storageUnitDao.saveAndRefresh(storageUnitEntity);
}

Example 49 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StorageUnitDaoTest method testGetS3StorageUnitsToCleanup.

@Test
public void testGetS3StorageUnitsToCleanup() {
    // Create a list of business object data keys.
    List<BusinessObjectDataKey> businessObjectDataKeys = new ArrayList<>();
    for (int i = 0; i < 7; i++) {
        businessObjectDataKeys.add(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Integer.toString(i), SUBPARTITION_VALUES, DATA_VERSION));
    }
    // Create database entities required for testing.
    List<StorageUnitEntity> storageUnitEntities = Lists.newArrayList();
    // Add 4 valid storage unit entries
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, StoragePlatformEntity.S3, businessObjectDataKeys.get(0), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, StoragePlatformEntity.S3, businessObjectDataKeys.get(1), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, StoragePlatformEntity.S3, businessObjectDataKeys.get(2), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME_2, StoragePlatformEntity.S3, businessObjectDataKeys.get(3), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    // Not a valid business object data status
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, StoragePlatformEntity.S3, businessObjectDataKeys.get(4), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.ARCHIVED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    // Not a valid storage unit status
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, StoragePlatformEntity.S3, businessObjectDataKeys.get(5), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.ARCHIVED, NO_STORAGE_DIRECTORY_PATH));
    // Not a valid storage platform
    storageUnitEntities.add(storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME_3, StoragePlatformEntity.TABLE_NAME, businessObjectDataKeys.get(6), LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH));
    // Set restore expiration time values.
    Timestamp currentTime = new Timestamp(System.currentTimeMillis());
    // Not a valid final destroy on date
    storageUnitEntities.get(0).setFinalDestroyOn(null);
    // Final destroy on date is not less than current timestamp
    storageUnitEntities.get(1).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, 1));
    // Valid final destroy on dates
    storageUnitEntities.get(2).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, -1));
    storageUnitEntities.get(3).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, -2));
    storageUnitEntities.get(4).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, -2));
    storageUnitEntities.get(5).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, -2));
    storageUnitEntities.get(6).setFinalDestroyOn(HerdDateUtils.addDays(currentTime, -2));
    // Retrieve the storage units and validate the results. Only two entities are expected to match all select criteria.
    List<StorageUnitEntity> result = storageUnitDao.getS3StorageUnitsToCleanup(MAX_RESULT);
    assertEquals(Arrays.asList(storageUnitEntities.get(3), storageUnitEntities.get(2)), result);
    // Try to retrieve the storage units with max result limit set to 1. Only a single storage unit entity should get selected.
    result = storageUnitDao.getS3StorageUnitsToCleanup(1);
    assertEquals(Collections.singletonList(storageUnitEntities.get(3)), result);
}

Example 50 with StorageUnitEntity

use of org.finra.herd.model.jpa.StorageUnitEntity in project herd by FINRAOS.

the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorage.

@Test
public void testGetStorageUnitByBusinessObjectDataAndStorage() {
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    StorageEntity storageEntity = storageUnitEntity.getStorage();
    // Test retrieval by entities.
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageEntity));
    // Test retrieval failures.
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(), storageEntity));
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageDaoTestHelper.createStorageEntity()));
}