use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class StoragePolicyDaoTestHelper method createStoragePolicyEntity.
/**
* Creates and persists a storage policy entity.
*
* @param storagePolicyKey the storage policy key
* @param storagePolicyRuleType the storage policy rule type
* @param storagePolicyRuleValue the storage policy rule value
* @param businessObjectDefinitionNamespace the business object definition namespace
* @param businessObjectDefinitionName the business object definition name
* @param businessObjectFormatUsage the business object usage
* @param businessObjectFormatFileType the business object format file type
* @param storageName the storage name
* @param storagePolicyTransitionType the transition type of the storage policy
* @param storagePolicyStatus the storage policy status
* @param storagePolicyVersion the storage policy version
* @param storagePolicyLatestVersion specifies if this storage policy is flagged as latest version or not
*
* @return the newly created storage policy entity
*/
public StoragePolicyEntity createStoragePolicyEntity(StoragePolicyKey storagePolicyKey, String storagePolicyRuleType, Integer storagePolicyRuleValue, String businessObjectDefinitionNamespace, String businessObjectDefinitionName, String businessObjectFormatUsage, String businessObjectFormatFileType, String storageName, String storagePolicyTransitionType, String storagePolicyStatus, Integer storagePolicyVersion, Boolean storagePolicyLatestVersion) {
// Create a storage policy namespace entity if needed.
NamespaceEntity storagePolicyNamespaceEntity = namespaceDao.getNamespaceByCd(storagePolicyKey.getNamespace());
if (storagePolicyNamespaceEntity == null) {
storagePolicyNamespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(storagePolicyKey.getNamespace());
}
// Create a storage policy rule type type entity if needed.
StoragePolicyRuleTypeEntity storagePolicyRuleTypeEntity = storagePolicyRuleTypeDao.getStoragePolicyRuleTypeByCode(storagePolicyRuleType);
if (storagePolicyRuleTypeEntity == null) {
storagePolicyRuleTypeEntity = storagePolicyRuleTypeDaoTestHelper.createStoragePolicyRuleTypeEntity(storagePolicyRuleType, AbstractDaoTest.DESCRIPTION);
}
// Create a business object definition entity if needed.
BusinessObjectDefinitionEntity businessObjectDefinitionEntity = null;
if (StringUtils.isNotBlank(businessObjectDefinitionName)) {
businessObjectDefinitionEntity = businessObjectDefinitionDao.getBusinessObjectDefinitionByKey(new BusinessObjectDefinitionKey(businessObjectDefinitionNamespace, businessObjectDefinitionName));
if (businessObjectDefinitionEntity == null) {
// Create a business object definition.
businessObjectDefinitionEntity = businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(businessObjectDefinitionNamespace, businessObjectDefinitionName, AbstractDaoTest.DATA_PROVIDER_NAME, AbstractDaoTest.BDEF_DESCRIPTION);
}
}
// Create a business object format file type entity if needed.
FileTypeEntity fileTypeEntity = null;
if (StringUtils.isNotBlank(businessObjectFormatFileType)) {
fileTypeEntity = fileTypeDao.getFileTypeByCode(businessObjectFormatFileType);
if (fileTypeEntity == null) {
fileTypeEntity = fileTypeDaoTestHelper.createFileTypeEntity(businessObjectFormatFileType);
}
}
// Create a storage entity of S3 storage platform type if needed.
StorageEntity storageEntity = storageDao.getStorageByName(storageName);
if (storageEntity == null) {
storageEntity = storageDaoTestHelper.createStorageEntity(storageName, StoragePlatformEntity.S3);
}
// Create a storage policy transition type entity, if not exists.
StoragePolicyTransitionTypeEntity storagePolicyTransitionTypeEntity = storagePolicyTransitionTypeDao.getStoragePolicyTransitionTypeByCode(storagePolicyTransitionType);
if (storagePolicyTransitionTypeEntity == null) {
storagePolicyTransitionTypeEntity = createStoragePolicyTransitionTypeEntity(storagePolicyTransitionType);
}
// Create a storage policy status entity, if not exists.
StoragePolicyStatusEntity storagePolicyStatusEntity = storagePolicyStatusDao.getStoragePolicyStatusByCode(storagePolicyStatus);
if (storagePolicyStatusEntity == null) {
storagePolicyStatusEntity = createStoragePolicyStatusEntity(storagePolicyStatus);
}
// Create a storage policy entity.
StoragePolicyEntity storagePolicyEntity = new StoragePolicyEntity();
storagePolicyEntity.setNamespace(storagePolicyNamespaceEntity);
storagePolicyEntity.setName(storagePolicyKey.getStoragePolicyName());
storagePolicyEntity.setStoragePolicyRuleType(storagePolicyRuleTypeEntity);
storagePolicyEntity.setStoragePolicyRuleValue(storagePolicyRuleValue);
storagePolicyEntity.setBusinessObjectDefinition(businessObjectDefinitionEntity);
storagePolicyEntity.setUsage(businessObjectFormatUsage);
storagePolicyEntity.setFileType(fileTypeEntity);
storagePolicyEntity.setStorage(storageEntity);
storagePolicyEntity.setStoragePolicyTransitionType(storagePolicyTransitionTypeEntity);
storagePolicyEntity.setStatus(storagePolicyStatusEntity);
storagePolicyEntity.setVersion(storagePolicyVersion);
storagePolicyEntity.setLatestVersion(storagePolicyLatestVersion);
return storagePolicyDao.saveAndRefresh(storagePolicyEntity);
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class BusinessObjectDataDaoImpl method getBusinessObjectDataPartitionValue.
/**
* Retrieves partition value per specified parameters that includes the aggregate function.
*
* <p>
* Returns null if the business object format key does not exist.
*
* @param partitionColumnPosition the partition column position (1-based numbering)
* @param businessObjectFormatKey the business object format key (case-insensitive). If a business object format version isn't specified, the latest
* available format version for each partition value will be used.
* @param businessObjectDataVersion the business object data version. If a business object data version isn't specified, the latest data version based on
* the specified business object data status will be used for each partition value.
* @param businessObjectDataStatus the business object data status. This parameter is ignored when the business object data version is specified.
* @param storageNames the optional list of storage names (case-insensitive)
* @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
* @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
* the storage platform type is specified
* @param aggregateFunction the aggregate function to use against partition values
* @param upperBoundPartitionValue the optional inclusive upper bound for the maximum available partition value
* @param lowerBoundPartitionValue the optional inclusive lower bound for the maximum available partition value
*
* @return the partition value
*/
private String getBusinessObjectDataPartitionValue(int partitionColumnPosition, final BusinessObjectFormatKey businessObjectFormatKey, final Integer businessObjectDataVersion, String businessObjectDataStatus, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, final AggregateFunction aggregateFunction, String upperBoundPartitionValue, String lowerBoundPartitionValue) {
// We cannot use businessObjectFormatKey passed in since it is case-insensitive. Case-insensitive values requires upper() function in the SQL query, and
// it has caused performance problems. So we need to extract case-sensitive business object format key from database so we can eliminate the upper()
// function.
BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(businessObjectFormatKey);
BusinessObjectFormatKey businessObjectFormatKeyCaseSensitive = (BusinessObjectFormatKey) businessObjectFormatKey.clone();
if (businessObjectFormatEntity == null) {
// Returns null if business object format key does not exist.
return null;
} else {
// Sets the exact values for business object format key from the database. Note that usage type is still case-insensitive.
businessObjectFormatKeyCaseSensitive.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
businessObjectFormatKeyCaseSensitive.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
businessObjectFormatKeyCaseSensitive.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
}
// Create the criteria builder and the criteria.
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
CriteriaQuery<String> criteria = builder.createQuery(String.class);
// The criteria root is the business object data.
Root<BusinessObjectDataEntity> businessObjectDataEntityRoot = criteria.from(BusinessObjectDataEntity.class);
// Join to the other tables we can filter on.
Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> businessObjectFormatEntityJoin = businessObjectDataEntityRoot.join(BusinessObjectDataEntity_.businessObjectFormat);
Join<BusinessObjectFormatEntity, FileTypeEntity> fileTypeEntityJoin = businessObjectFormatEntityJoin.join(BusinessObjectFormatEntity_.fileType);
Join<BusinessObjectFormatEntity, BusinessObjectDefinitionEntity> businessObjectDefinitionEntityJoin = businessObjectFormatEntityJoin.join(BusinessObjectFormatEntity_.businessObjectDefinition);
Join<BusinessObjectDefinitionEntity, NamespaceEntity> namespaceEntityJoin = businessObjectDefinitionEntityJoin.join(BusinessObjectDefinitionEntity_.namespace);
Join<BusinessObjectDataEntity, StorageUnitEntity> storageUnitEntityJoin = businessObjectDataEntityRoot.join(BusinessObjectDataEntity_.storageUnits);
Join<StorageUnitEntity, StorageUnitStatusEntity> storageUnitStatusEntityJoin = storageUnitEntityJoin.join(StorageUnitEntity_.status);
Join<StorageUnitEntity, StorageEntity> storageEntityJoin = storageUnitEntityJoin.join(StorageUnitEntity_.storage);
Join<StorageEntity, StoragePlatformEntity> storagePlatformEntityJoin = storageEntityJoin.join(StorageEntity_.storagePlatform);
// Create the path.
Expression<String> partitionValue;
SingularAttribute<BusinessObjectDataEntity, String> singleValuedAttribute = BUSINESS_OBJECT_DATA_PARTITIONS.get(partitionColumnPosition - 1);
switch(aggregateFunction) {
case GREATEST:
partitionValue = builder.greatest(businessObjectDataEntityRoot.get(singleValuedAttribute));
break;
case LEAST:
partitionValue = builder.least(businessObjectDataEntityRoot.get(singleValuedAttribute));
break;
default:
throw new IllegalArgumentException("Invalid aggregate function found: \"" + aggregateFunction + "\".");
}
// Create the standard restrictions (i.e. the standard where clauses).
Predicate mainQueryRestriction = builder.equal(namespaceEntityJoin.get(NamespaceEntity_.code), businessObjectFormatKeyCaseSensitive.getNamespace());
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectDefinitionEntityJoin.get(BusinessObjectDefinitionEntity_.name), businessObjectFormatKeyCaseSensitive.getBusinessObjectDefinitionName()));
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(builder.upper(businessObjectFormatEntityJoin.get(BusinessObjectFormatEntity_.usage)), businessObjectFormatKeyCaseSensitive.getBusinessObjectFormatUsage().toUpperCase()));
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(fileTypeEntityJoin.get(FileTypeEntity_.code), businessObjectFormatKeyCaseSensitive.getBusinessObjectFormatFileType()));
// If a business object format version was specified, use it.
if (businessObjectFormatKey.getBusinessObjectFormatVersion() != null) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectFormatEntityJoin.get(BusinessObjectFormatEntity_.businessObjectFormatVersion), businessObjectFormatKey.getBusinessObjectFormatVersion()));
}
// If a data version was specified, use it.
if (businessObjectDataVersion != null) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.equal(businessObjectDataEntityRoot.get(BusinessObjectDataEntity_.version), businessObjectDataVersion));
} else // Business object data version is not specified, so get the latest one as per specified business object data status in the specified storage.
{
Subquery<Integer> subQuery = getMaximumBusinessObjectDataVersionSubQuery(builder, criteria, businessObjectDataEntityRoot, businessObjectFormatEntityJoin, businessObjectDataStatus, storageNames, storagePlatformType, excludedStoragePlatformType, false);
mainQueryRestriction = builder.and(mainQueryRestriction, builder.in(businessObjectDataEntityRoot.get(BusinessObjectDataEntity_.version)).value(subQuery));
}
// Add an inclusive upper bound partition value restriction if specified.
if (upperBoundPartitionValue != null) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.lessThanOrEqualTo(businessObjectDataEntityRoot.get(singleValuedAttribute), upperBoundPartitionValue));
}
// Add an inclusive lower bound partition value restriction if specified.
if (lowerBoundPartitionValue != null) {
mainQueryRestriction = builder.and(mainQueryRestriction, builder.greaterThanOrEqualTo(businessObjectDataEntityRoot.get(singleValuedAttribute), lowerBoundPartitionValue));
}
// If specified, add restriction on storage.
mainQueryRestriction = builder.and(mainQueryRestriction, getQueryRestrictionOnStorage(builder, storageEntityJoin, storagePlatformEntityJoin, storageNames, storagePlatformType, excludedStoragePlatformType));
// Search across only "available" storage units.
mainQueryRestriction = builder.and(mainQueryRestriction, builder.isTrue(storageUnitStatusEntityJoin.get(StorageUnitStatusEntity_.available)));
criteria.select(partitionValue).where(mainQueryRestriction);
return entityManager.createQuery(criteria).getSingleResult();
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class BusinessObjectDataDaoImpl method getBusinessObjectDataFromStorageOlderThan.
@Override
public List<BusinessObjectDataEntity> getBusinessObjectDataFromStorageOlderThan(String storageName, int thresholdMinutes, List<String> businessObjectDataStatusesToIgnore) {
// Create the criteria builder and the criteria.
CriteriaBuilder builder = entityManager.getCriteriaBuilder();
CriteriaQuery<BusinessObjectDataEntity> criteria = builder.createQuery(BusinessObjectDataEntity.class);
// The criteria root is the business object data.
Root<BusinessObjectDataEntity> businessObjectDataEntity = criteria.from(BusinessObjectDataEntity.class);
// Join to the other tables we can filter on.
Join<BusinessObjectDataEntity, StorageUnitEntity> storageUnitEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.storageUnits);
Join<StorageUnitEntity, StorageEntity> storageEntity = storageUnitEntity.join(StorageUnitEntity_.storage);
Join<BusinessObjectDataEntity, BusinessObjectDataStatusEntity> businessObjectDataStatusEntity = businessObjectDataEntity.join(BusinessObjectDataEntity_.status);
// Compute threshold timestamp based on the current database timestamp and threshold minutes.
Timestamp thresholdTimestamp = HerdDateUtils.addMinutes(getCurrentTimestamp(), -thresholdMinutes);
// Create the standard restrictions (i.e. the standard where clauses).
Predicate queryRestriction = builder.equal(builder.upper(storageEntity.get(StorageEntity_.name)), storageName.toUpperCase());
queryRestriction = builder.and(queryRestriction, builder.not(businessObjectDataStatusEntity.get(BusinessObjectDataStatusEntity_.code).in(businessObjectDataStatusesToIgnore)));
queryRestriction = builder.and(queryRestriction, builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.createdOn), thresholdTimestamp));
// Order the results by file path.
Order orderByCreatedOn = builder.asc(businessObjectDataEntity.get(BusinessObjectDataEntity_.createdOn));
// Add the clauses for the query.
criteria.select(businessObjectDataEntity).where(queryRestriction).orderBy(orderByCreatedOn);
return entityManager.createQuery(criteria).getResultList();
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class BusinessObjectDataStorageFileServiceTest method createDataWithSubPartitions.
private void createDataWithSubPartitions() {
NamespaceEntity namespaceEntity = super.namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
DataProviderEntity dataProviderEntity = super.dataProviderDaoTestHelper.createDataProviderEntity(DATA_PROVIDER_NAME);
BusinessObjectDefinitionEntity businessObjectDefinitionEntity = super.businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(namespaceEntity, BDEF_NAME, dataProviderEntity, NO_BDEF_DESCRIPTION, NO_BDEF_DISPLAY_NAME, NO_ATTRIBUTES, NO_SAMPLE_DATA_FILES);
FileTypeEntity fileTypeEntity = super.fileTypeDaoTestHelper.createFileTypeEntity(FORMAT_FILE_TYPE_CODE, FORMAT_DESCRIPTION);
List<SchemaColumn> schemaColumns = new ArrayList<>();
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_2);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_3);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_4);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
{
SchemaColumn schemaColumn = new SchemaColumn();
schemaColumn.setName(PARTITION_KEY_5);
schemaColumn.setType("STRING");
schemaColumns.add(schemaColumn);
}
BusinessObjectFormatEntity businessObjectFormatEntity = super.businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(businessObjectDefinitionEntity, FORMAT_USAGE_CODE, fileTypeEntity, FORMAT_VERSION, null, true, PARTITION_KEY, null, NO_ATTRIBUTES, null, null, null, schemaColumns, null);
BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, PARTITION_VALUE, SUB_PARTITION_VALUES, DATA_VERSION, true, businessObjectDataStatusEntity.getCode());
StorageEntity storageEntity = super.storageDaoTestHelper.createStorageEntity(STORAGE_NAME);
StorageUnitEntity storageUnitEntity = super.storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
super.storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, FILE_PATH_1, FILE_SIZE_1_KB, null);
}
use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.
the class NotificationEventServiceTest method testProcessBusinessObjectDataRegistrationNotificationEventSyncMultipleNotifications.
@Test
public void testProcessBusinessObjectDataRegistrationNotificationEventSyncMultipleNotifications() throws Exception {
// Create job definition
JobDefinition jobDefinition = jobDefinitionServiceTestHelper.createJobDefinition(ACTIVITI_XML_LOG_VARIABLES_NO_REGEX_WITH_CLASSPATH);
List<JobAction> jobActions = new ArrayList<>();
jobActions.add(new JobAction(jobDefinition.getNamespace(), jobDefinition.getJobName(), CORRELATION_DATA));
// Create business object data with storage units.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3);
storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
// Create and persist a business object data notification registration entity.
notificationRegistrationDaoTestHelper.createBusinessObjectDataNotificationRegistrationEntity(new NotificationRegistrationKey(NAMESPACE, NOTIFICATION_NAME), NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN.name(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, STORAGE_NAME, BDATA_STATUS, null, jobActions, NotificationRegistrationStatusEntity.ENABLED);
notificationRegistrationDaoTestHelper.createBusinessObjectDataNotificationRegistrationEntity(new NotificationRegistrationKey(NAMESPACE, NOTIFICATION_NAME_2), NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN.name(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, STORAGE_NAME, BDATA_STATUS, null, jobActions, NotificationRegistrationStatusEntity.ENABLED);
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
// Trigger the notification
List<Object> notificationActions = notificationEventService.processBusinessObjectDataNotificationEventSync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_RGSTN, businessObjectDataKey, BDATA_STATUS, null);
assertTrue(notificationActions.size() == 2);
for (Object notificationAction : notificationActions) {
Job job = (Job) notificationAction;
assertNotNull(job);
assertEquals(TEST_ACTIVITI_NAMESPACE_CD, job.getNamespace());
assertEquals(TEST_ACTIVITI_JOB_NAME, job.getJobName());
Parameter parameter = null;
for (Parameter param : job.getParameters()) {
if (param.getName().equals("notification_correlationData")) {
parameter = param;
break;
}
}
assertNotNull(parameter);
assertEquals(CORRELATION_DATA, parameter.getValue());
}
}
Aggregations