Examples with StorageEntity org.finra.herd.model.jpa.StorageEntity used on opensource projects

Example 51 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class NotificationEventServiceTest method runProcessStorageUnitStatusChangeNotificationEventSyncTest.

private void runProcessStorageUnitStatusChangeNotificationEventSyncTest() throws Exception {
    // Create a job definition.
    JobDefinition jobDefinition = jobDefinitionServiceTestHelper.createJobDefinition(ACTIVITI_XML_LOG_VARIABLES_NO_REGEX_WITH_CLASSPATH);
    // Create a job action.
    List<JobAction> jobActions = new ArrayList<>();
    jobActions.add(new JobAction(jobDefinition.getNamespace(), jobDefinition.getJobName(), CORRELATION_DATA));
    // Create a business object format with a schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), schemaColumnDaoTestHelper.getTestPartitionColumns());
    // Create a business object data with a storage unit having a DISABLED status.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3);
    storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH);
    // Create and persist a storage unit notification registration entity.
    notificationRegistrationDaoTestHelper.createStorageUnitNotificationRegistrationEntity(new NotificationRegistrationKey(NAMESPACE, NOTIFICATION_NAME), NotificationEventTypeEntity.EventTypesStorageUnit.STRGE_UNIT_STTS_CHG.name(), BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, STORAGE_NAME, StorageUnitStatusEntity.ENABLED, StorageUnitStatusEntity.DISABLED, jobActions, NotificationRegistrationStatusEntity.ENABLED);
    // Trigger the notification.
    List<Object> notificationActions = notificationEventService.processStorageUnitNotificationEventSync(NotificationEventTypeEntity.EventTypesStorageUnit.STRGE_UNIT_STTS_CHG, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), STORAGE_NAME, StorageUnitStatusEntity.ENABLED, StorageUnitStatusEntity.DISABLED);
    // Validate the result job.
    Job job = (Job) notificationActions.get(0);
    assertEquals(new Job(job.getId(), null, TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME, null, null, null, null, Arrays.asList(new Parameter("notification_businessObjectDefinitionName", BDEF_NAME), new Parameter("notification_partitionValues", PARTITION_VALUE + "|" + StringUtils.join(SUBPARTITION_VALUES, "|")), new Parameter("notification_namespace", NAMESPACE), new Parameter("notification_businessObjectData", jsonHelper.objectToJson(businessObjectDataHelper.createBusinessObjectDataFromEntity(businessObjectDataEntity))), new Parameter("notification_businessObjectFormatUsage", FORMAT_USAGE_CODE), new Parameter(HERD_WORKFLOW_ENVIRONMENT, configurationHelper.getProperty(ConfigurationValue.HERD_ENVIRONMENT)), new Parameter("notification_businessObjectDefinitionNamespace", BDEF_NAMESPACE), new Parameter(ATTRIBUTE_NAME_1_MIXED_CASE, ATTRIBUTE_VALUE_1), new Parameter("notification_storageUnitEventType", NotificationEventTypeEntity.EventTypesStorageUnit.STRGE_UNIT_STTS_CHG.name()), new Parameter("notification_oldStorageUnitStatus", StorageUnitStatusEntity.DISABLED), new Parameter("notification_businessObjectDataVersion", DATA_VERSION.toString()), new Parameter("notification_name", NOTIFICATION_NAME), new Parameter("notification_partitionColumnNames", "PRTN_CLMN001|PRTN_CLMN002|PRTN_CLMN003|PRTN_CLMN004|PRTN_CLMN005"), new Parameter("notification_businessObjectFormatVersion", FORMAT_VERSION.toString()), new Parameter("notification_businessObjectFormatFileType", FORMAT_FILE_TYPE_CODE), new Parameter("notification_storageName", STORAGE_NAME), new Parameter("notification_newStorageUnitStatus", StorageUnitStatusEntity.ENABLED), new Parameter("notification_correlationData", CORRELATION_DATA)), null, null, null), job);
}

Example 52 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataStorageFileServiceTest method createData.

private void createData(String storageUnitDirectory, boolean s3Managed, Collection<String> files) {
    // Create and persist a "pre-registration" business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
    // Create and persist a business object data entity.
    BusinessObjectDataEntity bod = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, businessObjectDataStatusEntity.getCode());
    StorageEntity s;
    if (s3Managed) {
        s = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    } else {
        s = super.storageDaoTestHelper.createStorageEntity(STORAGE_NAME, STORAGE_PLATFORM_CODE);
    }
    StorageUnitEntity su = super.storageUnitDaoTestHelper.createStorageUnitEntity(s, bod, StorageUnitStatusEntity.ENABLED, storageUnitDirectory);
    for (String file : files) {
        StorageFileEntity f = super.storageFileDaoTestHelper.createStorageFileEntity(su, file, FILE_SIZE_1_KB, ROW_COUNT_1000);
        su.getStorageFiles().add(f);
    }
}

Example 53 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getNewBusinessObjectDataCreateRequest.

/**
 * Gets a newly created business object data create request.
 *
 * @param includeAttributes If true, attribute definitions and attributes will be included. Otherwise, not.
 *
 * @return the business object create request.
 */
public BusinessObjectDataCreateRequest getNewBusinessObjectDataCreateRequest(boolean includeAttributes) {
    // Crete a test business object format (and associated data).
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(includeAttributes);
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity();
    // Create a request to create business object data.
    BusinessObjectDataCreateRequest businessObjectDataCreateRequest = new BusinessObjectDataCreateRequest();
    businessObjectDataCreateRequest.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
    businessObjectDataCreateRequest.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
    businessObjectDataCreateRequest.setBusinessObjectFormatUsage(businessObjectFormatEntity.getUsage());
    businessObjectDataCreateRequest.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
    businessObjectDataCreateRequest.setBusinessObjectFormatVersion(businessObjectFormatEntity.getBusinessObjectFormatVersion());
    businessObjectDataCreateRequest.setPartitionKey(businessObjectFormatEntity.getPartitionKey());
    businessObjectDataCreateRequest.setPartitionValue(AbstractServiceTest.PARTITION_VALUE);
    businessObjectDataCreateRequest.setSubPartitionValues(AbstractServiceTest.SUBPARTITION_VALUES);
    List<StorageUnitCreateRequest> storageUnits = new ArrayList<>();
    businessObjectDataCreateRequest.setStorageUnits(storageUnits);
    StorageUnitCreateRequest storageUnit = new StorageUnitCreateRequest();
    storageUnits.add(storageUnit);
    storageUnit.setStorageName(storageEntity.getName());
    StorageDirectory storageDirectory = new StorageDirectory();
    storageUnit.setStorageDirectory(storageDirectory);
    storageDirectory.setDirectoryPath("Folder");
    List<StorageFile> storageFiles = new ArrayList<>();
    storageUnit.setStorageFiles(storageFiles);
    StorageFile storageFile1 = new StorageFile();
    storageFiles.add(storageFile1);
    storageFile1.setFilePath("Folder/file1.gz");
    storageFile1.setFileSizeBytes(0L);
    storageFile1.setRowCount(0L);
    StorageFile storageFile2 = new StorageFile();
    storageFiles.add(storageFile2);
    storageFile2.setFilePath("Folder/file2.gz");
    storageFile2.setFileSizeBytes(2999L);
    storageFile2.setRowCount(1000L);
    StorageFile storageFile3 = new StorageFile();
    storageFiles.add(storageFile3);
    storageFile3.setFilePath("Folder/file3.gz");
    storageFile3.setFileSizeBytes(Long.MAX_VALUE);
    storageFile3.setRowCount(Long.MAX_VALUE);
    if (includeAttributes) {
        businessObjectDataCreateRequest.setAttributes(businessObjectDefinitionServiceTestHelper.getNewAttributes());
    }
    List<BusinessObjectDataKey> businessObjectDataParents = new ArrayList<>();
    businessObjectDataCreateRequest.setBusinessObjectDataParents(businessObjectDataParents);
    // Create 2 parents.
    for (int i = 0; i < 2; i++) {
        BusinessObjectDataEntity parentBusinessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity();
        BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey();
        businessObjectDataKey.setNamespace(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getNamespace().getCode());
        businessObjectDataKey.setBusinessObjectDefinitionName(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getName());
        businessObjectDataKey.setBusinessObjectFormatUsage(parentBusinessObjectDataEntity.getBusinessObjectFormat().getUsage());
        businessObjectDataKey.setBusinessObjectFormatFileType(parentBusinessObjectDataEntity.getBusinessObjectFormat().getFileType().getCode());
        businessObjectDataKey.setBusinessObjectFormatVersion(parentBusinessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectFormatVersion());
        businessObjectDataKey.setPartitionValue(parentBusinessObjectDataEntity.getPartitionValue());
        businessObjectDataKey.setBusinessObjectDataVersion(parentBusinessObjectDataEntity.getVersion());
        businessObjectDataKey.setSubPartitionValues(businessObjectDataHelper.getSubPartitionValues(parentBusinessObjectDataEntity));
        businessObjectDataParents.add(businessObjectDataKey);
    }
    return businessObjectDataCreateRequest;
}

Example 54 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels.

/**
 * Creates and persists VALID two-level partitioned business object data with "available" storage units in an S3 storage.
 *
 * @param partitions the list of partitions, where each is represented by a primary value and a sub-partition value
 *
 * @return the list of created storage unit entities
 */
public List<StorageUnitEntity> createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(List<List<String>> partitions) {
    // Create a list of storage unit entities to be returned.
    List<StorageUnitEntity> result = new ArrayList<>();
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, "DATE", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME, "STRING", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.COLUMN_NAME, "NUMBER", AbstractServiceTest.COLUMN_SIZE, AbstractServiceTest.NO_COLUMN_REQUIRED, AbstractServiceTest.COLUMN_DEFAULT_VALUE, AbstractServiceTest.COLUMN_DESCRIPTION));
    // Use the first two columns as partition columns.
    List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 2);
    // Create a business object format entity with the schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FORMAT_DESCRIPTION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, AbstractServiceTest.NO_PARTITION_KEY_GROUP, AbstractServiceTest.NO_ATTRIBUTES, AbstractServiceTest.SCHEMA_DELIMITER_PIPE, AbstractServiceTest.SCHEMA_ESCAPE_CHARACTER_BACKSLASH, AbstractServiceTest.SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
    // Create an S3 storage entity.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    for (List<String> partition : partitions) {
        // Build an S3 key prefix according to the herd S3 naming convention.
        String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(AbstractServiceTest.NAMESPACE, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, partition.get(0), partitionColumns.subList(1, 2).toArray(new SchemaColumn[1]), Arrays.asList(partition.get(1)).toArray(new String[1]), AbstractServiceTest.DATA_VERSION);
        // Create a business object data entity.
        BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partition.get(0), Arrays.asList(partition.get(1)), AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        // Create an "available" storage unit with a storage directory path.
        result.add(storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix));
    }
    return result;
}

Example 55 with StorageEntity

use of org.finra.herd.model.jpa.StorageEntity in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForFinalizeRestoreTesting.

/**
 * Create and persist database entities required for the finalize restore testing.
 *
 * @param businessObjectDataKey the business object data key
 * @param storageName the storage name
 * @param s3BucketName the S3 bucket name
 * @param s3StorageUnitStatus the storage unit status
 *
 * @return the business object data entity
 */
public BusinessObjectDataEntity createDatabaseEntitiesForFinalizeRestoreTesting(BusinessObjectDataKey businessObjectDataKey, String storageName, String s3BucketName, String s3StorageUnitStatus) {
    // Create
    // Create and persist a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.BDATA_STATUS);
    // Create and persist an S3 storage entity.
    StorageEntity storageEntity;
    if (s3BucketName != null) {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), s3BucketName), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    } else {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, StoragePlatformEntity.S3);
    }
    // Create and persist a storage unit entity.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, s3StorageUnitStatus, AbstractServiceTest.NO_STORAGE_DIRECTORY_PATH);
    // Get the expected S3 key prefix for the business object data key.
    String s3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(businessObjectDataKey, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.PARTITION_KEY, AbstractServiceTest.NO_SUB_PARTITION_KEYS);
    // Create and add storage file entities to the storage unit.
    for (String relativeFilePath : AbstractServiceTest.LOCAL_FILES) {
        storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/%s", s3KeyPrefix, relativeFilePath), AbstractServiceTest.FILE_SIZE_1_KB, AbstractServiceTest.ROW_COUNT);
    }
    // Return the business object data entity.
    return businessObjectDataEntity;
}