Search in sources :

Example 51 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class StorageUnitDaoTestHelper method createStorageUnitEntity.

/**
 * Creates and persists a new storage unit entity.
 *
 * @param storageName the storage name
 * @param storagePlatform the storage platform
 * @param businessObjectDataKey the business object data key
 * @param businessObjectDataLatestVersion specifies if the business object data is flagged as latest version or not
 * @param businessObjectDataStatusCode the business object data status
 * @param storageUnitStatus the storage unit status
 * @param storageDirectoryPath the storage directory path
 *
 * @return the newly created storage unit entity
 */
public StorageUnitEntity createStorageUnitEntity(String storageName, String storagePlatform, BusinessObjectDataKey businessObjectDataKey, Boolean businessObjectDataLatestVersion, String businessObjectDataStatusCode, String storageUnitStatus, String storageDirectoryPath) {
    // Create a storage entity, if not exists.
    StorageEntity storageEntity = storageDao.getStorageByName(storageName);
    if (storageEntity == null) {
        storageEntity = storageDaoTestHelper.createStorageEntity(storageName, storagePlatform);
    }
    // Create a business object data entity, if not exists.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey);
    if (businessObjectDataEntity == null) {
        businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, businessObjectDataLatestVersion, businessObjectDataStatusCode);
    }
    // Create and return a storage unit entity.
    return createStorageUnitEntity(storageEntity, businessObjectDataEntity, storageUnitStatus, storageDirectoryPath);
}
Also used : StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity)

Example 52 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorage.

@Test
public void testGetStorageUnitByBusinessObjectDataAndStorage() {
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    StorageEntity storageEntity = storageUnitEntity.getStorage();
    // Test retrieval by entities.
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageEntity));
    // Test retrieval failures.
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(), storageEntity));
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageDaoTestHelper.createStorageEntity()));
}
Also used : StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity) Test(org.junit.Test)

Example 53 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorageName.

@Test
public void testGetStorageUnitByBusinessObjectDataAndStorageName() {
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
    BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
    // test retrieval by name
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME));
    // test retrieval by name, case insensitive
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME.toUpperCase()));
    assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME.toLowerCase()));
    // test retrieval failure
    assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, "I_DO_NOT_EXIST"));
}
Also used : StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity) Test(org.junit.Test)

Example 54 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
    // Get storage entity.
    StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    // Expected business object format file type to Hive file format mapping.
    HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
    businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
        // Prepare test data for the respective business object format file type.
        List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
        String partitionKey = partitionColumns.get(0).getName();
        BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
        for (String partitionValue : UNSORTED_PARTITION_VALUES) {
            BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
            String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
            // Create two storage files.
            for (int i = 0; i < 2; i++) {
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
            herdDao.saveAndRefresh(storageUnitEntity);
            herdDao.saveAndRefresh(businessObjectDataEntity);
        }
        // Retrieve business object format ddl.
        BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
        request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
        BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        // Validate the results.
        String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
        String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
    }
}
Also used : HashMap(java.util.HashMap) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) BusinessObjectFormatDdl(org.finra.herd.model.api.xml.BusinessObjectFormatDdl) SchemaColumn(org.finra.herd.model.api.xml.SchemaColumn) BusinessObjectFormatDdlRequest(org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest) StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity) BusinessObjectFormatEntity(org.finra.herd.model.jpa.BusinessObjectFormatEntity) Test(org.junit.Test)

Example 55 with BusinessObjectDataEntity

use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.

the class CleanupDestroyedBusinessObjectDataServiceImplTest method createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting.

/**
 * Creates business object data and associated entities required for the cleanup destroyed business object data unit tests.
 */
public BusinessObjectDataStorageUnitKey createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting() {
    List<SchemaColumn> columns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    // Create a business object format entity.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_COMMA, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, columns, partitionColumns);
    // Create S3 storage entity.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    // Create a business object data entity parent.
    BusinessObjectDataEntity businessObjectDataEntityParent = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_3, FORMAT_USAGE_CODE_3, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_3, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
    // Create business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, Lists.newArrayList(businessObjectDataEntityParent));
    // Create a business object data entity child.
    businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, PARTITION_VALUE_2, SUBPARTITION_VALUES_2, DATA_VERSION_2, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, Lists.newArrayList(businessObjectDataEntity));
    // Create and persist a business object data attribute entity.
    businessObjectDataAttributeDaoTestHelper.createBusinessObjectDataAttributeEntity(businessObjectDataEntity, ATTRIBUTE_NAME, ATTRIBUTE_VALUE);
    // Create the storage units.
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.DISABLED, STORAGE_DIRECTORY_PATH);
    storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, STORAGE_DIRECTORY_PATH, FILE_SIZE, ROW_COUNT);
    return storageUnitHelper.createStorageUnitKeyFromEntity(storageUnitEntity);
}
Also used : Attribute(org.finra.herd.model.api.xml.Attribute) StorageUnitEntity(org.finra.herd.model.jpa.StorageUnitEntity) SchemaColumn(org.finra.herd.model.api.xml.SchemaColumn) StorageEntity(org.finra.herd.model.jpa.StorageEntity) BusinessObjectDataEntity(org.finra.herd.model.jpa.BusinessObjectDataEntity)

Aggregations

BusinessObjectDataEntity (org.finra.herd.model.jpa.BusinessObjectDataEntity)278 Test (org.junit.Test)184 BusinessObjectDataKey (org.finra.herd.model.api.xml.BusinessObjectDataKey)138 StorageUnitEntity (org.finra.herd.model.jpa.StorageUnitEntity)105 ArrayList (java.util.ArrayList)70 StorageEntity (org.finra.herd.model.jpa.StorageEntity)67 AbstractServiceTest (org.finra.herd.service.AbstractServiceTest)63 BusinessObjectData (org.finra.herd.model.api.xml.BusinessObjectData)57 BusinessObjectFormatEntity (org.finra.herd.model.jpa.BusinessObjectFormatEntity)54 StorageUnitStatusEntity (org.finra.herd.model.jpa.StorageUnitStatusEntity)31 Attribute (org.finra.herd.model.api.xml.Attribute)28 NotificationMessageDefinition (org.finra.herd.model.api.xml.NotificationMessageDefinition)27 NotificationMessageDefinitions (org.finra.herd.model.api.xml.NotificationMessageDefinitions)27 ConfigurationEntity (org.finra.herd.model.jpa.ConfigurationEntity)27 BusinessObjectDataStatusEntity (org.finra.herd.model.jpa.BusinessObjectDataStatusEntity)25 Predicate (javax.persistence.criteria.Predicate)24 NotificationMessage (org.finra.herd.model.dto.NotificationMessage)24 BusinessObjectDataAttributeEntity (org.finra.herd.model.jpa.BusinessObjectDataAttributeEntity)23 StoragePolicyKey (org.finra.herd.model.api.xml.StoragePolicyKey)21 CriteriaBuilder (javax.persistence.criteria.CriteriaBuilder)19