use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class StorageUnitDaoTestHelper method createStorageUnitEntity.
/**
* Creates and persists a new storage unit entity.
*
* @param storageName the storage name
* @param storagePlatform the storage platform
* @param businessObjectDataKey the business object data key
* @param businessObjectDataLatestVersion specifies if the business object data is flagged as latest version or not
* @param businessObjectDataStatusCode the business object data status
* @param storageUnitStatus the storage unit status
* @param storageDirectoryPath the storage directory path
*
* @return the newly created storage unit entity
*/
public StorageUnitEntity createStorageUnitEntity(String storageName, String storagePlatform, BusinessObjectDataKey businessObjectDataKey, Boolean businessObjectDataLatestVersion, String businessObjectDataStatusCode, String storageUnitStatus, String storageDirectoryPath) {
// Create a storage entity, if not exists.
StorageEntity storageEntity = storageDao.getStorageByName(storageName);
if (storageEntity == null) {
storageEntity = storageDaoTestHelper.createStorageEntity(storageName, storagePlatform);
}
// Create a business object data entity, if not exists.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey);
if (businessObjectDataEntity == null) {
businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectDataKey, businessObjectDataLatestVersion, businessObjectDataStatusCode);
}
// Create and return a storage unit entity.
return createStorageUnitEntity(storageEntity, businessObjectDataEntity, storageUnitStatus, storageDirectoryPath);
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorage.
@Test
public void testGetStorageUnitByBusinessObjectDataAndStorage() {
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
StorageEntity storageEntity = storageUnitEntity.getStorage();
// Test retrieval by entities.
assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageEntity));
// Test retrieval failures.
assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(), storageEntity));
assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorage(businessObjectDataEntity, storageDaoTestHelper.createStorageEntity()));
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class StorageUnitDaoTest method testGetStorageUnitByBusinessObjectDataAndStorageName.
@Test
public void testGetStorageUnitByBusinessObjectDataAndStorageName() {
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, STORAGE_UNIT_STATUS, STORAGE_DIRECTORY_PATH);
BusinessObjectDataEntity businessObjectDataEntity = storageUnitEntity.getBusinessObjectData();
// test retrieval by name
assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME));
// test retrieval by name, case insensitive
assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME.toUpperCase()));
assertEquals(storageUnitEntity, storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, STORAGE_NAME.toLowerCase()));
// test retrieval failure
assertNull(storageUnitDao.getStorageUnitByBusinessObjectDataAndStorageName(businessObjectDataEntity, "I_DO_NOT_EXIST"));
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.
@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
// Get storage entity.
StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
// Expected business object format file type to Hive file format mapping.
HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
// Prepare test data for the respective business object format file type.
List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
String partitionKey = partitionColumns.get(0).getName();
BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
for (String partitionValue : UNSORTED_PARTITION_VALUES) {
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
// Create two storage files.
for (int i = 0; i < 2; i++) {
storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
}
herdDao.saveAndRefresh(storageUnitEntity);
herdDao.saveAndRefresh(businessObjectDataEntity);
}
// Retrieve business object format ddl.
BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
// Validate the results.
String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
}
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class CleanupDestroyedBusinessObjectDataServiceImplTest method createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting.
/**
* Creates business object data and associated entities required for the cleanup destroyed business object data unit tests.
*/
public BusinessObjectDataStorageUnitKey createDatabaseEntitiesForCleanupDestroyedBusinessObjectDataTesting() {
List<SchemaColumn> columns = schemaColumnDaoTestHelper.getTestSchemaColumns();
List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
// Create a business object format entity.
businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_COMMA, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, columns, partitionColumns);
// Create S3 storage entity.
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
// Create a business object data entity parent.
BusinessObjectDataEntity businessObjectDataEntityParent = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_3, FORMAT_USAGE_CODE_3, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE_3, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
// Create business object data entity.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.DELETED, Lists.newArrayList(businessObjectDataEntityParent));
// Create a business object data entity child.
businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, FORMAT_VERSION_2, PARTITION_VALUE_2, SUBPARTITION_VALUES_2, DATA_VERSION_2, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, Lists.newArrayList(businessObjectDataEntity));
// Create and persist a business object data attribute entity.
businessObjectDataAttributeDaoTestHelper.createBusinessObjectDataAttributeEntity(businessObjectDataEntity, ATTRIBUTE_NAME, ATTRIBUTE_VALUE);
// Create the storage units.
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.DISABLED, STORAGE_DIRECTORY_PATH);
storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, STORAGE_DIRECTORY_PATH, FILE_SIZE, ROW_COUNT);
return storageUnitHelper.createStorageUnitKeyFromEntity(storageUnitEntity);
}
Aggregations