use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataServiceGetBusinessObjectDataTest method testGetBusinessObjectDataNoDataVersionSpecified.
/**
* This test validates business object data get when business object format version is specified, but business object data version is not.
*/
@Test
public void testGetBusinessObjectDataNoDataVersionSpecified() {
// Create business object data status entities.
businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS);
businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS_2);
// Create business object data entities with subpartition values.
List<BusinessObjectDataEntity> businessObjectDataEntities = Arrays.asList(businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID), businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, SECOND_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS), businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID), businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, SECOND_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS), businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, THIRD_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS));
BusinessObjectData resultBusinessObjectData;
// Retrieve a business object data for the initial business object format version.
resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, NO_BDATA_STATUS, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the result for the initial business object format version.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntities.get(0).getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, resultBusinessObjectData);
// Retrieve a business object data for the second business object format version.
resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, NO_BDATA_STATUS, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the result for the second business object format version.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntities.get(2).getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, resultBusinessObjectData);
// Try to retrieve business object data for the third business object format version.
try {
businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, THIRD_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, NO_BDATA_STATUS, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
fail("Should throw an ObjectNotFoundException when latest valid business object data does not exist.");
} catch (ObjectNotFoundException e) {
assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, THIRD_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION, BusinessObjectDataStatusEntity.VALID), e.getMessage());
}
// Retrieve a business object data by explicitly specifying VALID business object data status.
resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, BusinessObjectDataStatusEntity.VALID, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the returned object, it should be the latest VALID business object data version available for the initial business object format version.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntities.get(0).getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, resultBusinessObjectData);
// Retrieve a business object data by explicitly specifying business object data status testing value.
resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, BDATA_STATUS, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the returned object, it should be the latest business object data version with
// the test business object data status available for the initial business object format version.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntities.get(1).getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, SECOND_DATA_VERSION, LATEST_VERSION_FLAG_SET, BDATA_STATUS, resultBusinessObjectData);
// Try to retrieve a business object data by explicitly specifying an incorrect business object data status value.
try {
businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION), PARTITION_KEY, BDATA_STATUS_2, NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
fail("Should throw an ObjectNotFoundException when business object data does not exist.");
} catch (ObjectNotFoundException e) {
assertEquals(businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataNotFoundErrorMessage(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, NO_DATA_VERSION, BDATA_STATUS_2), e.getMessage());
}
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataServiceGetBusinessObjectDataTest method testGetBusinessObjectDataUpperCaseParameters.
@Test
public void testGetBusinessObjectDataUpperCaseParameters() {
// Create and persist a business object data entity using lower case values.
businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE.toLowerCase(), BDEF_NAME.toLowerCase(), FORMAT_USAGE_CODE.toLowerCase(), FORMAT_FILE_TYPE_CODE.toLowerCase(), FORMAT_VERSION, FORMAT_DESCRIPTION, false, PARTITION_KEY.toLowerCase());
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE.toLowerCase(), BDEF_NAME.toLowerCase(), FORMAT_USAGE_CODE.toLowerCase(), FORMAT_FILE_TYPE_CODE.toLowerCase(), FORMAT_VERSION, PARTITION_VALUE.toLowerCase(), convertListToLowerCase(SUBPARTITION_VALUES), DATA_VERSION, true, BDATA_STATUS.toLowerCase());
// Retrieve business object data entity using upper case input parameters (except for case-sensitive partition values).
// Please note that we do not specify business object data version in order for the business object data status to have an effect.
BusinessObjectData resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(NAMESPACE.toUpperCase(), BDEF_NAME.toUpperCase(), FORMAT_USAGE_CODE.toUpperCase(), FORMAT_FILE_TYPE_CODE.toUpperCase(), FORMAT_VERSION, PARTITION_VALUE.toLowerCase(), convertListToLowerCase(SUBPARTITION_VALUES), NO_DATA_VERSION), PARTITION_KEY.toUpperCase(), BDATA_STATUS.toUpperCase(), NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the returned object.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntity.getId(), NAMESPACE.toLowerCase(), BDEF_NAME.toLowerCase(), FORMAT_USAGE_CODE.toLowerCase(), FORMAT_FILE_TYPE_CODE.toLowerCase(), FORMAT_VERSION, PARTITION_VALUE.toLowerCase(), convertListToLowerCase(SUBPARTITION_VALUES), DATA_VERSION, true, BDATA_STATUS.toLowerCase(), resultBusinessObjectData);
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataServiceGetBusinessObjectDataTest method testGetBusinessObjectDataTrimParameters.
@Test
public void testGetBusinessObjectDataTrimParameters() {
// Create and persist a business object data entity.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
// Retrieve the business object data using input parameters with leading and trailing empty spaces.
// Please note that we do not specify business object data version in order for the business object data status to have an effect.
BusinessObjectData resultBusinessObjectData = businessObjectDataService.getBusinessObjectData(new BusinessObjectDataKey(addWhitespace(NAMESPACE), addWhitespace(BDEF_NAME), addWhitespace(FORMAT_USAGE_CODE), addWhitespace(FORMAT_FILE_TYPE_CODE), FORMAT_VERSION, addWhitespace(PARTITION_VALUE), addWhitespace(SUBPARTITION_VALUES), NO_DATA_VERSION), addWhitespace(PARTITION_KEY), addWhitespace(BDATA_STATUS), NO_INCLUDE_BUSINESS_OBJECT_DATA_STATUS_HISTORY, NO_INCLUDE_STORAGE_UNIT_STATUS_HISTORY);
// Validate the returned object.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntity.getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS, resultBusinessObjectData);
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataServiceRestoreBusinessObjectDataTest method testRestoreBusinessObjectDataNonGlacierStorageClass.
@Test
public void testRestoreBusinessObjectDataNonGlacierStorageClass() throws Exception {
// Create S3FileTransferRequestParamsDto to access the S3 bucket.
// Since test S3 key prefix represents a directory, we add a trailing '/' character to it.
S3FileTransferRequestParamsDto glacierS3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(S3_BUCKET_NAME).withS3KeyPrefix(S3_BUCKET_NAME + "/" + TEST_S3_KEY_PREFIX + "/").build();
// Create a business object data key.
BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
// Create database entities required for testing.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataServiceTestHelper.createDatabaseEntitiesForInitiateRestoreTesting(businessObjectDataKey);
// Get the storage unit entity.
StorageUnitEntity storageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(STORAGE_NAME, businessObjectDataEntity);
try {
// Put relative non-Glacier storage class files into the S3 bucket.
for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Standard);
metadata.setOngoingRestore(false);
s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, storageFileEntity.getPath(), new ByteArrayInputStream(new byte[storageFileEntity.getFileSizeBytes().intValue()]), metadata), NO_S3_CLIENT);
}
// Initiate a restore request for the business object data.
BusinessObjectData businessObjectData = businessObjectDataService.restoreBusinessObjectData(businessObjectDataKey, EXPIRATION_IN_DAYS);
// Validate the returned object.
businessObjectDataServiceTestHelper.validateBusinessObjectData(businessObjectDataEntity.getId(), businessObjectDataKey, LATEST_VERSION_FLAG_SET, BDATA_STATUS, businessObjectData);
// Validate that the origin storage unit status is RESTORING.
assertEquals(StorageUnitStatusEntity.RESTORING, storageUnitEntity.getStatus().getCode());
// Validate that there is still no ongoing restore request for all non-Glacier objects.
for (StorageFileEntity storageFileEntity : storageUnitEntity.getStorageFiles()) {
ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(S3_BUCKET_NAME, storageFileEntity.getPath(), NO_S3_CLIENT);
assertFalse(objectMetadata.getOngoingRestore());
}
} finally {
// Delete test files from S3 storage.
if (!s3Dao.listDirectory(glacierS3FileTransferRequestParamsDto).isEmpty()) {
s3Dao.deleteDirectory(glacierS3FileTransferRequestParamsDto);
}
s3Operations.rollback();
}
}
use of org.finra.herd.model.jpa.BusinessObjectDataEntity in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLargePartitionValueListWithAutoDiscovery.
@Test
@Ignore
public void testGenerateBusinessObjectDataDdlLargePartitionValueListWithAutoDiscovery() {
final int PRIMARY_PARTITION_VALUE_LIST_SIZE = 10000;
final int SECOND_LEVEL_PARTITION_VALUES_PER_BUSINESS_OBJECT_DATA = 1;
final int STORAGE_FILES_PER_PARTITION = 1;
// Prepare test data and build a list of partition values to generate business object data DDL for.
// Build a list of schema columns.
List<SchemaColumn> schemaColumns = new ArrayList<>();
schemaColumns.add(new SchemaColumn(PARTITION_KEY, "DATE", NO_COLUMN_SIZE, COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, NO_COLUMN_DESCRIPTION));
schemaColumns.add(new SchemaColumn(COLUMN_NAME, "NUMBER", COLUMN_SIZE, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, COLUMN_DESCRIPTION));
schemaColumns.add(new SchemaColumn(COLUMN_NAME_2, "STRING", NO_COLUMN_SIZE, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, NO_COLUMN_DESCRIPTION));
// Use the first two columns as partition columns.
List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 2);
// Create a business object format entity with the schema.
BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
// Create an S3 storage entity.
StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME);
// Create relative business object data, storage unit, and storage file entities.
List<String> partitionValues = new ArrayList<>();
for (int i = 0; i < PRIMARY_PARTITION_VALUE_LIST_SIZE; i++) {
String partitionValue = String.format("%s-%03d", PARTITION_VALUE, i);
partitionValues.add(partitionValue);
// Create a business object data entity.
BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BusinessObjectDataStatusEntity.VALID);
// Build an S3 key prefix according to the herd S3 naming convention.
String s3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, PARTITION_KEY, partitionValue, null, null, DATA_VERSION);
// Create a storage unit with a storage directory path.
StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix);
// Create storage file entities.
for (int j = 0; j < SECOND_LEVEL_PARTITION_VALUES_PER_BUSINESS_OBJECT_DATA; j++) {
// Build a storage file directory path that includes the relative second level partition value - needed for auto discovery.
String storageFileDirectoryPath = String.format("%s/%s=%s-%03d", s3KeyPrefix, COLUMN_NAME, PARTITION_VALUE_2, j);
for (int k = 0; k < STORAGE_FILES_PER_PARTITION; k++) {
String storageFilePath = String.format("%s/%03d.data", storageFileDirectoryPath, k);
storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, storageFilePath, FILE_SIZE_1_KB, ROW_COUNT_1000);
}
}
herdDao.saveAndRefresh(storageUnitEntity);
}
// Retrieve business object data ddl for the entire list of partition values.
BusinessObjectDataDdl businessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, partitionValues, NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, NO_INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
// Validate the results.
assertNotNull(businessObjectDataDdl);
}
Aggregations