use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlInvalidParameters.
@Test
public void testGenerateBusinessObjectDataDdlInvalidParameters() {
// Prepare test data.
businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting();
BusinessObjectDataDdlRequest request;
// Try to retrieve business object data ddl using non-existing format.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
request.setBusinessObjectDefinitionName("I_DO_NOT_EXIST");
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an ObjectNotFoundException when non-existing business object format is used.");
} catch (ObjectNotFoundException e) {
assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage(request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
}
// Try to retrieve business object data ddl using non-existing partition key (partition column).
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
request.getPartitionValueFilters().get(0).setPartitionKey("I_DO_NOT_EXIST");
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when non-existing partition key is used.");
} catch (IllegalArgumentException e) {
assertEquals(String.format("The partition key \"%s\" does not exist in first %d partition columns in the schema for business object format " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", businessObjectFormatFileType: \"%s\", " + "businessObjectFormatVersion: %d}.", request.getPartitionValueFilters().get(0).getPartitionKey(), BusinessObjectDataEntity.MAX_SUBPARTITIONS + 1, request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
}
// Try to retrieve business object data ddl when both partition value filter and partition value filter list are specified.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
request.setPartitionValueFilter(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME.toUpperCase(), new ArrayList<>(UNSORTED_PARTITION_VALUES), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when both a list of partition value filters and a standalone partition value filter are specified.");
} catch (IllegalArgumentException e) {
assertEquals("A list of partition value filters and a standalone partition value filter cannot be both specified.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value filter has none or more than one partition value filter option specified.
for (PartitionValueFilter partitionValueFilter : businessObjectDataServiceTestHelper.getInvalidPartitionValueFilters()) {
request = new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(partitionValueFilter), NO_STANDALONE_PARTITION_VALUE_FILTER, DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, NO_INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value filter has more than one partition value filter option.");
} catch (IllegalArgumentException e) {
assertEquals("Exactly one partition value filter option must be specified.", e.getMessage());
}
}
// Try to retrieve business object data ddl when partition value token is specified with a partition value range.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN, END_PARTITION_VALUE);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified with a partition value range.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified with a partition value range.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value token is specified with a partition value range.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN, END_PARTITION_VALUE);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified with a partition value range.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified with a partition value range.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value token is specified with a partition value range.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(START_PARTITION_VALUE, BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified with a partition value range.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified with a partition value range.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value token is specified with a partition value range.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(END_PARTITION_VALUE, BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified with a partition value range.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified with a partition value range.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value filter has start partition value which is greater than the end partition value.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(END_PARTITION_VALUE, START_PARTITION_VALUE, CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when start partition value which is greater than the end partition value.");
} catch (IllegalArgumentException e) {
assertEquals(String.format("The start partition value \"%s\" cannot be greater than the end partition value \"%s\".", END_PARTITION_VALUE, START_PARTITION_VALUE), e.getMessage());
}
// Try to retrieve business object data ddl when partition value filter has specifies a range that results in no valid partition values.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest("I_DO_NOT_EXIST_1", "I_DO_NOT_EXIST_2", null, CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value filter has specifies a range that results in no valid partition values.");
} catch (IllegalArgumentException e) {
assertEquals(String.format("Partition value range [\"%s\", \"%s\"] contains no valid partition values in partition key group \"%s\". " + "Business object format: {%s}", request.getPartitionValueFilters().get(0).getPartitionValueRange().getStartPartitionValue(), request.getPartitionValueFilters().get(0).getPartitionValueRange().getEndPartitionValue(), PARTITION_KEY_GROUP, businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatKeyAsString(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION)), e.getMessage());
}
// Try to retrieve business object data ddl when partition value filter has specifies a range that results in no valid partition values.
String invalidPartitionValue = "INVALID_PARTITION_VALUE_/";
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(Arrays.asList(invalidPartitionValue), CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value contains a '/' character.");
} catch (IllegalArgumentException e) {
assertEquals(String.format("Partition value \"%s\" can not contain a '/' character.", request.getPartitionValueFilters().get(0).getPartitionValues().get(0)), e.getMessage());
}
// Try to retrieve business object data ddl when partition value token is specified as a partition value.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(Arrays.asList(BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN), CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified as a partition value.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified as one of partition values.", e.getMessage());
}
// Try to retrieve business object data ddl when partition value token is specified as a partition value.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(Arrays.asList(BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN), CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when partition value token is specified as a partition value.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified as one of partition values.", e.getMessage());
}
// Try to retrieve business object data ddl when both a list of storage names and standalone storage name are specified.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
request.setStorageName(BLANK_TEXT);
request.setStorageNames(STORAGE_NAMES);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when both a list of storage names and standalone storage name are specified.");
} catch (IllegalArgumentException e) {
assertEquals("A list of storage names and a standalone storage name cannot be both specified.", e.getMessage());
}
// Try to retrieve business object data ddl passing a non-existing storage as a standalone storage name.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
request.setStorageName("I_DO_NOT_EXIST");
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an ObjectNotFoundException when non-existing storage is used as a standalone storage name.");
} catch (ObjectNotFoundException e) {
assertEquals(String.format("Storage with name \"%s\" doesn't exist.", request.getStorageName()), e.getMessage());
}
// Try to retrieve business object data ddl passing a non-existing storage in the list of storage names.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
request.setStorageName(null);
request.setStorageNames(Arrays.asList("I_DO_NOT_EXIST"));
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an ObjectNotFoundException when non-existing storage is used in the list of storage names.");
} catch (ObjectNotFoundException e) {
assertEquals(String.format("Storage with name \"%s\" doesn't exist.", request.getStorageNames().get(0)), e.getMessage());
}
// Try to retrieve business object data ddl using non-existing custom ddl.
request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
request.setCustomDdlName("I_DO_NOT_EXIST");
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an ObjectNotFoundException when non-existing custom ddl is used.");
} catch (ObjectNotFoundException e) {
assertEquals(String.format("Custom DDL with name \"%s\" does not exist for business object format with namespace \"%s\", " + "business object definition name \"%s\", format usage \"%s\", format file type \"%s\", and format version \"%d\".", request.getCustomDdlName(), request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
}
}
use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlPartitionValueRangeExpectedPartitionValueMatchesMaxPartitionValueToken.
@Test
public void testGenerateBusinessObjectDataDdlPartitionValueRangeExpectedPartitionValueMatchesMaxPartitionValueToken() {
// Prepare test data with expected partition value set to the maximum partition value token.
businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting();
String startPartitionValue = BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN.replace("maximum", "a");
String endPartitionValue = BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN.replace("maximum", "z");
expectedPartitionValueDaoTestHelper.createExpectedPartitionValueEntities(PARTITION_KEY_GROUP, Arrays.asList(startPartitionValue, BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN, endPartitionValue));
// Try to generate business object data ddl when expected partition value matches to the maximum partition value token.
BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(startPartitionValue, endPartitionValue, CUSTOM_DDL_NAME);
try {
businessObjectDataService.generateBusinessObjectDataDdl(request);
fail("Should throw an IllegalArgumentException when expected partition value matches to the maximum partition value token.");
} catch (IllegalArgumentException e) {
assertEquals("A partition value token cannot be specified as one of the expected partition values.", e.getMessage());
}
}
use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionDeletedBdataArchived.
@Test
public void testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionDeletedBdataArchived() {
// Create two VALID sub-partitions both with "available" storage units in a non-Glacier storage.
List<StorageUnitEntity> storageUnitEntities = businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1), Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_2)));
// Update the second sub-partition business object data status to DELETED.
storageUnitEntities.get(1).getBusinessObjectData().setStatus(businessObjectDataStatusDao.getBusinessObjectDataStatusByCode(BusinessObjectDataStatusEntity.DELETED));
// Update the second sub-partition storage unit status to ARCHIVED.
storageUnitEntities.get(1).setStatus(storageUnitStatusDao.getStorageUnitStatusByCode(StorageUnitStatusEntity.ARCHIVED));
// Retrieve business object data DDL with "IncludeAllRegisteredSubPartitions" option enabled.
BusinessObjectDataDdl resultBusinessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
// Validate the response object. Both sub-partitions should. Only the first sub-partition should be present in the generated DDL.
assertEquals(new BusinessObjectDataDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdlTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1)))), resultBusinessObjectDataDdl);
}
use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlMissingBusinessObjectDataAllowMissingDataAllDataNoExists.
@Test
public void testGenerateBusinessObjectDataDdlMissingBusinessObjectDataAllowMissingDataAllDataNoExists() {
// Prepare test data.
List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, CUSTOM_DDL_NAME, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
// Retrieve business object data ddl when all of the business object data is not available and "allow missing data" flag is set to "true".
BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(Arrays.asList("I_DO_NOT_EXIST"), CUSTOM_DDL_NAME);
assertTrue(request.isAllowMissingData());
BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
// Validate the results.
String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, null, null, false, true, true);
businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}
use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.
the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlMissingOptionalParametersPartitionValueRange.
@Test
public void testGenerateBusinessObjectDataDdlMissingOptionalParametersPartitionValueRange() {
// Prepare test data without custom ddl.
businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), schemaColumnDaoTestHelper.getTestPartitionColumns(), false, CUSTOM_DDL_NAME, true, NO_ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
expectedPartitionValueDaoTestHelper.createExpectedPartitionValueProcessDatesForApril2014(PARTITION_KEY_GROUP);
// Retrieve business object data ddl request without optional parameters.
BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(START_PARTITION_VALUE, END_PARTITION_VALUE);
request.setBusinessObjectFormatVersion(null);
request.setBusinessObjectDataVersion(null);
request.setStorageName(null);
request.setIncludeDropTableStatement(null);
request.setIncludeIfNotExistsOption(null);
request.setIncludeAllRegisteredSubPartitions(null);
BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
// Validate the results.
String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, PROCESS_DATE_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, false, false);
businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}
Aggregations