Examples with BusinessObjectDataDdlRequest org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest used on opensource projects

Example 56 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlEscapeBackslashInRowFormat.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlEscapeBackslashInRowFormat() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, BACKSLASH, BACKSLASH, BACKSLASH, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results - please note that we do not escape single backslash in null value.
    String expectedRowFormat = "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\\\' ESCAPED BY '\\\\' NULL DEFINED AS '\\'";
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}

Example 57 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlMissingBusinessObjectDataDoNotAllowMissingData.

@Test
public void testGenerateBusinessObjectDataDdlMissingBusinessObjectDataDoNotAllowMissingData() {
    // Prepare test data.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, CUSTOM_DDL_NAME, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Try to retrieve business object data ddl for the non-existing business object data with "allow missing data" flag set to "false".
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
    request.getPartitionValueFilters().get(0).setPartitionValues(Arrays.asList("I_DO_NOT_EXIST"));
    request.setAllowMissingData(false);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an ObjectNotFoundException when business object data does not exist.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %s, partitionValue: \"%s\", " + "subpartitionValues: \",,,\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion(), request.getPartitionValueFilters().get(0).getPartitionValues().get(request.getPartitionValueFilters().get(0).getPartitionValues().size() - 1), request.getBusinessObjectDataVersion(), request.getStorageName()), e.getMessage());
    }
}

Example 58 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlPartitionValueRangeExpectedPartitionValueMatchesMinPartitionValueToken.

@Test
public void testGenerateBusinessObjectDataDdlPartitionValueRangeExpectedPartitionValueMatchesMinPartitionValueToken() {
    // Prepare test data with expected partition value set to minimum partition value token.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting();
    String startPartitionValue = BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN.replace("minimum", "a");
    String endPartitionValue = BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN.replace("minimum", "z");
    expectedPartitionValueDaoTestHelper.createExpectedPartitionValueEntities(PARTITION_KEY_GROUP, Arrays.asList(startPartitionValue, BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN, endPartitionValue));
    // Try to generate business object data ddl when expected partition value matches to the maximum partition value token.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(startPartitionValue, endPartitionValue, CUSTOM_DDL_NAME);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an IllegalArgumentException when expected partition value matches to the minimum partition value token.");
    } catch (IllegalArgumentException e) {
        assertEquals("A partition value token cannot be specified as one of the expected partition values.", e.getMessage());
    }
}

Example 59 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlNotSupportedSchemaColumnDataType.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlNotSupportedSchemaColumnDataType() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> schemaColumns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    SchemaColumn schemaColumn = new SchemaColumn();
    schemaColumns.add(schemaColumn);
    schemaColumn.setName("COLUMN");
    schemaColumn.setType("UNKNOWN");
    String partitionKey = schemaColumns.get(0).getName();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, schemaColumnDaoTestHelper.getTestPartitionColumns(), false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Try to retrieve business object data ddl for the format that uses unsupported schema column data type.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an IllegalArgumentException when business object format has a column with an unsupported data type.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Column \"%s\" has an unsupported data type \"%s\" in the schema for business object format " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d}.", schemaColumn.getName(), schemaColumn.getType(), request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}

Example 60 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionInvalid.

@Test
public void testGenerateBusinessObjectDataDdlIncludeAllRegisteredSubPartitionsSecondSubPartitionInvalid() {
    // Create two VALID sub-partitions both with "available" storage units in a non-Glacier storage.
    List<StorageUnitEntity> storageUnitEntities = businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1), Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_2)));
    // Update the second sub-partition business object data status to INVALID.
    storageUnitEntities.get(1).getBusinessObjectData().setStatus(businessObjectDataStatusDao.getBusinessObjectDataStatusByCode(BusinessObjectDataStatusEntity.INVALID));
    // Try to retrieve business object data DDL with "IncludeAllRegisteredSubPartitions" option enabled.
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
        fail("Suppose to throw an ObjectNotFoundException when second sub-partition has an INVALID status.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d, partitionValue: \"%s\", " + "subpartitionValues: \"%s\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, PARTITION_VALUE, SUB_PARTITION_VALUE_2, DATA_VERSION, STORAGE_NAME), e.getMessage());
    }
}