Examples with BusinessObjectDataDdl org.finra.herd.model.api.xml.BusinessObjectDataDdl used on opensource projects

Example 31 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlPartitionColumnIsAlsoRegularColumn.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlPartitionColumnIsAlsoRegularColumn() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> schemaColumns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    // Override the first schema column to be a partition column.
    schemaColumns.set(0, partitionColumns.get(0));
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl without specifying custom ddl name.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(partitionColumns.size(), "ORGNL_PRTN_CLMN001", "DATE", ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}

Example 32 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlSubpartitionKeysHaveHyphens.

@Test
public void testGenerateBusinessObjectDataDdlSubpartitionKeysHaveHyphens() {
    // Prepare test data with subpartition using key values with hyphens instead of underscores.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, true, CUSTOM_DDL_NAME, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.  Please note that we expect hyphens in subpartition key values.
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, true, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}

Example 33 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitions.

@Test
public void testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitions() {
    // Create two VALID sub-partitions both with "available" storage units in a non-Glacier storage.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1), Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_2)));
    // Retrieve business object data DDL with flag set to suppress scan for unregistered sub-partitions.
    BusinessObjectDataDdl resultBusinessObjectDataDdl = businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
    // Validate the response object. Both sub-partitions should be present in the generated DDL.
    assertEquals(new BusinessObjectDataDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdlTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1), Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_2)))), resultBusinessObjectDataDdl);
}

Example 34 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlEscapeBackslashInRowFormat.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlEscapeBackslashInRowFormat() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, BACKSLASH, BACKSLASH, BACKSLASH, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results - please note that we do not escape single backslash in null value.
    String expectedRowFormat = "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\\\' ESCAPED BY '\\\\' NULL DEFINED AS '\\'";
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}

Example 35 with BusinessObjectDataDdl

use of org.finra.herd.model.api.xml.BusinessObjectDataDdl in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNotAvailableStorageUnitDoNotAllowMissingData.

@Test
public void testGenerateBusinessObjectDataDdlNotAvailableStorageUnitDoNotAllowMissingData() {
    // Prepare database entities required for testing.
    StorageUnitEntity storageUnitEntity = businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(PARTITION_VALUE);
    // Generate DDL for a collection of business object data.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(Arrays.asList(PARTITION_VALUE));
    request.setIncludeDropPartitions(true);
    request.setAllowMissingData(false);
    BusinessObjectDataDdl result = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the response object.
    assertEquals(new BusinessObjectDataDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_VALUE)), result);
    // Update the storage unit status to a non-available one.
    storageUnitEntity.setStatus(storageUnitStatusDaoTestHelper.createStorageUnitStatusEntity(STORAGE_UNIT_STATUS, DESCRIPTION, NO_STORAGE_UNIT_STATUS_AVAILABLE_FLAG_SET));
    herdDao.saveAndRefresh(storageUnitEntity);
    // Try to retrieve business object data ddl when storage unit has a non-available status and with "allow missing data" flag set to "false".
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an ObjectNotFoundException when business object data is not available.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %s, partitionValue: \"%s\", " + "subpartitionValues: \",,,\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion(), request.getPartitionValueFilters().get(0).getPartitionValues().get(request.getPartitionValueFilters().get(0).getPartitionValues().size() - 1), request.getBusinessObjectDataVersion(), request.getStorageName()), e.getMessage());
    }
}