Examples with BusinessObjectFormatDdl org.finra.herd.model.api.xml.BusinessObjectFormatDdl used on opensource projects

Example 11 with BusinessObjectFormatDdl

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdl in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaNullValue.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaNullValue() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, null, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Retrieve business object format ddl.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedRowFormat = "ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' ESCAPED BY '\\\\' NULL DEFINED AS ''";
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 12 with BusinessObjectFormatDdl

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdl in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
    // Get storage entity.
    StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    // Expected business object format file type to Hive file format mapping.
    HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
    businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
        // Prepare test data for the respective business object format file type.
        List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
        String partitionKey = partitionColumns.get(0).getName();
        BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
        for (String partitionValue : UNSORTED_PARTITION_VALUES) {
            BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
            String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
            // Create two storage files.
            for (int i = 0; i < 2; i++) {
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
            herdDao.saveAndRefresh(storageUnitEntity);
            herdDao.saveAndRefresh(businessObjectDataEntity);
        }
        // Retrieve business object format ddl.
        BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
        request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
        BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        // Validate the results.
        String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
        String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
    }
}

Example 13 with BusinessObjectFormatDdl

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdl in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlNoPartitioning.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlNoPartitioning() {
    // Prepare test data without custom ddl.
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.NO_PARTITIONING_PARTITION_KEY, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), null, null);
    // Retrieve business object format ddl for a non-partitioned table and without specifying custom ddl name.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(0, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 14 with BusinessObjectFormatDdl

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdl in project herd by FINRAOS.

the class BusinessObjectFormatRestControllerTest method testGenerateBusinessObjectFormatDdl.

@Test
public void testGenerateBusinessObjectFormatDdl() {
    BusinessObjectFormatDdl ddl = new BusinessObjectFormatDdl();
    ddl.setBusinessObjectDefinitionName(BDEF_NAME);
    ddl.setCustomDdlName(CUSTOM_DDL_NAME);
    ddl.setNamespace(NAMESPACE);
    ddl.setDdl(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(AbstractServiceTest.PARTITION_COLUMNS.length, AbstractServiceTest.FIRST_COLUMN_NAME, AbstractServiceTest.FIRST_COLUMN_DATA_TYPE, AbstractServiceTest.ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true));
    // Retrieve business object format ddl.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(CUSTOM_DDL_NAME);
    when(businessObjectFormatService.generateBusinessObjectFormatDdl(request)).thenReturn(ddl);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatRestController.generateBusinessObjectFormatDdl(request);
    // Verify the external calls.
    verify(businessObjectFormatService).generateBusinessObjectFormatDdl(request);
    verifyNoMoreInteractions(businessObjectFormatService);
    // Validate the returned object.
    assertEquals(ddl, resultDdl);
}

Example 15 with BusinessObjectFormatDdl

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdl in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaEscapeCharacter.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaEscapeCharacter() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, SCHEMA_DELIMITER_PIPE, null, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Retrieve business object format ddl.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedRowFormat = "ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' NULL DEFINED AS '\\N'";
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}