Examples with BusinessObjectFormatDdlRequest org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest used on opensource projects

Example 16 with BusinessObjectFormatDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlAllKnownFileTypes() {
    // Get storage entity.
    StorageEntity storageEntity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    // Expected business object format file type to Hive file format mapping.
    HashMap<String, String> businessObjectFormatFileTypeMap = new HashMap<>();
    businessObjectFormatFileTypeMap.put(FileTypeEntity.BZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.GZ_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.ORC_FILE_TYPE, Hive13DdlGenerator.ORC_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.PARQUET_FILE_TYPE, Hive13DdlGenerator.PARQUET_HIVE_FILE_FORMAT);
    businessObjectFormatFileTypeMap.put(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT);
    for (String businessObjectFormatFileType : businessObjectFormatFileTypeMap.keySet()) {
        // Prepare test data for the respective business object format file type.
        List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
        String partitionKey = partitionColumns.get(0).getName();
        BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, businessObjectDefinitionServiceTestHelper.getNewAttributes(), SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
        for (String partitionValue : UNSORTED_PARTITION_VALUES) {
            BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, partitionValue, NO_SUBPARTITION_VALUES, DATA_VERSION, true, BDATA_STATUS);
            String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), storageEntity.getName());
            StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
            // Create two storage files.
            for (int i = 0; i < 2; i++) {
                storageFileDaoTestHelper.createStorageFileEntity(storageUnitEntity, String.format("%s/data%d.dat", s3KeyPrefix, i), FILE_SIZE_1_KB, ROW_COUNT_1000);
            }
            herdDao.saveAndRefresh(storageUnitEntity);
            herdDao.saveAndRefresh(businessObjectDataEntity);
        }
        // Retrieve business object format ddl.
        BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
        request.setBusinessObjectFormatFileType(businessObjectFormatFileType);
        BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        // Validate the results.
        String expectedHiveFileFormat = businessObjectFormatFileTypeMap.get(businessObjectFormatFileType);
        String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, expectedHiveFileFormat, businessObjectFormatFileType, true, true);
        businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, businessObjectFormatFileType, FORMAT_VERSION, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, null, expectedDdl, resultDdl);
    }
}

Example 17 with BusinessObjectFormatDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlNotSupportedSchemaColumnDataType.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlNotSupportedSchemaColumnDataType() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> schemaColumns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    SchemaColumn schemaColumn = new SchemaColumn();
    schemaColumns.add(schemaColumn);
    schemaColumn.setName("COLUMN");
    schemaColumn.setType("UNKNOWN");
    String partitionKey = schemaColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, schemaColumnDaoTestHelper.getTestPartitionColumns(), null);
    // Try to retrieve business object format ddl for the format that uses unsupported schema column data type.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    try {
        businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        fail("Should throw an IllegalArgumentException when business object format has a column with an unsupported data type.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Column \"%s\" has an unsupported data type \"%s\" in the schema for business object format " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d}.", schemaColumn.getName(), schemaColumn.getType(), request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}

Example 18 with BusinessObjectFormatDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlNoPartitioning.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlNoPartitioning() {
    // Prepare test data without custom ddl.
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, Hive13DdlGenerator.NO_PARTITIONING_PARTITION_KEY, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), null, null);
    // Retrieve business object format ddl for a non-partitioned table and without specifying custom ddl name.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(0, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 19 with BusinessObjectFormatDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest in project herd by FINRAOS.

the class BusinessObjectFormatServiceTestHelper method getTestBusinessObjectFormatDdlRequest.

/**
 * Creates and returns a business object format ddl request using passed parameters along with some hard-coded test values.
 *
 * @param customDdlName the custom DDL name
 *
 * @return the newly created business object format ddl request
 */
public BusinessObjectFormatDdlRequest getTestBusinessObjectFormatDdlRequest(String customDdlName) {
    BusinessObjectFormatDdlRequest request = new BusinessObjectFormatDdlRequest();
    request.setNamespace(AbstractServiceTest.NAMESPACE);
    request.setBusinessObjectDefinitionName(AbstractServiceTest.BDEF_NAME);
    request.setBusinessObjectFormatUsage(AbstractServiceTest.FORMAT_USAGE_CODE);
    request.setBusinessObjectFormatFileType(FileTypeEntity.TXT_FILE_TYPE);
    request.setBusinessObjectFormatVersion(AbstractServiceTest.FORMAT_VERSION);
    request.setOutputFormat(BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL);
    request.setTableName(AbstractServiceTest.TABLE_NAME);
    request.setCustomDdlName(customDdlName);
    request.setIncludeDropTableStatement(true);
    request.setIncludeIfNotExistsOption(true);
    return request;
}

Example 20 with BusinessObjectFormatDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectFormatDdlRequest in project herd by FINRAOS.

the class Hive13DdlGeneratorTest method testGenerateReplaceColumnsStatement.

/**
 * Asserts that generateReplaceColumnsStatement() generates the correct DDL statement.
 */
@Test
public void testGenerateReplaceColumnsStatement() {
    BusinessObjectFormatDdlRequest businessObjectFormatDdlRequest = new BusinessObjectFormatDdlRequest();
    businessObjectFormatDdlRequest.setTableName(TABLE_NAME);
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, true, PARTITION_KEY);
    {
        SchemaColumnEntity schemaColumnEntity = new SchemaColumnEntity();
        schemaColumnEntity.setPosition(0);
        schemaColumnEntity.setName("col1");
        schemaColumnEntity.setType("varchar");
        schemaColumnEntity.setSize("255");
        schemaColumnEntity.setDescription("lorem ipsum");
        businessObjectFormatEntity.getSchemaColumns().add(schemaColumnEntity);
    }
    {
        SchemaColumnEntity schemaColumnEntity = new SchemaColumnEntity();
        schemaColumnEntity.setPosition(1);
        schemaColumnEntity.setName("col2");
        schemaColumnEntity.setType("date");
        businessObjectFormatEntity.getSchemaColumns().add(schemaColumnEntity);
    }
    String actual = hive13DdlGenerator.generateReplaceColumnsStatement(businessObjectFormatDdlRequest, businessObjectFormatEntity);
    String expected = "ALTER TABLE `" + businessObjectFormatDdlRequest.getTableName() + "` REPLACE COLUMNS (\n" + "    `col1` VARCHAR(255) COMMENT 'lorem ipsum',\n" + "    `col2` DATE);";
    Assert.assertEquals("generated DDL", expected, actual);
}