Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 81 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlSingleLevelPartitioning.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlSingleLevelPartitioning() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Retrieve business object format ddl without specifying custom ddl name.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 82 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testCreateBusinessObjectFormatInitialVersionExistsWithSchemaNonAdditiveSchemaChangesColumnSizeIncreasedForNotAllowedDataType.

@Test
public void testCreateBusinessObjectFormatInitialVersionExistsWithSchemaNonAdditiveSchemaChangesColumnSizeIncreasedForNotAllowedDataType() {
    // Create relative database entities.
    businessObjectFormatServiceTestHelper.createTestDatabaseEntitiesForBusinessObjectFormatTesting();
    // Create an initial format schema.
    Schema initialSchema = new Schema(Arrays.asList(new SchemaColumn(COLUMN_NAME, COLUMN_DATA_TYPE, COLUMN_SIZE, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, COLUMN_DESCRIPTION)), NO_PARTITION_COLUMNS, SCHEMA_NULL_VALUE_BACKSLASH_N, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, PARTITION_KEY_GROUP);
    // Create an initial version of the business object format.
    businessObjectFormatService.createBusinessObjectFormat(new BusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, NO_ATTRIBUTES, NO_ATTRIBUTE_DEFINITIONS, initialSchema));
    // Create an updated schema having a regular column size increased.
    Schema updatedSchema = (Schema) initialSchema.clone();
    updatedSchema.getColumns().get(0).setSize(COLUMN_SIZE_2);
    try {
        // Try to create a second version of the business object format with a new schema
        // having regular column size increased but for a non-allowed column data type.
        businessObjectFormatService.createBusinessObjectFormat(new BusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, NO_ATTRIBUTES, NO_ATTRIBUTE_DEFINITIONS, updatedSchema));
        fail("Should throw an IllegalArgumentException when the new format version is not \"additive\" to the previous format version.");
    } catch (IllegalArgumentException e) {
        assertEquals("New format version schema is not \"additive\" to the previous format version schema. " + "Non-additive changes detected to the previously defined regular (non-partitioning) columns.", e.getMessage());
    }
}

Example 83 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaDelimiterCharacter.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlMissingSchemaDelimiterCharacter() {
    // Prepare test data.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, null, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Retrieve business object format ddl.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results.
    String expectedRowFormat = "ROW FORMAT DELIMITED NULL DEFINED AS '\\N'";
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 84 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLatestFormatVersionDataNotAvailableInStorage.

@Test
public void testGenerateBusinessObjectDataDdlLatestFormatVersionDataNotAvailableInStorage() {
    // Create database entities for two versions of a business object format.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1 + SUBPARTITION_VALUES.size());
    BusinessObjectFormatEntity businessObjectFormatV0Entity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, NO_LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
    BusinessObjectFormatEntity businessObjectFormatV1Entity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns);
    // Create two storage entities.
    StorageEntity storage1Entity = storageDao.getStorageByName(StorageEntity.MANAGED_STORAGE);
    StorageEntity storage2Entity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2);
    // Register initial version of the business object data for initial format version in both storages.
    BusinessObjectDataEntity businessObjectDataV0V0Entity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatV0Entity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataV0V0Entity), storage1Entity.getName());
    for (StorageEntity storageEntity : Arrays.asList(storage1Entity, storage2Entity)) {
        StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataV0V0Entity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
        storageUnitEntity.setDirectoryPath(s3KeyPrefix);
        herdDao.saveAndRefresh(storageUnitEntity);
    }
    herdDao.saveAndRefresh(businessObjectDataV0V0Entity);
    // Register initial version of the business object data for second format version, but only in the second storage.
    BusinessObjectDataEntity businessObjectDataV1V0Entity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS);
    s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatV1Entity, businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataV1V0Entity), storage2Entity.getName());
    StorageUnitEntity storageUnitEntity = storageUnitDaoTestHelper.createStorageUnitEntity(storage2Entity, businessObjectDataV1V0Entity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    storageUnitEntity.setDirectoryPath(s3KeyPrefix);
    herdDao.saveAndRefresh(storageUnitEntity);
    herdDao.saveAndRefresh(businessObjectDataV1V0Entity);
    // Retrieve business object data ddl for the first storage without specifying business object format version.
    BusinessObjectDataDdlRequest request = new BusinessObjectDataDdlRequest();
    request.setNamespace(NAMESPACE);
    request.setBusinessObjectDefinitionName(BDEF_NAME);
    request.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    request.setBusinessObjectFormatFileType(FileTypeEntity.TXT_FILE_TYPE);
    request.setBusinessObjectFormatVersion(null);
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    request.setPartitionValueFilter(partitionValueFilter);
    partitionValueFilter.setPartitionKey(FIRST_PARTITION_COLUMN_NAME);
    partitionValueFilter.setPartitionValues(Arrays.asList(PARTITION_VALUE));
    request.setBusinessObjectDataVersion(INITIAL_DATA_VERSION);
    request.setStorageName(StorageEntity.MANAGED_STORAGE);
    request.setOutputFormat(BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL);
    request.setTableName(TABLE_NAME);
    request.setCustomDdlName(null);
    request.setIncludeDropTableStatement(true);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    assertNotNull(resultDdl);
}

Example 85 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlPartitionColumnIsAlsoRegularColumn.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlPartitionColumnIsAlsoRegularColumn() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> schemaColumns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    // Override the first schema column to be a partition column.
    schemaColumns.set(0, partitionColumns.get(0));
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl without specifying custom ddl name.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(partitionColumns.size(), "ORGNL_PRTN_CLMN001", "DATE", ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}