Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 96 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceCheckBusinessObjectDataAvailabilityTest method testCheckBusinessObjectDataAvailabilityMultiplePartitionValueFilters.

/**
 * This test validates the getBusinessObjectDataEntities() functionality when request contains multiple partition value filters and when business object
 * data entities actually exist.
 */
@Test
public void testCheckBusinessObjectDataAvailabilityMultiplePartitionValueFilters() {
    final int PRIMARY_PARTITION_VALUES_SIZE = 5;
    String[][] subPartitionValues = new String[][] { { null, null, null, null }, { "A", null, null, null }, { "A", "X", null, null }, { "A", "X", "0", null }, { "A", "X", "0", "0" }, { "A", "X", "0", "1" }, { "A", "X", "1", "0" }, { "A", "X", "1", "1" }, { "A", "Y", "0", "0" }, { "A", "Y", "0", "1" }, { "A", "Y", "1", "0" }, { "A", "Y", "1", "1" } };
    String[] secondStorageSubPartitionValues = new String[] { "A", "X", "2", "2" };
    // Create the relative database entities.
    List<SchemaColumn> columns = schemaColumnDaoTestHelper.getTestSchemaColumns();
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    // Create a business object format entity with the schema.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, partitionKey, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, columns, partitionColumns);
    // Create two storage entities if they do not exist.
    List<StorageEntity> storageEntities = Arrays.asList(storageDaoTestHelper.createStorageEntity(STORAGE_NAME), storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2));
    // Create the relative business object data entities with the relative storage units.
    // Please note that we will create some extra business object data entities that
    // will not be selected as available per partition value mismatch or a different storage.
    List<String> primaryPartitionValues = new ArrayList<>();
    for (int i = 0; i < PRIMARY_PARTITION_VALUES_SIZE; i++) {
        BusinessObjectDataEntity businessObjectDataEntity;
        String primaryPartitionValue = String.format("%s-%s", PARTITION_VALUE, i);
        primaryPartitionValues.add(primaryPartitionValue);
        for (String[] subPartitionValuesSet : subPartitionValues) {
            businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, Arrays.asList(subPartitionValuesSet), DATA_VERSION, true, BusinessObjectDataStatusEntity.VALID);
            storageUnitDaoTestHelper.createStorageUnitEntity(storageEntities.get(0), businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
        }
        // Add an extra business object data that is only present in the second storage.
        businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, primaryPartitionValue, Arrays.asList(secondStorageSubPartitionValues), DATA_VERSION, true, BusinessObjectDataStatusEntity.VALID);
        storageUnitDaoTestHelper.createStorageUnitEntity(storageEntities.get(1), businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    }
    // Check business object data availability request with a large list of partition values using multiple partition value filters.
    BusinessObjectDataAvailabilityRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataAvailabilityRequest(null);
    List<PartitionValueFilter> partitionValueFilters = new ArrayList<>();
    request.setPartitionValueFilters(partitionValueFilters);
    partitionValueFilters.add(new PartitionValueFilter(partitionColumns.get(0).getName(), primaryPartitionValues, NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
    partitionValueFilters.add(new PartitionValueFilter(partitionColumns.get(1).getName(), Arrays.asList("A"), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
    partitionValueFilters.add(new PartitionValueFilter(partitionColumns.get(2).getName(), Arrays.asList("X", "Y"), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
    partitionValueFilters.add(new PartitionValueFilter(partitionColumns.get(3).getName(), Arrays.asList("0", "1"), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
    partitionValueFilters.add(new PartitionValueFilter(partitionColumns.get(4).getName(), Arrays.asList("0", "1"), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE));
    request.setBusinessObjectFormatVersion(null);
    request.setBusinessObjectDataVersion(null);
    BusinessObjectDataAvailability resultAvailability = businessObjectDataService.checkBusinessObjectDataAvailability(request);
    // Validate the results.
    assertNotNull(resultAvailability);
    assertEquals(PRIMARY_PARTITION_VALUES_SIZE * 8, resultAvailability.getAvailableStatuses().size());
    assertEquals(0, resultAvailability.getNotAvailableStatuses().size());
}

Example 97 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLatestFormatHasLessPartitionColumnsThenBusinessObjectData.

/**
 * This test case reproduces an error when business object data has more or equal sub-partition values then the latest business object format version.
 */
@Test
public void testGenerateBusinessObjectDataDdlLatestFormatHasLessPartitionColumnsThenBusinessObjectData() {
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(PARTITION_KEY, "DATE", NO_COLUMN_SIZE, COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(COLUMN_NAME, "NUMBER", COLUMN_SIZE, NO_COLUMN_REQUIRED, COLUMN_DEFAULT_VALUE, COLUMN_DESCRIPTION));
    // Build two list of partition columns, so the second format version would have one less partition column defined.
    List<SchemaColumn> partitionColumns1 = schemaColumns.subList(0, 2);
    List<SchemaColumn> partitionColumns2 = schemaColumns.subList(0, 1);
    // Create an initial version of business object format with the schema having two partition columns.
    BusinessObjectFormatEntity businessObjectFormatEntity1 = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, FORMAT_DESCRIPTION, NO_LATEST_VERSION_FLAG_SET, PARTITION_KEY, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns1);
    // Create a second version of business object format with the schema having only one partition column.
    businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION, FORMAT_DESCRIPTION, LATEST_VERSION_FLAG_SET, PARTITION_KEY, NO_PARTITION_KEY_GROUP, NO_ATTRIBUTES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns2);
    // Provide sub-partition column name and value.
    List<SchemaColumn> subPartitionColumns = partitionColumns1.subList(1, 2);
    List<String> subPartitionValues = Arrays.asList(SUBPARTITION_VALUES.get(0));
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity1, PARTITION_VALUE, subPartitionValues, DATA_VERSION, true, BusinessObjectDataStatusEntity.VALID);
    // Create an S3 storage entity.
    // Add the "bucket name" attribute to the storage along with the key prefix velocity template.
    StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
    // Build an S3 key prefix according to the Data Management S3 naming convention.
    String s3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, PARTITION_KEY, PARTITION_VALUE, subPartitionColumns.toArray(new SchemaColumn[subPartitionColumns.size()]), subPartitionValues.toArray(new String[subPartitionValues.size()]), DATA_VERSION);
    // Create a storage unit with a storage directory path.
    storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix);
    // or equal sub-partition values then the latest business object format version.
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, null, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, null, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
        fail("Suppose to throw an IllegalArgumentException when business object data has more or " + "equal sub-partition values then the latest business object format version.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Number of subpartition values specified for the business object data is greater than or equal to " + "the number of partition columns defined in the schema of the business object format selected for DDL generation. " + "Business object data: {%s},  business object format: {%s}", businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, subPartitionValues, DATA_VERSION), businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatKeyAsString(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, SECOND_FORMAT_VERSION)), e.getMessage());
    }
}

Example 98 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlMissingSchemaDelimiterCharacter.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlMissingSchemaDelimiterCharacter() {
    // Prepare test data.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, null, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Retrieve business object data ddl.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    String expectedRowFormat = "ROW FORMAT DELIMITED NULL DEFINED AS '\\N'";
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}