Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 36 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlForwardSlashInPartitionColumnName.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlForwardSlashInPartitionColumnName() {
    // Prepare test data without custom ddl.
    String invalidPartitionColumnName = "INVALID_/_PRTN_CLMN";
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    partitionColumns.get(0).setName(invalidPartitionColumnName);
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, PARTITION_KEY, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Try to retrieve business object data ddl for the format that uses unsupported schema column data type.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    request.getPartitionValueFilters().get(0).setPartitionKey(invalidPartitionColumnName);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an IllegalArgumentException when partition column name contains a '/' character.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Partition column name \"%s\" can not contain a '/' character. Business object format: " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d}", invalidPartitionColumnName, request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}

Example 37 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlUsingStorageDirectoriesNoAutoDiscovery.

@Test
public void testGenerateBusinessObjectDataDdlUsingStorageDirectoriesNoAutoDiscovery() {
    // Prepare test data with storage units having no storage files, but only the relative storage directory path values.
    // For auto-discovery not to occur, number of partition columns is equal to the number of partition values.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1 + SUBPARTITION_VALUES.size());
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, FIRST_PARTITION_COLUMN_NAME, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, CUSTOM_DDL_NAME, false, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    BusinessObjectDataDdlRequest request;
    BusinessObjectDataDdl resultDdl;
    // Retrieve business object data ddl.
    request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
    resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    String expectedDdl = businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, STORAGE_1_AVAILABLE_PARTITION_VALUES, SUBPARTITION_VALUES, false, true, true);
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, expectedDdl, resultDdl);
}

Example 38 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixUndefinedVelocityVariable.

@Test
public void testGetS3KeyPrefixUndefinedVelocityVariable() {
    // Create database entities required for testing.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(true);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    // Create an undefined velocity variable.
    String undefinedVelocityVariable = "$UNDEFINED_VARIABLE";
    // Create an S3 storage entity with the S3 key prefix template containing an undefined variable.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), undefinedVelocityVariable)));
    // Try to get an S3 key prefix when the S3 key prefix velocity template contains an undefined variable.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME_2, false);
        fail("Should throw an MethodInvocationException when the S3 key prefix velocity template contains an undefined variable.");
    } catch (MethodInvocationException e) {
        assertEquals(String.format("Variable %s has not been set at %s[line 1, column 1]", undefinedVelocityVariable, configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE)), e.getMessage());
    }
}

Example 39 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefix.

@Test
public void testGetS3KeyPrefix() {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, SUBPARTITION_VALUES.size() + 1);
    // Get an S3 key prefix by passing all parameters including partition key, business object data version,
    // and "create new version" flag (has no effect when data version is specified).
    S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
    // Get the expected S3 key prefix value using the business object data version.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), DATA_VERSION);
    // Validate the results.
    assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
}

Example 40 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixUpperCaseParameters.

@Test
public void testGetS3KeyPrefixUpperCaseParameters() {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    List<SchemaColumn> testSubPartitionColumns = testPartitionColumns.subList(1, SUBPARTITION_VALUES.size() + 1);
    // Get an S3 key prefix using upper case input parameters (except for case-sensitive partition values).
    S3KeyPrefixInformation resultS3KeyPrefixInformation = storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE.toUpperCase(), BDEF_NAME.toUpperCase(), FORMAT_USAGE_CODE.toUpperCase(), FORMAT_FILE_TYPE_CODE.toUpperCase(), FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey.toUpperCase(), STORAGE_NAME.toUpperCase(), false);
    // Get the expected S3 key prefix value using the business object data version.
    String expectedS3KeyPrefix = getExpectedS3KeyPrefix(NAMESPACE, DATA_PROVIDER_NAME, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, testPartitionKey, PARTITION_VALUE, testSubPartitionColumns.toArray(new SchemaColumn[testSubPartitionColumns.size()]), SUBPARTITION_VALUES.toArray(new String[SUBPARTITION_VALUES.size()]), DATA_VERSION);
    // Validate the results.
    assertEquals(new S3KeyPrefixInformation(expectedS3KeyPrefix), resultS3KeyPrefixInformation);
}