Examples with BusinessObjectDataDdlRequest org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest used on opensource projects

Example 31 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlSingleLevelPartitioningPartitionValueRange.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlSingleLevelPartitioningPartitionValueRange() {
    // Prepare test data without custom ddl and with partition key using NO_PARTITIONING_PARTITION_KEY.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns().subList(0, 1);
    String partitionKey = Hive13DdlGenerator.NO_PARTITIONING_PARTITION_KEY;
    partitionColumns.get(0).setName(partitionKey);
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, PARTITION_KEY_GROUP, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, NO_SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    expectedPartitionValueDaoTestHelper.createExpectedPartitionValueProcessDatesForApril2014(PARTITION_KEY_GROUP);
    // Retrieve business object data ddl without specifying custom ddl name.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(START_PARTITION_VALUE, END_PARTITION_VALUE);
    request.getPartitionValueFilters().get(0).setPartitionKey(partitionKey);
    businessObjectDataService.generateBusinessObjectDataDdl(request);
    // TODO: Validate the results.
    businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(partitionColumns.size(), FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, ROW_FORMAT, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, PROCESS_DATE_PARTITION_VALUES, NO_SUBPARTITION_VALUES, false, true, true);
}

Example 32 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitionsDirectoryPathMismatchS3KeyPrefix.

@Test
public void testGenerateBusinessObjectDataDdlSuppressScanForUnregisteredSubPartitionsDirectoryPathMismatchS3KeyPrefix() {
    // Create one VALID sub-partition with an "available" storage unit a non-Glacier storage.
    List<StorageUnitEntity> storageUnitEntities = businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTestingTwoPartitionLevels(Arrays.asList(Arrays.asList(PARTITION_VALUE, SUB_PARTITION_VALUE_1)));
    // Save the original S3 key prefix (directory path) and update the storage unit with a directory path that does not match the S3 key prefix.
    String originalS3KeyPrefix = storageUnitEntities.get(0).getDirectoryPath();
    storageUnitEntities.get(0).setDirectoryPath(BLANK_TEXT);
    // sub-partitions when its storage unit directory path does not match the expected S3 key prefix.
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage directory path \"%s\" registered with business object data {%s} in \"%s\" storage does not match " + "the expected S3 key prefix \"%s\".", BLANK_TEXT, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_1), DATA_VERSION)), STORAGE_NAME, originalS3KeyPrefix), e.getMessage());
    }
}

Example 33 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlForwardSlashInPartitionColumnName.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlForwardSlashInPartitionColumnName() {
    // Prepare test data without custom ddl.
    String invalidPartitionColumnName = "INVALID_/_PRTN_CLMN";
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    partitionColumns.get(0).setName(invalidPartitionColumnName);
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, PARTITION_KEY, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, SUBPARTITION_VALUES, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Try to retrieve business object data ddl for the format that uses unsupported schema column data type.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    request.getPartitionValueFilters().get(0).setPartitionKey(invalidPartitionColumnName);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an IllegalArgumentException when partition column name contains a '/' character.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Partition column name \"%s\" can not contain a '/' character. Business object format: " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d}", invalidPartitionColumnName, request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}

Example 34 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLowerCaseParameters.

@Test
public void testGenerateBusinessObjectDataDdlLowerCaseParameters() {
    // Prepare test data.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting();
    // Retrieve business object data ddl request with all parameter values in lower case (except for case-sensitive partition values).
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES, CUSTOM_DDL_NAME);
    request.setBusinessObjectDefinitionName(request.getBusinessObjectDefinitionName().toLowerCase());
    request.setBusinessObjectFormatUsage(request.getBusinessObjectFormatUsage().toLowerCase());
    request.setBusinessObjectFormatFileType(request.getBusinessObjectFormatFileType().toLowerCase());
    request.getPartitionValueFilters().get(0).setPartitionKey(request.getPartitionValueFilters().get(0).getPartitionKey().toLowerCase());
    request.setStorageName(request.getStorageName().toLowerCase());
    request.setCustomDdlName(request.getCustomDdlName().toLowerCase());
    BusinessObjectDataDdl resultDdl = businessObjectDataService.generateBusinessObjectDataDdl(request);
    // Validate the results.
    businessObjectDataServiceTestHelper.validateBusinessObjectDataDdl(request, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataDdl(), resultDdl);
}

Example 35 with BusinessObjectDataDdlRequest

use of org.finra.herd.model.api.xml.BusinessObjectDataDdlRequest in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlNoCustomDdlNoSchemaExists.

@Test
public void testGenerateBusinessObjectDataDdlNoCustomDdlNoSchemaExists() {
    // Prepare test data without schema.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(FileTypeEntity.TXT_FILE_TYPE, PARTITION_KEY, null, BusinessObjectDataEntity.FIRST_PARTITION_COLUMN_POSITION, UNSORTED_PARTITION_VALUES, NO_SUBPARTITION_VALUES, null, null, null, null, null, false, null, true, ALLOW_DUPLICATE_BUSINESS_OBJECT_DATA);
    // Try to retrieve business object data ddl when the business object format has no schema.
    // Retrieve business object data ddl without specifying custom ddl name.
    BusinessObjectDataDdlRequest request = businessObjectDataServiceTestHelper.getTestBusinessObjectDataDdlRequest(UNSORTED_PARTITION_VALUES);
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(request);
        fail("Should throw an IllegalArgumentException when business object format has no schema.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Business object format with namespace \"%s\", business object definition name \"%s\", format usage \"%s\", " + "format file type \"%s\", and format version \"%s\" doesn't have schema information.", request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}