Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 76 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixKeyPrefixVelocityTemplateProducesBlankS3KeyPrefix.

@Test
public void testGetS3KeyPrefixKeyPrefixVelocityTemplateProducesBlankS3KeyPrefix() {
    // Create database entities required for testing.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(true);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    final String testS3KeyPrefixVelocityTemplate = "#if($CollectionUtils.isNotEmpty($businessObjectDataSubPartitions.keySet()))#end";
    // Create an S3 storage entity with the S3 key prefix template that would result in a blank S3 key prefix value.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), testS3KeyPrefixVelocityTemplate)));
    // Try to get an S3 key prefix specifying an S3 storage with the S3 key prefix velocity template that results in a blank S3 key prefix value.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME_2, false);
        fail("Should throw an IllegalArgumentException when specifying an S3 storage with the S3 key prefix velocity template " + "that results in a blank S3 key prefix value.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("S3 key prefix velocity template \"%s\" configured for \"%s\" storage results in an empty S3 key prefix.", testS3KeyPrefixVelocityTemplate, STORAGE_NAME_2), e.getMessage());
    }
}

Example 77 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixMissingKeyPrefixVelocityTemplate.

@Test
public void testGetS3KeyPrefixMissingKeyPrefixVelocityTemplate() {
    // Create database entities required for testing.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(true);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    // Create an S3 storage entity without the S3 key prefix template configured.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME_2, StoragePlatformEntity.S3);
    // Try to get an S3 key prefix specifying an S3 storage without the S3 key prefix velocity template configured.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME_2, false);
        fail("Should throw an IllegalArgumentException when specifying an S3 storage without the S3 key prefix velocity template configured.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage \"%s\" has no S3 key prefix velocity template configured.", STORAGE_NAME_2), e.getMessage());
    }
    // Create an S3 storage entity with the S3 key prefix template configured to a blank string.
    storageDaoTestHelper.createStorageEntity(STORAGE_NAME_3, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), BLANK_TEXT)));
    // Try to get an S3 key prefix specifying an S3 storage with the S3 key prefix velocity template being a blank string.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME_3, false);
        fail("Should throw an IllegalArgumentException when specifying an S3 storage with the S3 key prefix velocity template being a blank string.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Storage \"%s\" has no S3 key prefix velocity template configured.", STORAGE_NAME_3), e.getMessage());
    }
}

Example 78 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class StorageUnitServiceGetS3KeyPrefixTest method testGetS3KeyPrefixInvalidParameters.

@Test
public void testGetS3KeyPrefixInvalidParameters() {
    // Create database entities required for testing. Please note that we are not passing the flag to create a business object data entity.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForGetS3KeyPrefixTesting(false);
    // Get the test partition columns.
    List<SchemaColumn> testPartitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String testPartitionKey = testPartitionColumns.get(0).getName();
    // Try to get an S3 key prefix using invalid namespace.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey("I_DO_NOT_EXIST", BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an ObjectNotFoundException when not able to find business object format.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage("I_DO_NOT_EXIST", BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid business object definition name.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, "I_DO_NOT_EXIST", FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an ObjectNotFoundException when not able to find business object format.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage(NAMESPACE, "I_DO_NOT_EXIST", FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid format usage.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, "I_DO_NOT_EXIST", FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an ObjectNotFoundException when not able to find business object format.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage(NAMESPACE, BDEF_NAME, "I_DO_NOT_EXIST", FORMAT_FILE_TYPE_CODE, FORMAT_VERSION), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid format file type.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, "I_DO_NOT_EXIST", FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an ObjectNotFoundException when not able to find business object format.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, "I_DO_NOT_EXIST", FORMAT_VERSION), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid business object format version.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INVALID_FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an ObjectNotFoundException when not able to find business object format.");
    } catch (ObjectNotFoundException e) {
        assertEquals(businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatNotFoundErrorMessage(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INVALID_FORMAT_VERSION), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid partition key.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), "I_DO_NOT_EXIST", STORAGE_NAME, false);
        fail("Should throw an IllegalArgumentException when using an invalid partition key.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Partition key \"%s\" doesn't match configured business object format partition key \"%s\".", "I_DO_NOT_EXIST", testPartitionKey), e.getMessage());
    }
    // Try to get an S3 key prefix using invalid storage name.
    try {
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION), testPartitionKey, "I_DO_NOT_EXIST", false);
        fail("Should throw an ObjectNotFoundException when using an invalid storage name.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Storage with name \"%s\" doesn't exist.", "I_DO_NOT_EXIST"), e.getMessage());
    }
    // Try to get an S3 key prefix using too many subpartition values.
    try {
        List<String> testSubPartitionValues = new ArrayList<>(SUBPARTITION_VALUES);
        testSubPartitionValues.add("EXTRA_SUBPARTITION_VALUE");
        storageUnitService.getS3KeyPrefix(new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, testSubPartitionValues, DATA_VERSION), testPartitionKey, STORAGE_NAME, false);
        fail("Should throw an IllegalArgumentException when passing too many subpartition values.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Exceeded maximum number of allowed subpartitions: %d.", BusinessObjectDataEntity.MAX_SUBPARTITIONS), e.getMessage());
    }
}

Example 79 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlForwardSlashInPartitionColumnName.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlForwardSlashInPartitionColumnName() {
    // Prepare test data without custom ddl.
    String invalidPartitionColumnName = "INVALID_/_PRTN_CLMN";
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    partitionColumns.get(0).setName(invalidPartitionColumnName);
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, PARTITION_KEY, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Try to retrieve business object format ddl for the format that uses unsupported schema column data type.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    try {
        businessObjectFormatService.generateBusinessObjectFormatDdl(request);
        fail("Should throw an IllegalArgumentException when partition column name contains a '/' character.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Partition column name \"%s\" can not contain a '/' character. Business object format: " + "{namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d}", invalidPartitionColumnName, request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion()), e.getMessage());
    }
}

Example 80 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testCreateBusinessObjectFormatNoSchemaColumnName.

@Test
public void testCreateBusinessObjectFormatNoSchemaColumnName() {
    // Create a business object format create request.
    BusinessObjectFormatCreateRequest request = businessObjectFormatServiceTestHelper.createBusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, businessObjectDefinitionServiceTestHelper.getNewAttributes(), businessObjectFormatServiceTestHelper.getTestAttributeDefinitions(), businessObjectFormatServiceTestHelper.getTestSchema());
    // Try to create a business object format with an empty column name.
    SchemaColumn schemaColumn = new SchemaColumn();
    request.getSchema().getColumns().add(schemaColumn);
    schemaColumn.setName(BLANK_TEXT);
    schemaColumn.setType("TYPE");
    try {
        businessObjectFormatService.createBusinessObjectFormat(request);
        fail("Should throw an IllegalArgumentException when schema has an empty column name.");
    } catch (IllegalArgumentException e) {
        assertEquals("A schema column name must be specified.", e.getMessage());
    }
}