Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 61 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getExpectedBusinessObjectDataDdlTwoPartitionLevels.

/**
 * Returns the actual HIVE DDL expected to be generated.
 *
 * @param partitions the list of partitions, where each is represented by a primary value and a sub-partition value
 *
 * @return the actual HIVE DDL expected to be generated
 */
public String getExpectedBusinessObjectDataDdlTwoPartitionLevels(List<List<String>> partitions) {
    // Build ddl expected to be generated.
    StringBuilder ddlBuilder = new StringBuilder();
    ddlBuilder.append("DROP TABLE IF EXISTS `" + AbstractServiceTest.TABLE_NAME + "`;\n");
    ddlBuilder.append("\n");
    ddlBuilder.append("CREATE EXTERNAL TABLE IF NOT EXISTS `" + AbstractServiceTest.TABLE_NAME + "` (\n");
    ddlBuilder.append("    `ORGNL_" + AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME + "` DATE,\n");
    ddlBuilder.append("    `ORGNL_" + AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME + "` STRING,\n");
    ddlBuilder.append("    `" + AbstractServiceTest.COLUMN_NAME + "` DECIMAL(" + AbstractServiceTest.COLUMN_SIZE + ") COMMENT '" + AbstractServiceTest.COLUMN_DESCRIPTION + "')\n");
    ddlBuilder.append("PARTITIONED BY (`" + AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME + "` DATE, `" + AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME + "` STRING)\n");
    ddlBuilder.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' ESCAPED BY '\\\\' NULL DEFINED AS '\\N'\n");
    ddlBuilder.append("STORED AS TEXTFILE;");
    // Add the alter table drop partition statement.
    ddlBuilder.append("\n\n");
    ddlBuilder.append("ALTER TABLE `" + AbstractServiceTest.TABLE_NAME + "` DROP IF EXISTS PARTITION (`" + AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME + "`='" + partitions.get(0).get(0) + "');");
    ddlBuilder.append("\n");
    for (List<String> partition : partitions) {
        // Build an expected S3 key prefix.
        String expectedS3KeyPrefix = AbstractServiceTest.getExpectedS3KeyPrefix(AbstractServiceTest.NAMESPACE, AbstractServiceTest.DATA_PROVIDER_NAME, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, partition.get(0), Arrays.asList(new SchemaColumn(AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME, "STRING", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION)).toArray(new SchemaColumn[1]), Arrays.asList(partition.get(1)).toArray(new String[1]), AbstractServiceTest.DATA_VERSION);
        // Add the alter table add partition statement.
        ddlBuilder.append("\n");
        ddlBuilder.append("ALTER TABLE `" + AbstractServiceTest.TABLE_NAME + "` ADD IF NOT EXISTS PARTITION (`" + AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME + "`='" + partition.get(0) + "', `" + AbstractServiceTest.SECOND_PARTITION_COLUMN_NAME + "`='" + partition.get(1) + "') LOCATION 's3n://" + AbstractServiceTest.S3_BUCKET_NAME + "/" + expectedS3KeyPrefix + "';");
    }
    String expectedDdl = ddlBuilder.toString();
    return expectedDdl;
}

Example 62 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method createDatabaseEntitiesForBusinessObjectDataDdlTesting.

/**
 * Creates and persists database entities required for generate business object data ddl testing.
 */
public StorageUnitEntity createDatabaseEntitiesForBusinessObjectDataDdlTesting(String partitionValue, String s3KeyPrefix) {
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, "DATE", AbstractServiceTest.NO_COLUMN_SIZE, AbstractServiceTest.COLUMN_REQUIRED, AbstractServiceTest.NO_COLUMN_DEFAULT_VALUE, AbstractServiceTest.NO_COLUMN_DESCRIPTION));
    schemaColumns.add(new SchemaColumn(AbstractServiceTest.COLUMN_NAME, "NUMBER", AbstractServiceTest.COLUMN_SIZE, AbstractServiceTest.NO_COLUMN_REQUIRED, AbstractServiceTest.COLUMN_DEFAULT_VALUE, AbstractServiceTest.COLUMN_DESCRIPTION));
    // Use the first column as a partition column.
    List<SchemaColumn> partitionColumns = schemaColumns.subList(0, 1);
    // Create a business object format entity with the schema.
    BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.FORMAT_DESCRIPTION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, AbstractServiceTest.NO_PARTITION_KEY_GROUP, AbstractServiceTest.NO_ATTRIBUTES, AbstractServiceTest.SCHEMA_DELIMITER_PIPE, AbstractServiceTest.SCHEMA_ESCAPE_CHARACTER_BACKSLASH, AbstractServiceTest.SCHEMA_NULL_VALUE_BACKSLASH_N, schemaColumns, partitionColumns);
    if (partitionValue != null) {
        // Create a business object data entity.
        BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, partitionValue, AbstractServiceTest.NO_SUBPARTITION_VALUES, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID);
        // Create an S3 storage entity.
        StorageEntity storageEntity = storageDaoTestHelper.createStorageEntity(AbstractServiceTest.STORAGE_NAME, StoragePlatformEntity.S3, Arrays.asList(new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), AbstractServiceTest.S3_BUCKET_NAME), new Attribute(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KEY_PREFIX_VELOCITY_TEMPLATE), AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE)));
        // Create a storage unit with a storage directory path.
        return storageUnitDaoTestHelper.createStorageUnitEntity(storageEntity, businessObjectDataEntity, StorageUnitStatusEntity.ENABLED, s3KeyPrefix);
    }
    return null;
}

Example 63 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTestHelper method validateBusinessObjectFormat.

/**
 * Validates business object format contents against specified arguments and expected (hard coded) test values.
 *
 * @param expectedBusinessObjectFormatId the expected business object format ID
 * @param expectedNamespaceCode the expected namespace code
 * @param expectedBusinessObjectDefinitionName the expected business object definition name
 * @param expectedBusinessObjectFormatUsage the expected business object format usage
 * @param expectedBusinessObjectFormatFileType the expected business object format file type
 * @param expectedBusinessObjectFormatVersion the expected business object format version
 * @param expectedIsLatestVersion the expected business object format version
 * @param expectedPartitionKey the expected business object format partition key
 * @param expectedDescription the expected business object format description
 * @param expectedAttributes the expected attributes
 * @param expectedAttributeDefinitions the list of expected attribute definitions
 * @param expectedSchema the expected business object format schema
 * @param actualBusinessObjectFormat the BusinessObjectFormat object instance to be validated
 */
public void validateBusinessObjectFormat(Integer expectedBusinessObjectFormatId, String expectedNamespaceCode, String expectedBusinessObjectDefinitionName, String expectedBusinessObjectFormatUsage, String expectedBusinessObjectFormatFileType, Integer expectedBusinessObjectFormatVersion, Boolean expectedIsLatestVersion, String expectedPartitionKey, String expectedDescription, List<Attribute> expectedAttributes, List<AttributeDefinition> expectedAttributeDefinitions, Schema expectedSchema, BusinessObjectFormat actualBusinessObjectFormat) {
    assertNotNull(actualBusinessObjectFormat);
    if (expectedBusinessObjectFormatId != null) {
        assertEquals(expectedBusinessObjectFormatId, Integer.valueOf(actualBusinessObjectFormat.getId()));
    }
    assertEquals(expectedNamespaceCode, actualBusinessObjectFormat.getNamespace());
    assertEquals(expectedBusinessObjectDefinitionName, actualBusinessObjectFormat.getBusinessObjectDefinitionName());
    assertEquals(expectedBusinessObjectFormatUsage, actualBusinessObjectFormat.getBusinessObjectFormatUsage());
    assertEquals(expectedBusinessObjectFormatFileType, actualBusinessObjectFormat.getBusinessObjectFormatFileType());
    assertEquals(expectedBusinessObjectFormatVersion, Integer.valueOf(actualBusinessObjectFormat.getBusinessObjectFormatVersion()));
    assertEquals(expectedIsLatestVersion, actualBusinessObjectFormat.isLatestVersion());
    assertEquals(expectedPartitionKey, actualBusinessObjectFormat.getPartitionKey());
    AbstractServiceTest.assertEqualsIgnoreNullOrEmpty("description", expectedDescription, actualBusinessObjectFormat.getDescription());
    // Ignoring the order, check if the actual list of attributes matches the expected list.
    if (!CollectionUtils.isEmpty(expectedAttributes)) {
        assertEquals(expectedAttributes, actualBusinessObjectFormat.getAttributes());
    } else {
        assertEquals(0, actualBusinessObjectFormat.getAttributes().size());
    }
    // Ignoring the order, check if the actual list of attribute definitions matches the expected list.
    if (!CollectionUtils.isEmpty(expectedAttributeDefinitions)) {
        assertEquals(expectedAttributeDefinitions, actualBusinessObjectFormat.getAttributeDefinitions());
    } else {
        assertEquals(0, actualBusinessObjectFormat.getAttributeDefinitions().size());
    }
    // Validate the schema.
    if (expectedSchema != null) {
        assertNotNull(actualBusinessObjectFormat.getSchema());
        AbstractServiceTest.assertEqualsIgnoreNullOrEmpty("null value", expectedSchema.getNullValue(), actualBusinessObjectFormat.getSchema().getNullValue());
        AbstractServiceTest.assertEqualsIgnoreNullOrEmpty("delimiter", expectedSchema.getDelimiter(), actualBusinessObjectFormat.getSchema().getDelimiter());
        AbstractServiceTest.assertEqualsIgnoreNullOrEmpty("escape character", expectedSchema.getEscapeCharacter(), actualBusinessObjectFormat.getSchema().getEscapeCharacter());
        assertEquals(expectedSchema.getPartitionKeyGroup(), actualBusinessObjectFormat.getSchema().getPartitionKeyGroup());
        assertEquals(expectedSchema.getColumns().size(), actualBusinessObjectFormat.getSchema().getColumns().size());
        for (int i = 0; i < expectedSchema.getColumns().size(); i++) {
            SchemaColumn expectedSchemaColumn = expectedSchema.getColumns().get(i);
            SchemaColumn actualSchemaColumn = actualBusinessObjectFormat.getSchema().getColumns().get(i);
            assertEquals(expectedSchemaColumn.getName(), actualSchemaColumn.getName());
            assertEquals(expectedSchemaColumn.getType(), actualSchemaColumn.getType());
            assertEquals(expectedSchemaColumn.getSize(), actualSchemaColumn.getSize());
            assertEquals(expectedSchemaColumn.isRequired(), actualSchemaColumn.isRequired());
            assertEquals(expectedSchemaColumn.getDefaultValue(), actualSchemaColumn.getDefaultValue());
            assertEquals(expectedSchemaColumn.getDescription(), actualSchemaColumn.getDescription());
        }
        if (CollectionUtils.isEmpty(expectedSchema.getPartitions())) {
            assertTrue(CollectionUtils.isEmpty(actualBusinessObjectFormat.getSchema().getPartitions()));
        } else {
            for (int i = 0; i < expectedSchema.getPartitions().size(); i++) {
                SchemaColumn expectedPartitionColumn = expectedSchema.getPartitions().get(i);
                SchemaColumn actualPartitionColumn = actualBusinessObjectFormat.getSchema().getPartitions().get(i);
                assertEquals(expectedPartitionColumn.getName(), actualPartitionColumn.getName());
                assertEquals(expectedPartitionColumn.getType(), actualPartitionColumn.getType());
                assertEquals(expectedPartitionColumn.getSize(), actualPartitionColumn.getSize());
                assertEquals(expectedPartitionColumn.isRequired(), actualPartitionColumn.isRequired());
                assertEquals(expectedPartitionColumn.getDefaultValue(), actualPartitionColumn.getDefaultValue());
                assertEquals(expectedPartitionColumn.getDescription(), actualPartitionColumn.getDescription());
            }
        }
    } else {
        assertNull(actualBusinessObjectFormat.getSchema());
    }
}

Example 64 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class RelationalTableRegistrationServiceTestHelper method getExpectedSchemaColumns.

/**
 * Gets a list of expected schema columns.
 *
 * @return the list of schema columns
 */
List<SchemaColumn> getExpectedSchemaColumns() {
    List<SchemaColumn> expectedSchemaColumns = new ArrayList<>();
    expectedSchemaColumns.add(new SchemaColumn("BUS_OBJCT_DFNTN_ID", "INTEGER", "10", true, null, null));
    expectedSchemaColumns.add(new SchemaColumn("CREAT_USER_ID", "VARCHAR", "255", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("CREAT_TS", "TIMESTAMP", "23", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("UPDT_USER_ID", "VARCHAR", "255", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("UPDT_TS", "TIMESTAMP", "23", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("DESC_TX", "VARCHAR", "500", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("DSPLY_NAME_TX", "VARCHAR", "255", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("NAME_TX", "VARCHAR", "255", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("DATA_PRVDR_CD", "VARCHAR", "255", true, null, null));
    expectedSchemaColumns.add(new SchemaColumn("DESC_BUS_OBJCT_FRMT_ID", "INTEGER", "10", false, null, null));
    expectedSchemaColumns.add(new SchemaColumn("NAME_SPACE_CD", "VARCHAR", "255", true, null, null));
    return expectedSchemaColumns;
}

Example 65 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceImpl method validateSchemaColumns.

/**
 * Validates a list of schema columns.
 *
 * @param schemaColumns the list of schema columns.
 * @param schemaEqualityValidationMap a map of schema column names to their schema column. This is used to check equality across all data schema columns as
 * well as partition schema columns.
 */
private void validateSchemaColumns(List<SchemaColumn> schemaColumns, LinkedHashMap<String, SchemaColumn> schemaEqualityValidationMap) {
    // Validate schema columns if they are specified.
    if (!CollectionUtils.isEmpty(schemaColumns)) {
        // Create a schema column name map that we will use to check for duplicate
        // columns for the specified list of schema columns (i.e. data or partition).
        LinkedHashMap<String, SchemaColumn> schemaColumnNameValidationMap = new LinkedHashMap<>();
        // Loop through each schema column in the list.
        for (SchemaColumn schemaColumn : schemaColumns) {
            // Perform validation.
            Assert.hasText(schemaColumn.getName(), "A schema column name must be specified.");
            Assert.hasText(schemaColumn.getType(), "A schema column data type must be specified.");
            // Remove leading and trailing spaces.
            schemaColumn.setName(schemaColumn.getName().trim());
            schemaColumn.setType(schemaColumn.getType().trim());
            schemaColumn.setSize(schemaColumn.getSize() == null ? null : schemaColumn.getSize().trim());
            schemaColumn.setDefaultValue(schemaColumn.getDefaultValue() == null ? null : schemaColumn.getDefaultValue().trim());
            // Ensure the column name isn't a duplicate within this list only by using a map.
            String lowercaseSchemaColumnName = schemaColumn.getName().toLowerCase();
            if (schemaColumnNameValidationMap.containsKey(lowercaseSchemaColumnName)) {
                throw new IllegalArgumentException(String.format("Duplicate schema column name \"%s\" found.", schemaColumn.getName()));
            }
            schemaColumnNameValidationMap.put(lowercaseSchemaColumnName, schemaColumn);
            // Ensure a partition column and a data column are equal (i.e. contain the same configuration).
            SchemaColumn existingSchemaColumn = schemaEqualityValidationMap.get(lowercaseSchemaColumnName);
            if ((existingSchemaColumn != null) && !schemaColumn.equals(existingSchemaColumn)) {
                throw new IllegalArgumentException("Schema data and partition column configurations with name \"" + schemaColumn.getName() + "\" have conflicting values. All column values are case sensitive and must be identical.");
            }
            schemaEqualityValidationMap.put(lowercaseSchemaColumnName, schemaColumn);
        }
    }
}