Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 6 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class RelationalTableRegistrationHelperServiceImpl method retrieveRelationalTableColumnsImpl.

/**
 * Retrieves a list of actual schema columns for the specified relational table. This method uses actual JDBC connection to retrieve a description of table
 * columns.
 *
 * @param relationalStorageAttributesDto the relational storage attributes DTO
 * @param relationalSchemaName the name of the relational database schema
 * @param relationalTableName the name of the relational table
 *
 * @return the list of schema columns for the specified relational table
 */
List<SchemaColumn> retrieveRelationalTableColumnsImpl(RelationalStorageAttributesDto relationalStorageAttributesDto, String relationalSchemaName, String relationalTableName) {
    // Get the JDBC password value.
    String password = getPassword(relationalStorageAttributesDto);
    // Create and initialize a driver manager data source (a simple implementation of the standard JDBC interface).
    // We only support PostgreSQL database type.
    DriverManagerDataSource driverManagerDataSource = new DriverManagerDataSource();
    driverManagerDataSource.setUrl(relationalStorageAttributesDto.getJdbcUrl());
    driverManagerDataSource.setUsername(relationalStorageAttributesDto.getJdbcUsername());
    driverManagerDataSource.setPassword(password);
    driverManagerDataSource.setDriverClassName(JdbcServiceImpl.DRIVER_POSTGRES);
    // Create an empty result list.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    // Connect to the database and retrieve the relational table columns.
    try (Connection connection = driverManagerDataSource.getConnection()) {
        DatabaseMetaData databaseMetaData = connection.getMetaData();
        // Check if the specified relational table exists in the database.
        try (ResultSet tables = databaseMetaData.getTables(null, relationalSchemaName, relationalTableName, null)) {
            Assert.isTrue(tables.next(), String.format("Relational table with \"%s\" name not found under \"%s\" schema at jdbc.url=\"%s\" for jdbc.username=\"%s\".", relationalTableName, relationalSchemaName, driverManagerDataSource.getUrl(), driverManagerDataSource.getUsername()));
        }
        // Retrieve the relational table columns.
        try (ResultSet columns = databaseMetaData.getColumns(null, null, relationalTableName, null)) {
            while (columns.next()) {
                SchemaColumn schemaColumn = new SchemaColumn();
                schemaColumn.setName(columns.getString("COLUMN_NAME"));
                schemaColumn.setType(columns.getString("TYPE_NAME"));
                schemaColumn.setSize(columns.getString("COLUMN_SIZE"));
                schemaColumn.setRequired(columns.getInt("NULLABLE") == 0);
                schemaColumn.setDefaultValue(columns.getString("COLUMN_DEF"));
                schemaColumns.add(schemaColumn);
            }
        }
    } catch (SQLException e) {
        throw new IllegalArgumentException(String.format("Failed to retrieve description of a relational table with \"%s\" name under \"%s\" schema " + "at jdbc.url=\"%s\" using jdbc.username=\"%s\". Reason: %s", relationalTableName, relationalSchemaName, driverManagerDataSource.getUrl(), driverManagerDataSource.getUsername(), e.getMessage()), e);
    }
    return schemaColumns;
}

Example 7 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceImpl method validateBusinessObjectFormatSchema.

/**
 * Validates the business object format schema. This method also trims some of the schema column attributes parameters.
 *
 * @param schema the business object format schema
 * @param partitionKey the business object format partition key
 *
 * @throws IllegalArgumentException if any validation errors were found.
 */
private void validateBusinessObjectFormatSchema(Schema schema, String partitionKey) {
    // Validate optional schema information.
    if (schema != null) {
        // Perform validation.
        Assert.notNull(schema.getNullValue(), "A schema null value can not be null.");
        // Remove leading and trailing spaces.
        // Note that we don't trim the row option characters since they could contain white space or non-ASCII characters which would get removed if
        // "trim"ed.
        schema.setPartitionKeyGroup(schema.getPartitionKeyGroup() == null ? null : schema.getPartitionKeyGroup().trim());
        Assert.isTrue(!CollectionUtils.isEmpty(schema.getColumns()), "A schema must have at least one column.");
        // Validates the schema columns. We are creating a map here since it is used across both data and partition columns.
        LinkedHashMap<String, SchemaColumn> schemaEqualityValidationMap = new LinkedHashMap<>();
        validateSchemaColumns(schema.getColumns(), schemaEqualityValidationMap);
        validateSchemaColumns(schema.getPartitions(), schemaEqualityValidationMap);
        // the partition key will go away and this check will be removed.
        if (!CollectionUtils.isEmpty(schema.getPartitions())) {
            SchemaColumn schemaColumn = schema.getPartitions().get(0);
            if (!partitionKey.equalsIgnoreCase(schemaColumn.getName())) {
                throw new IllegalArgumentException(String.format("Partition key \"%s\" does not match the first schema partition column name \"%s\".", partitionKey, schemaColumn.getName()));
            }
        }
    }
}

Example 8 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class SchemaColumnDaoTestHelper method getTestSchemaColumns.

/**
 * Returns a list of schema columns that use passed attributes and hard coded test values.
 *
 * @param columnNamePrefix the column name prefix to use for the test columns
 * @param offset the offset index to start generating columns with
 * @param numColumns the number of columns
 * @param randomSuffix the random suffix
 *
 * @return the list of test schema columns
 */
public List<SchemaColumn> getTestSchemaColumns(String columnNamePrefix, Integer offset, Integer numColumns, String randomSuffix) {
    // Build a list of schema columns.
    List<SchemaColumn> columns = new ArrayList<>();
    for (int i = 0; i < numColumns; i++) {
        SchemaColumn schemaColumn = new SchemaColumn();
        columns.add(schemaColumn);
        // Set a value for the required column name field.
        schemaColumn.setName(String.format("%s-%d%s", columnNamePrefix, i + offset, randomSuffix));
        // Set a value for the required column type field.
        schemaColumn.setType(String.format("Type-%d", i + offset));
        // Set a value for the optional column size field for every other column.
        schemaColumn.setSize(i % 2 == 0 ? null : String.format("Size-%d", i + offset));
        // Set a value for the optional column required flag for each two out of 3 columns with the flag value alternating between true and false.
        schemaColumn.setRequired(i % 3 == 0 ? null : i % 2 == 0);
        // Set a value for the optional default value field for every other column.
        schemaColumn.setDefaultValue(i % 2 == 0 ? null : String.format("Clmn-Dflt-Value-%d%s", i, randomSuffix));
        // Set a value for the optional column size field for every other column.
        schemaColumn.setDescription(i % 2 == 0 ? null : String.format("Clmn-Desc-%d%s", i, randomSuffix));
    }
    return columns;
}

Example 9 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testGenerateBusinessObjectFormatDdlNoCustomDdlEscapeBackslashInRowFormat.

@Test
public void testGenerateBusinessObjectFormatDdlNoCustomDdlEscapeBackslashInRowFormat() {
    // Prepare test data without custom ddl.
    List<SchemaColumn> partitionColumns = schemaColumnDaoTestHelper.getTestPartitionColumns();
    String partitionKey = partitionColumns.get(0).getName();
    businessObjectFormatServiceTestHelper.createDatabaseEntitiesForBusinessObjectFormatDdlTesting(FileTypeEntity.TXT_FILE_TYPE, partitionKey, BACKSLASH, BACKSLASH, BACKSLASH, schemaColumnDaoTestHelper.getTestSchemaColumns(), partitionColumns, null);
    // Retrieve business object format ddl.
    BusinessObjectFormatDdlRequest request = businessObjectFormatServiceTestHelper.getTestBusinessObjectFormatDdlRequest(null);
    BusinessObjectFormatDdl resultDdl = businessObjectFormatService.generateBusinessObjectFormatDdl(request);
    // Validate the results - please note that we do not escape single backslash in null value.
    String expectedRowFormat = "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\\\' ESCAPED BY '\\\\' NULL DEFINED AS '\\'";
    String expectedDdl = businessObjectFormatServiceTestHelper.getExpectedBusinessObjectFormatDdl(PARTITION_COLUMNS.length, FIRST_COLUMN_NAME, FIRST_COLUMN_DATA_TYPE, expectedRowFormat, Hive13DdlGenerator.TEXT_HIVE_FILE_FORMAT, FileTypeEntity.TXT_FILE_TYPE, true, true);
    businessObjectFormatServiceTestHelper.validateBusinessObjectFormatDdl(null, expectedDdl, resultDdl);
}

Example 10 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatServiceTest method testCreateBusinessObjectFormatInitialVersionExistsWithSchemaNonAdditiveSchemaChangesColumnSizeIncreasedOriginalColumnSizeNotPositiveInteger.

@Test
public void testCreateBusinessObjectFormatInitialVersionExistsWithSchemaNonAdditiveSchemaChangesColumnSizeIncreasedOriginalColumnSizeNotPositiveInteger() {
    // Create relative database entities.
    businessObjectFormatServiceTestHelper.createTestDatabaseEntitiesForBusinessObjectFormatTesting();
    int i = 0;
    for (String originalColumnSize : Arrays.asList("NOT_AN_INTEGER", NEGATIVE_COLUMN_SIZE, ZERO_COLUMN_SIZE)) {
        // Create an initial format schema with a regular column size having not a positive integer value.
        Schema initialSchema = new Schema(Arrays.asList(new SchemaColumn(COLUMN_NAME, COLUMN_DATA_TYPE_CHAR, originalColumnSize, NO_COLUMN_REQUIRED, NO_COLUMN_DEFAULT_VALUE, COLUMN_DESCRIPTION)), NO_PARTITION_COLUMNS, SCHEMA_NULL_VALUE_BACKSLASH_N, SCHEMA_DELIMITER_PIPE, SCHEMA_ESCAPE_CHARACTER_BACKSLASH, PARTITION_KEY_GROUP);
        // We need to specify a unique format usage for each data type being tested to avoid an already exists exception.
        String formatUsage = String.format("%s_%d", FORMAT_USAGE_CODE, i++);
        // Create an initial version of the business object format.
        businessObjectFormatService.createBusinessObjectFormat(new BusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, formatUsage, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, NO_ATTRIBUTES, NO_ATTRIBUTE_DEFINITIONS, initialSchema));
        // Create an updated schema having a regular column size set to a positive integer.
        Schema updatedSchema = (Schema) initialSchema.clone();
        updatedSchema.getColumns().get(0).setSize(COLUMN_SIZE);
        try {
            // Try to create a second version of the business object format with a new schema
            // having regular column size changed from a non-integer to a positive integer.
            businessObjectFormatService.createBusinessObjectFormat(new BusinessObjectFormatCreateRequest(NAMESPACE, BDEF_NAME, formatUsage, FORMAT_FILE_TYPE_CODE, PARTITION_KEY, FORMAT_DESCRIPTION, NO_ATTRIBUTES, NO_ATTRIBUTE_DEFINITIONS, updatedSchema));
            fail("Should throw an IllegalArgumentException when the new format version is not \"additive\" to the previous format version.");
        } catch (IllegalArgumentException e) {
            assertEquals("New format version schema is not \"additive\" to the previous format version schema. " + "Non-additive changes detected to the previously defined regular (non-partitioning) columns.", e.getMessage());
        }
    }
}