Examples with SchemaColumn org.finra.herd.model.api.xml.SchemaColumn used on opensource projects

Example 51 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class Hive13DdlGeneratorTest method testGetHivePartitionsMultiplePathsFound.

@Test
public void testGetHivePartitionsMultiplePathsFound() {
    // Create a test business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, BDATA_STATUS);
    List<SchemaColumn> autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2"));
    List<String> partitionPaths = Arrays.asList("/COLUMN1=111/COLUMN2=222", "/column1=111/COLUMN2=222");
    List<String> storageFilePaths = getStorageFilePaths(Arrays.asList(partitionPaths.get(0) + "/file.dat", partitionPaths.get(1) + "/file.dat"));
    try {
        hive13DdlGenerator.getHivePartitions(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity), autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME);
        fail("Should throw an IllegalArgumentException when multiple locations exist for the same Hive partition.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Found two different locations for the same Hive partition. " + "Storage: {%s}, business object data: {%s}, S3 key prefix: {%s}, path[1]: {%s}, path[2]: {%s}", STORAGE_NAME, businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), TEST_S3_KEY_PREFIX, partitionPaths.get(0), partitionPaths.get(1)), e.getMessage());
    }
}

Example 52 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class Hive13DdlGeneratorTest method testGetHivePartitionsPatternMismatch.

@Test
public void testGetHivePartitionsPatternMismatch() {
    // Create a test business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, DATA_VERSION, true, BDATA_STATUS);
    List<SchemaColumn> autoDiscoverableSubPartitionColumns = getPartitionColumns(Arrays.asList("Column1", "column2"));
    String pattern = hive13DdlGenerator.getHivePathPattern(autoDiscoverableSubPartitionColumns).pattern();
    List<String> badFilePaths = // extra folder
    Arrays.asList(// extra folder
    "/column1=a/column2=b/extra-folder/file.dat", // partition columns out of order
    "/column2=a/column1=b/file.dat", // missing partition sub-directory
    "/column1=a/file.dat", // missing partition value
    "/column1=a/column2=/file.dat", // missing partition value
    "/column1=a/column2/file.dat", // slash in a partition value
    "/column1=a/a/column2=2/file.dat", // missing trailing '/' character
    "/column1=a/column2=2");
    // Get business object data key.
    BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity);
    for (int i = 0; i < badFilePaths.size(); i++) {
        List<String> storageFilePaths = getStorageFilePaths(badFilePaths.subList(i, i + 1));
        try {
            hive13DdlGenerator.getHivePartitions(businessObjectDataKey, autoDiscoverableSubPartitionColumns, TEST_S3_KEY_PREFIX, storageFilePaths, businessObjectDataEntity, STORAGE_NAME);
            fail("Should throw an IllegalArgumentException when storage file does not match the expected Hive sub-directory pattern.");
        } catch (IllegalArgumentException e) {
            assertEquals(String.format("Registered storage file or directory does not match the expected Hive sub-directory pattern. " + "Storage: {%s}, file/directory: {%s}, business object data: {%s}, S3 key prefix: {%s}, pattern: {^%s$}", STORAGE_NAME, storageFilePaths.get(0), businessObjectDataHelper.businessObjectDataEntityAltKeyToString(businessObjectDataEntity), TEST_S3_KEY_PREFIX, pattern), e.getMessage());
        }
    }
}

Example 53 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectDataDaoTest method createBusinessObjectEntityForPartitionValueFilterTest.

private BusinessObjectDataEntity createBusinessObjectEntityForPartitionValueFilterTest() {
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumn.setName(PARTITION_KEY);
        schemaColumn.setType("STRING");
        schemaColumns.add(schemaColumn);
    }
    {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumn.setName(PARTITION_KEY + "1");
        schemaColumn.setType("STRING");
        schemaColumns.add(schemaColumn);
    }
    {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumn.setName(PARTITION_KEY + "2");
        schemaColumn.setType("STRING");
        schemaColumns.add(schemaColumn);
    }
    {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumn.setName(PARTITION_KEY + "3");
        schemaColumn.setType("STRING");
        schemaColumns.add(schemaColumn);
    }
    {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumn.setName(PARTITION_KEY + "4");
        schemaColumn.setType("STRING");
        schemaColumns.add(schemaColumn);
    }
    NamespaceEntity namespaceEntity = super.namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    DataProviderEntity dataProviderEntity = super.dataProviderDaoTestHelper.createDataProviderEntity(DATA_PROVIDER_NAME);
    BusinessObjectDefinitionEntity businessObjectDefinitionEntity = super.businessObjectDefinitionDaoTestHelper.createBusinessObjectDefinitionEntity(namespaceEntity, BDEF_NAME, dataProviderEntity, NO_BDEF_DESCRIPTION, NO_BDEF_DISPLAY_NAME, NO_ATTRIBUTES, NO_SAMPLE_DATA_FILES);
    FileTypeEntity fileTypeEntity = super.fileTypeDaoTestHelper.createFileTypeEntity(FORMAT_FILE_TYPE_CODE, FORMAT_DESCRIPTION);
    BusinessObjectFormatEntity businessObjectFormatEntity = super.businessObjectFormatDaoTestHelper.createBusinessObjectFormatEntity(businessObjectDefinitionEntity, FORMAT_USAGE_CODE, fileTypeEntity, FORMAT_VERSION, null, true, PARTITION_KEY, null, NO_ATTRIBUTES, null, null, null, schemaColumns, schemaColumns);
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = businessObjectDataStatusDaoTestHelper.createBusinessObjectDataStatusEntity(BDATA_STATUS, DESCRIPTION, BDATA_STATUS_PRE_REGISTRATION_FLAG_SET);
    return businessObjectDataDaoTestHelper.createBusinessObjectDataEntity(businessObjectFormatEntity, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION, true, businessObjectDataStatusEntity.getCode());
}

Example 54 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class BusinessObjectFormatDaoTestHelper method createBusinessObjectFormatEntity.

/**
 * Creates and persists a new business object format entity.
 *
 * @return the newly created business object format entity.
 */
public BusinessObjectFormatEntity createBusinessObjectFormatEntity(BusinessObjectDefinitionEntity businessObjectDefinitionEntity, String businessObjectFormatUsage, FileTypeEntity fileTypeEntity, Integer businessObjectFormatVersion, String businessObjectFormatDescription, Boolean businessObjectFormatLatestVersion, String businessObjectFormatPartitionKey, PartitionKeyGroupEntity partitionKeyGroupEntity, List<Attribute> attributes, String schemaDelimiterCharacter, String schemaEscapeCharacter, String schemaNullValue, List<SchemaColumn> schemaColumns, List<SchemaColumn> partitionColumns) {
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    businessObjectFormatEntity.setBusinessObjectDefinition(businessObjectDefinitionEntity);
    businessObjectFormatEntity.setDescription(businessObjectFormatDescription);
    businessObjectFormatEntity.setFileType(fileTypeEntity);
    businessObjectFormatEntity.setBusinessObjectFormatVersion(businessObjectFormatVersion);
    businessObjectFormatEntity.setLatestVersion(businessObjectFormatLatestVersion);
    businessObjectFormatEntity.setUsage(businessObjectFormatUsage);
    businessObjectFormatEntity.setPartitionKey(businessObjectFormatPartitionKey);
    businessObjectFormatEntity.setPartitionKeyGroup(partitionKeyGroupEntity);
    // Create the attributes if they are specified.
    if (!CollectionUtils.isEmpty(attributes)) {
        List<BusinessObjectFormatAttributeEntity> attributeEntities = new ArrayList<>();
        businessObjectFormatEntity.setAttributes(attributeEntities);
        for (Attribute attribute : attributes) {
            BusinessObjectFormatAttributeEntity attributeEntity = new BusinessObjectFormatAttributeEntity();
            attributeEntities.add(attributeEntity);
            attributeEntity.setBusinessObjectFormat(businessObjectFormatEntity);
            attributeEntity.setName(attribute.getName());
            attributeEntity.setValue(attribute.getValue());
        }
    }
    if (schemaColumns != null && !schemaColumns.isEmpty()) {
        businessObjectFormatEntity.setDelimiter(schemaDelimiterCharacter);
        businessObjectFormatEntity.setEscapeCharacter(schemaEscapeCharacter);
        businessObjectFormatEntity.setNullValue(schemaNullValue);
        List<SchemaColumnEntity> schemaColumnEntities = new ArrayList<>();
        businessObjectFormatEntity.setSchemaColumns(schemaColumnEntities);
        int columnPosition = 1;
        for (SchemaColumn schemaColumn : schemaColumns) {
            SchemaColumnEntity schemaColumnEntity = new SchemaColumnEntity();
            schemaColumnEntities.add(schemaColumnEntity);
            schemaColumnEntity.setBusinessObjectFormat(businessObjectFormatEntity);
            schemaColumnEntity.setPosition(columnPosition);
            schemaColumnEntity.setPartitionLevel(null);
            schemaColumnEntity.setName(schemaColumn.getName());
            schemaColumnEntity.setType(schemaColumn.getType());
            schemaColumnEntity.setSize(schemaColumn.getSize());
            schemaColumnEntity.setDescription(schemaColumn.getDescription());
            schemaColumnEntity.setRequired(schemaColumn.isRequired());
            schemaColumnEntity.setDefaultValue(schemaColumn.getDefaultValue());
            columnPosition++;
        }
        if (partitionColumns != null && !partitionColumns.isEmpty()) {
            int partitionLevel = 1;
            for (SchemaColumn schemaColumn : partitionColumns) {
                // Check if this partition column belongs to the list of regular schema columns.
                int schemaColumnIndex = schemaColumns.indexOf(schemaColumn);
                if (schemaColumnIndex >= 0) {
                    // Retrieve the relative column entity and set its partition level.
                    schemaColumnEntities.get(schemaColumnIndex).setPartitionLevel(partitionLevel);
                } else {
                    // Add this partition column as a new schema column entity.
                    SchemaColumnEntity schemaColumnEntity = new SchemaColumnEntity();
                    schemaColumnEntities.add(schemaColumnEntity);
                    schemaColumnEntity.setBusinessObjectFormat(businessObjectFormatEntity);
                    schemaColumnEntity.setPosition(null);
                    schemaColumnEntity.setPartitionLevel(partitionLevel);
                    schemaColumnEntity.setName(schemaColumn.getName());
                    schemaColumnEntity.setType(schemaColumn.getType());
                    schemaColumnEntity.setSize(schemaColumn.getSize());
                    schemaColumnEntity.setDescription(schemaColumn.getDescription());
                    schemaColumnEntity.setRequired(schemaColumn.isRequired());
                    schemaColumnEntity.setDefaultValue(schemaColumn.getDefaultValue());
                }
                partitionLevel++;
            }
        }
    }
    return businessObjectFormatDao.saveAndRefresh(businessObjectFormatEntity);
}

Example 55 with SchemaColumn

use of org.finra.herd.model.api.xml.SchemaColumn in project herd by FINRAOS.

the class SchemaColumnDaoTestHelper method getTestSchemaColumns.

/**
 * Returns a list of schema columns that use hard coded test values.
 *
 * @param columnNamePrefix the column name prefix
 * @param schemaColumnDataTypes the list of schema column data types
 *
 * @return the list of test schema column entities
 */
public List<SchemaColumn> getTestSchemaColumns(String columnNamePrefix, String[][] schemaColumnDataTypes) {
    // Build a list of schema columns.
    List<SchemaColumn> schemaColumns = new ArrayList<>();
    int index = 1;
    for (String[] schemaColumnDataType : schemaColumnDataTypes) {
        SchemaColumn schemaColumn = new SchemaColumn();
        schemaColumns.add(schemaColumn);
        String columnName = String.format("%s%03d", columnNamePrefix, index);
        schemaColumn.setName(columnName);
        schemaColumn.setType(schemaColumnDataType[0]);
        schemaColumn.setSize(schemaColumnDataType[1]);
        index++;
    }
    // Column comment is an optional field, so provide comment for the second column only.
    schemaColumns.get(1).setDescription(String.format("This is '%s' column. Here are \\'single\\' and \"double\" quotes along with a backslash \\.", schemaColumns.get(1).getName()));
    return schemaColumns;
}