Examples with BusinessObjectFormatKey org.finra.herd.model.api.xml.BusinessObjectFormatKey used on opensource projects

Example 1 with BusinessObjectFormatKey

use of org.finra.herd.model.api.xml.BusinessObjectFormatKey in project herd by FINRAOS.

the class BusinessObjectFormatHelper method getBusinessObjectFormatKey.

/**
 * Creates a business object format key from specified business object format entity.
 *
 * @param businessObjectFormatEntity the business object format entity
 *
 * @return the business object format key
 */
public BusinessObjectFormatKey getBusinessObjectFormatKey(BusinessObjectFormatEntity businessObjectFormatEntity) {
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey();
    businessObjectFormatKey.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
    businessObjectFormatKey.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
    businessObjectFormatKey.setBusinessObjectFormatUsage(businessObjectFormatEntity.getUsage());
    businessObjectFormatKey.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
    businessObjectFormatKey.setBusinessObjectFormatVersion(businessObjectFormatEntity.getBusinessObjectFormatVersion());
    return businessObjectFormatKey;
}

Example 2 with BusinessObjectFormatKey

use of org.finra.herd.model.api.xml.BusinessObjectFormatKey in project herd by FINRAOS.

the class BusinessObjectFormatHelper method createBusinessObjectFormatFromEntity.

/**
 * Creates the business object format from the persisted entity.
 *
 * @param businessObjectFormatEntity the newly persisted business object format entity.
 *
 * @param checkLatestVersion need to check latest version
 *
 * @return the business object format.
 */
public BusinessObjectFormat createBusinessObjectFormatFromEntity(BusinessObjectFormatEntity businessObjectFormatEntity, Boolean checkLatestVersion) {
    BusinessObjectFormat businessObjectFormat = new BusinessObjectFormat();
    businessObjectFormat.setId(businessObjectFormatEntity.getId());
    businessObjectFormat.setNamespace(businessObjectFormatEntity.getBusinessObjectDefinition().getNamespace().getCode());
    businessObjectFormat.setBusinessObjectDefinitionName(businessObjectFormatEntity.getBusinessObjectDefinition().getName());
    businessObjectFormat.setBusinessObjectFormatUsage(businessObjectFormatEntity.getUsage());
    businessObjectFormat.setBusinessObjectFormatFileType(businessObjectFormatEntity.getFileType().getCode());
    businessObjectFormat.setBusinessObjectFormatVersion(businessObjectFormatEntity.getBusinessObjectFormatVersion());
    businessObjectFormat.setLatestVersion(businessObjectFormatEntity.getLatestVersion());
    businessObjectFormat.setPartitionKey(businessObjectFormatEntity.getPartitionKey());
    businessObjectFormat.setDescription(businessObjectFormatEntity.getDescription());
    // Add in the attributes.
    List<Attribute> attributes = new ArrayList<>();
    businessObjectFormat.setAttributes(attributes);
    for (BusinessObjectFormatAttributeEntity attributeEntity : businessObjectFormatEntity.getAttributes()) {
        Attribute attribute = new Attribute();
        attributes.add(attribute);
        attribute.setName(attributeEntity.getName());
        attribute.setValue(attributeEntity.getValue());
    }
    // Add in the attribute definitions.
    List<AttributeDefinition> attributeDefinitions = new ArrayList<>();
    businessObjectFormat.setAttributeDefinitions(attributeDefinitions);
    for (BusinessObjectDataAttributeDefinitionEntity attributeDefinitionEntity : businessObjectFormatEntity.getAttributeDefinitions()) {
        AttributeDefinition attributeDefinition = new AttributeDefinition();
        attributeDefinitions.add(attributeDefinition);
        attributeDefinition.setName(attributeDefinitionEntity.getName());
        attributeDefinition.setPublish(attributeDefinitionEntity.getPublish());
    }
    // Only add schema information if this format has any schema columns defined.
    if (!businessObjectFormatEntity.getSchemaColumns().isEmpty()) {
        Schema schema = new Schema();
        businessObjectFormat.setSchema(schema);
        schema.setNullValue(businessObjectFormatEntity.getNullValue());
        schema.setDelimiter(businessObjectFormatEntity.getDelimiter());
        schema.setEscapeCharacter(businessObjectFormatEntity.getEscapeCharacter());
        schema.setPartitionKeyGroup(businessObjectFormatEntity.getPartitionKeyGroup() != null ? businessObjectFormatEntity.getPartitionKeyGroup().getPartitionKeyGroupName() : null);
        // Create two lists of schema column entities: one for the data columns and one for the partition columns.
        List<SchemaColumnEntity> dataSchemaColumns = new ArrayList<>();
        List<SchemaColumnEntity> partitionSchemaColumns = new ArrayList<>();
        for (SchemaColumnEntity schemaColumnEntity : businessObjectFormatEntity.getSchemaColumns()) {
            // We can determine which list (or both) a column entity belongs to depending on whether it has a position and/or partition level set.
            if (schemaColumnEntity.getPosition() != null) {
                dataSchemaColumns.add(schemaColumnEntity);
            }
            if (schemaColumnEntity.getPartitionLevel() != null) {
                partitionSchemaColumns.add(schemaColumnEntity);
            }
        }
        // Sort the data schema columns on the position.
        Collections.sort(dataSchemaColumns, new SchemaColumnPositionComparator());
        // Sort the partition schema columns on the partition level.
        Collections.sort(partitionSchemaColumns, new SchemaColumnPartitionLevelComparator());
        // Add in the data schema columns.
        List<SchemaColumn> schemaColumns = new ArrayList<>();
        schema.setColumns(schemaColumns);
        for (SchemaColumnEntity schemaColumnEntity : dataSchemaColumns) {
            schemaColumns.add(createSchemaColumn(schemaColumnEntity));
        }
        // columns which isn't valid from an XSD standpoint.
        if (partitionSchemaColumns.size() > 0) {
            schemaColumns = new ArrayList<>();
            schema.setPartitions(schemaColumns);
            for (SchemaColumnEntity schemaColumnEntity : partitionSchemaColumns) {
                schemaColumns.add(createSchemaColumn(schemaColumnEntity));
            }
        }
    }
    BusinessObjectFormatEntity latestVersionBusinessObjectFormatEntity = businessObjectFormatEntity;
    // use the latest version if it is not
    if (checkLatestVersion) {
        BusinessObjectFormatKey businessObjectFormatKey = getBusinessObjectFormatKey(businessObjectFormatEntity);
        businessObjectFormatKey.setBusinessObjectFormatVersion(null);
        latestVersionBusinessObjectFormatEntity = businessObjectFormatDao.getBusinessObjectFormatByAltKey(businessObjectFormatKey);
    }
    // add business object format parent
    List<BusinessObjectFormatKey> businessObjectFormatParents = new ArrayList();
    businessObjectFormat.setBusinessObjectFormatParents(businessObjectFormatParents);
    for (BusinessObjectFormatEntity businessObjectFormatEntityParent : latestVersionBusinessObjectFormatEntity.getBusinessObjectFormatParents()) {
        BusinessObjectFormatKey businessObjectFormatParent = getBusinessObjectFormatKey(businessObjectFormatEntityParent);
        businessObjectFormatParent.setBusinessObjectFormatVersion(null);
        businessObjectFormatParents.add(businessObjectFormatParent);
    }
    // add business object format children
    List<BusinessObjectFormatKey> businessObjectFormatChildren = new ArrayList();
    businessObjectFormat.setBusinessObjectFormatChildren(businessObjectFormatChildren);
    for (BusinessObjectFormatEntity businessObjectFormatEntityChild : latestVersionBusinessObjectFormatEntity.getBusinessObjectFormatChildren()) {
        BusinessObjectFormatKey businessObjectFormatChild = getBusinessObjectFormatKey(businessObjectFormatEntityChild);
        businessObjectFormatChild.setBusinessObjectFormatVersion(null);
        businessObjectFormatChildren.add(businessObjectFormatChild);
    }
    // add retention information
    businessObjectFormat.setRecordFlag(latestVersionBusinessObjectFormatEntity.isRecordFlag());
    businessObjectFormat.setRetentionPeriodInDays(latestVersionBusinessObjectFormatEntity.getRetentionPeriodInDays());
    if (latestVersionBusinessObjectFormatEntity.getRetentionType() != null) {
        businessObjectFormat.setRetentionType(latestVersionBusinessObjectFormatEntity.getRetentionType().getCode());
    }
    return businessObjectFormat;
}

Example 3 with BusinessObjectFormatKey

use of org.finra.herd.model.api.xml.BusinessObjectFormatKey in project herd by FINRAOS.

the class Hive13DdlGenerator method processPartitionFiltersForGenerateDdl.

/**
 * Processes partition filters for DDL generation as per generate DDL request.
 *
 * @param generateDdlRequest the generate DDL request
 * @param sb the string builder to be updated with the "alter table add partition" statements
 * @param replacements the hash map of string values to be used to substitute the custom DDL tokens with their actual values
 * @param businessObjectFormatEntity the business object format entity
 * @param businessObjectFormat the business object format
 * @param ifNotExistsOption specifies if generated DDL contains "if not exists" option
 */
private void processPartitionFiltersForGenerateDdl(GenerateDdlRequest generateDdlRequest, StringBuilder sb, HashMap<String, String> replacements, BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectFormat businessObjectFormat, String ifNotExistsOption) {
    // Get the business object format key from the entity.
    BusinessObjectFormatKey businessObjectFormatKey = businessObjectFormatHelper.getBusinessObjectFormatKey(generateDdlRequest.businessObjectFormatEntity);
    // Override the business object format version with the original (optional) value from the request.
    businessObjectFormatKey.setBusinessObjectFormatVersion(generateDdlRequest.businessObjectFormatVersion);
    // Retrieve a list of storage unit entities for the specified list of partition filters. The entities will be sorted by partition values and storages.
    // For a non-partitioned table, there should only exist a single business object data entity (with partitionValue equals to "none"). We do validate that
    // all specified storages are of "S3" storage platform type, so we specify S3 storage platform type in the herdDao call below, so we select storage
    // units only from all S3 storages, when the specified list of storages is empty. We also specify to select only "available" storage units.
    List<StorageUnitEntity> storageUnitEntities = storageUnitDao.getStorageUnitsByPartitionFiltersAndStorages(businessObjectFormatKey, generateDdlRequest.partitionFilters, generateDdlRequest.businessObjectDataVersion, BusinessObjectDataStatusEntity.VALID, generateDdlRequest.storageNames, StoragePlatformEntity.S3, null, true);
    // Exclude duplicate business object data per specified list of storage names.
    // If storage names are not specified, the method fails on business object data instances registered with multiple storages.
    storageUnitEntities = excludeDuplicateBusinessObjectData(storageUnitEntities, generateDdlRequest.storageNames, generateDdlRequest.storageEntities);
    // Build a list of matched partition filters. Please note that each request partition
    // filter might result in multiple available business object data entities.
    List<List<String>> matchedAvailablePartitionFilters = new ArrayList<>();
    List<List<String>> availablePartitions = new ArrayList<>();
    for (StorageUnitEntity storageUnitEntity : storageUnitEntities) {
        BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(storageUnitEntity.getBusinessObjectData());
        matchedAvailablePartitionFilters.add(businessObjectDataHelper.getPartitionFilter(businessObjectDataKey, generateDdlRequest.partitionFilters.get(0)));
        availablePartitions.add(businessObjectDataHelper.getPrimaryAndSubPartitionValues(businessObjectDataKey));
    }
    // If request specifies to include all registered sub-partitions, fail if any of "non-available" registered sub-partitions are found.
    if (generateDdlRequest.businessObjectDataVersion == null && BooleanUtils.isTrue(generateDdlRequest.includeAllRegisteredSubPartitions) && !CollectionUtils.isEmpty(matchedAvailablePartitionFilters)) {
        notAllowNonAvailableRegisteredSubPartitions(businessObjectFormatKey, matchedAvailablePartitionFilters, availablePartitions, generateDdlRequest.storageNames);
    }
    // Fail on any missing business object data unless the flag is set to allow missing business object data.
    if (!BooleanUtils.isTrue(generateDdlRequest.allowMissingData)) {
        // Get a list of unmatched partition filters.
        List<List<String>> unmatchedPartitionFilters = new ArrayList<>(generateDdlRequest.partitionFilters);
        unmatchedPartitionFilters.removeAll(matchedAvailablePartitionFilters);
        // Throw an exception if we have any unmatched partition filters.
        if (!unmatchedPartitionFilters.isEmpty()) {
            // Get the first unmatched partition filter and throw exception.
            List<String> unmatchedPartitionFilter = getFirstUnmatchedPartitionFilter(unmatchedPartitionFilters);
            throw new ObjectNotFoundException(String.format("Business object data {namespace: \"%s\", businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", " + "businessObjectFormatFileType: \"%s\", businessObjectFormatVersion: %d, partitionValue: \"%s\", " + "subpartitionValues: \"%s\", businessObjectDataVersion: %d} is not available in \"%s\" storage(s).", businessObjectFormatKey.getNamespace(), businessObjectFormatKey.getBusinessObjectDefinitionName(), businessObjectFormatKey.getBusinessObjectFormatUsage(), businessObjectFormatKey.getBusinessObjectFormatFileType(), businessObjectFormatKey.getBusinessObjectFormatVersion(), unmatchedPartitionFilter.get(0), StringUtils.join(unmatchedPartitionFilter.subList(1, unmatchedPartitionFilter.size()), ","), generateDdlRequest.businessObjectDataVersion, StringUtils.join(generateDdlRequest.storageNames, ",")));
        }
    }
    // the table is non-partitioned, and there is no business object data found.
    if (generateDdlRequest.customDdlEntity == null && !generateDdlRequest.isPartitioned && CollectionUtils.isEmpty(storageUnitEntities)) {
        // Add a LOCATION clause with a token.
        sb.append(String.format("LOCATION '%s';", NON_PARTITIONED_TABLE_LOCATION_CUSTOM_DDL_TOKEN));
    } else // The table is partitioned, custom DDL is specified, or there is at least one business object data instance found.
    {
        // If drop partitions flag is set and the table is partitioned, drop partitions specified by the partition filters.
        if (generateDdlRequest.isPartitioned && BooleanUtils.isTrue(generateDdlRequest.includeDropPartitions)) {
            // Add a drop partition statement for each partition filter entry.
            for (List<String> partitionFilter : generateDdlRequest.partitionFilters) {
                sb.append(String.format("ALTER TABLE `%s` DROP IF EXISTS PARTITION (", generateDdlRequest.tableName));
                // Specify all partition column values as per this partition filter.
                List<String> partitionKeyValuePairs = new ArrayList<>();
                for (int i = 0; i < partitionFilter.size(); i++) {
                    if (StringUtils.isNotBlank(partitionFilter.get(i))) {
                        // We cannot hit ArrayIndexOutOfBoundsException on getPartitions() since partitionFilter would
                        // not have a value set at an index that is greater or equal than the number of partitions in the schema.
                        String partitionColumnName = businessObjectFormat.getSchema().getPartitions().get(i).getName();
                        partitionKeyValuePairs.add(String.format("`%s`='%s'", partitionColumnName, partitionFilter.get(i)));
                    }
                }
                sb.append(StringUtils.join(partitionKeyValuePairs, ", "));
                sb.append(");\n");
            }
            sb.append('\n');
        }
        // Process storage unit entities.
        if (!CollectionUtils.isEmpty(storageUnitEntities)) {
            processStorageUnitsForGenerateDdl(generateDdlRequest, sb, replacements, businessObjectFormatEntity, businessObjectFormat, ifNotExistsOption, storageUnitEntities);
        }
    }
}

Example 4 with BusinessObjectFormatKey

use of org.finra.herd.model.api.xml.BusinessObjectFormatKey in project herd by FINRAOS.

the class Hive13DdlGenerator method generateCreateTableDdl.

/**
 * Generates the create table Hive 13 DDL as per specified business object data DDL request.
 *
 * @param request the business object data DDL request
 * @param businessObjectFormatEntity the business object format entity
 * @param customDdlEntity the optional custom DDL entity
 * @param storageNames the list of storage names
 * @param storageEntities the list of storage entities
 * @param s3BucketNames the map of storage entities to the relative S3 bucket names
 *
 * @return the create table Hive DDL
 */
@Override
public String generateCreateTableDdl(BusinessObjectDataDdlRequest request, BusinessObjectFormatEntity businessObjectFormatEntity, CustomDdlEntity customDdlEntity, List<String> storageNames, List<StorageEntity> storageEntities, Map<StorageEntity, String> s3BucketNames) {
    // Get business object format key from the request.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(request.getNamespace(), request.getBusinessObjectDefinitionName(), request.getBusinessObjectFormatUsage(), request.getBusinessObjectFormatFileType(), request.getBusinessObjectFormatVersion());
    // Build partition filters based on the specified partition value filters.
    // We do validate that all specified storages are of "S3" storage platform type, so we specify S3 storage platform type in
    // the call below, so we select storage units only from all S3 storages, when the specified list of storages is empty.
    List<List<String>> partitionFilters = businessObjectDataDaoHelper.buildPartitionFilters(request.getPartitionValueFilters(), request.getPartitionValueFilter(), businessObjectFormatKey, request.getBusinessObjectDataVersion(), storageNames, StoragePlatformEntity.S3, null, businessObjectFormatEntity);
    // If the partitionKey="partition" and partitionValue="none", then DDL should
    // return a DDL which treats business object data as a table, not a partition.
    boolean isPartitioned = !businessObjectFormatEntity.getPartitionKey().equalsIgnoreCase(NO_PARTITIONING_PARTITION_KEY) || partitionFilters.size() != 1 || !partitionFilters.get(0).get(0).equalsIgnoreCase(NO_PARTITIONING_PARTITION_VALUE);
    // Generate the create table Hive 13 DDL.
    GenerateDdlRequest generateDdlRequest = new GenerateDdlRequest();
    generateDdlRequest.allowMissingData = request.isAllowMissingData();
    generateDdlRequest.businessObjectDataVersion = request.getBusinessObjectDataVersion();
    generateDdlRequest.businessObjectFormatEntity = businessObjectFormatEntity;
    generateDdlRequest.businessObjectFormatVersion = request.getBusinessObjectFormatVersion();
    generateDdlRequest.customDdlEntity = customDdlEntity;
    generateDdlRequest.includeAllRegisteredSubPartitions = request.isIncludeAllRegisteredSubPartitions();
    generateDdlRequest.includeDropPartitions = request.isIncludeDropPartitions();
    generateDdlRequest.includeDropTableStatement = request.isIncludeDropTableStatement();
    generateDdlRequest.includeIfNotExistsOption = request.isIncludeIfNotExistsOption();
    generateDdlRequest.isPartitioned = isPartitioned;
    generateDdlRequest.partitionFilters = partitionFilters;
    generateDdlRequest.s3BucketNames = s3BucketNames;
    generateDdlRequest.storageEntities = storageEntities;
    generateDdlRequest.storageNames = storageNames;
    generateDdlRequest.suppressScanForUnregisteredSubPartitions = request.isSuppressScanForUnregisteredSubPartitions();
    generateDdlRequest.tableName = request.getTableName();
    return generateCreateTableDdlHelper(generateDdlRequest);
}

Example 5 with BusinessObjectFormatKey

use of org.finra.herd.model.api.xml.BusinessObjectFormatKey in project herd by FINRAOS.

the class BusinessObjectDataInitiateDestroyHelperServiceImplTest method testValidateBusinessObjectDataInvalidPrimaryPartitionValue.

@Test
public void testValidateBusinessObjectDataInvalidPrimaryPartitionValue() {
    // Create a version-less business object format key.
    BusinessObjectFormatKey businessObjectFormatKey = new BusinessObjectFormatKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, NO_FORMAT_VERSION);
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a retention type entity.
    RetentionTypeEntity retentionTypeEntity = new RetentionTypeEntity();
    retentionTypeEntity.setCode(RetentionTypeEntity.PARTITION_VALUE);
    // Create a business object format entity.
    BusinessObjectFormatEntity businessObjectFormatEntity = new BusinessObjectFormatEntity();
    businessObjectFormatEntity.setLatestVersion(true);
    businessObjectFormatEntity.setRetentionType(retentionTypeEntity);
    businessObjectFormatEntity.setRetentionPeriodInDays(RETENTION_PERIOD_DAYS);
    // Create a business object data entity.
    BusinessObjectDataEntity businessObjectDataEntity = new BusinessObjectDataEntity();
    businessObjectDataEntity.setBusinessObjectFormat(businessObjectFormatEntity);
    businessObjectDataEntity.setPartitionValue(PARTITION_VALUE);
    // Mock the external calls.
    when(businessObjectDataHelper.getDateFromString(PARTITION_VALUE)).thenReturn(null);
    when(businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING);
    // Try to call the method under test.
    try {
        businessObjectDataInitiateDestroyHelperServiceImpl.validateBusinessObjectData(businessObjectDataEntity, businessObjectDataKey);
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Primary partition value \"%s\" cannot get converted to a valid date. Business object data: {%s}", PARTITION_VALUE, BUSINESS_OBJECT_DATA_KEY_AS_STRING), e.getMessage());
    }
    // Verify the external calls.
    verify(businessObjectDataHelper).getDateFromString(PARTITION_VALUE);
    verify(businessObjectDataHelper).businessObjectDataKeyToString(businessObjectDataKey);
    verifyNoMoreInteractionsHelper();
}