Examples with PartitionValueFilter org.finra.herd.model.api.xml.PartitionValueFilter used on opensource projects

Example 1 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataHelper method validatePartitionValueFilters.

/**
 * Validates a list of partition value filters or a standalone partition filter. This method makes sure that a partition value filter contains exactly one
 * partition value range or a non-empty partition value list. This method also makes sure that there is no more than one partition value range specified
 * across all partition value filters.
 *
 * @param partitionValueFilters the list of partition value filters to validate
 * @param standalonePartitionValueFilter the standalone partition value filter to validate
 * @param allowPartitionValueTokens specifies whether the partition value filter is allowed to contain partition value tokens
 */
public void validatePartitionValueFilters(List<PartitionValueFilter> partitionValueFilters, PartitionValueFilter standalonePartitionValueFilter, boolean allowPartitionValueTokens) {
    // Make sure that request does not contain both a list of partition value filters and a standalone partition value filter.
    Assert.isTrue(partitionValueFilters == null || standalonePartitionValueFilter == null, "A list of partition value filters and a standalone partition value filter cannot be both specified.");
    List<PartitionValueFilter> partitionValueFiltersToValidate = new ArrayList<>();
    if (partitionValueFilters != null) {
        partitionValueFiltersToValidate.addAll(partitionValueFilters);
    }
    if (standalonePartitionValueFilter != null) {
        partitionValueFiltersToValidate.add(standalonePartitionValueFilter);
    }
    // Make sure that at least one partition value filter is specified.
    Assert.notEmpty(partitionValueFiltersToValidate, "At least one partition value filter must be specified.");
    // Validate and trim partition value filters.
    int partitionValueRangesCount = 0;
    for (PartitionValueFilter partitionValueFilter : partitionValueFiltersToValidate) {
        // Partition key is required when request contains a partition value filter list.
        if (partitionValueFilters != null) {
            Assert.hasText(partitionValueFilter.getPartitionKey(), "A partition key must be specified.");
        }
        // Trim partition key value.
        if (StringUtils.isNotBlank(partitionValueFilter.getPartitionKey())) {
            partitionValueFilter.setPartitionKey(partitionValueFilter.getPartitionKey().trim());
        }
        PartitionValueRange partitionValueRange = partitionValueFilter.getPartitionValueRange();
        List<String> partitionValues = partitionValueFilter.getPartitionValues();
        LatestBeforePartitionValue latestBeforePartitionValue = partitionValueFilter.getLatestBeforePartitionValue();
        LatestAfterPartitionValue latestAfterPartitionValue = partitionValueFilter.getLatestAfterPartitionValue();
        // Validate that we have exactly one partition filter option specified.
        List<Boolean> partitionFilterOptions = Arrays.asList(partitionValueRange != null, partitionValues != null, latestBeforePartitionValue != null, latestAfterPartitionValue != null);
        Assert.isTrue(Collections.frequency(partitionFilterOptions, Boolean.TRUE) == 1, "Exactly one partition value filter option must be specified.");
        if (partitionValueRange != null) {
            // A "partition value range" filter option is specified.
            // Only one partition value range is allowed across all partition value filters.
            partitionValueRangesCount++;
            Assert.isTrue(partitionValueRangesCount < 2, "Cannot specify more than one partition value range.");
            // Validate start partition value for the partition value range.
            Assert.hasText(partitionValueRange.getStartPartitionValue(), "A start partition value for the partition value range must be specified.");
            partitionValueRange.setStartPartitionValue(partitionValueRange.getStartPartitionValue().trim());
            // Validate end partition value for the partition value range.
            Assert.hasText(partitionValueRange.getEndPartitionValue(), "An end partition value for the partition value range must be specified.");
            partitionValueRange.setEndPartitionValue(partitionValueRange.getEndPartitionValue().trim());
            // Validate that partition value tokens are not specified as start and end partition values.
            // This check is required, regardless if partition value tokens are allowed or not.
            Assert.isTrue(!partitionValueRange.getStartPartitionValue().equals(BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN) && !partitionValueRange.getStartPartitionValue().equals(BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN) && !partitionValueRange.getEndPartitionValue().equals(BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN) && !partitionValueRange.getEndPartitionValue().equals(BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN), "A partition value token cannot be specified with a partition value range.");
            // Using string compare, validate that start partition value is less than or equal to end partition value.
            Assert.isTrue(partitionValueRange.getStartPartitionValue().compareTo(partitionValueRange.getEndPartitionValue()) <= 0, String.format("The start partition value \"%s\" cannot be greater than the end partition value \"%s\".", partitionValueRange.getStartPartitionValue(), partitionValueRange.getEndPartitionValue()));
        } else if (partitionValues != null) {
            // A "partition value list" filter option is specified.
            // Validate that the list contains at least one partition value.
            Assert.isTrue(!partitionValues.isEmpty(), "At least one partition value must be specified.");
            for (int i = 0; i < partitionValues.size(); i++) {
                String partitionValue = partitionValues.get(i);
                Assert.hasText(partitionValue, "A partition value must be specified.");
                partitionValue = partitionValue.trim();
                // When partition value tokens are not allowed, validate that they are not specified as one of partition values.
                if (!allowPartitionValueTokens) {
                    Assert.isTrue(!partitionValue.equals(BusinessObjectDataService.MAX_PARTITION_VALUE_TOKEN) && !partitionValue.equals(BusinessObjectDataService.MIN_PARTITION_VALUE_TOKEN), "A partition value token cannot be specified as one of partition values.");
                }
                partitionValues.set(i, partitionValue);
            }
        } else if (latestBeforePartitionValue != null) {
            // A "latest before partition value" filter option is specified.
            Assert.hasText(latestBeforePartitionValue.getPartitionValue(), "A partition value must be specified.");
            latestBeforePartitionValue.setPartitionValue(latestBeforePartitionValue.getPartitionValue().trim());
        } else {
            // A "latest after partition value" filter option is specified.
            Assert.hasText(latestAfterPartitionValue.getPartitionValue(), "A partition value must be specified.");
            latestAfterPartitionValue.setPartitionValue(latestAfterPartitionValue.getPartitionValue().trim());
        }
    }
}

Example 2 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataSearchHelper method validateBusinessObjectDataSearchKey.

/**
 * Validates a business object data search key.
 *
 * @param businessObjectDataSearchKey the business object data search key
 */
void validateBusinessObjectDataSearchKey(BusinessObjectDataSearchKey businessObjectDataSearchKey) {
    Assert.notNull(businessObjectDataSearchKey, "A business object data search key must be specified.");
    businessObjectDataSearchKey.setNamespace(alternateKeyHelper.validateStringParameter("namespace", businessObjectDataSearchKey.getNamespace()));
    businessObjectDataSearchKey.setBusinessObjectDefinitionName(alternateKeyHelper.validateStringParameter("business object definition name", businessObjectDataSearchKey.getBusinessObjectDefinitionName()));
    if (businessObjectDataSearchKey.getBusinessObjectFormatUsage() != null) {
        businessObjectDataSearchKey.setBusinessObjectFormatUsage(alternateKeyHelper.validateStringParameter("business object format usage", businessObjectDataSearchKey.getBusinessObjectFormatUsage()));
    }
    if (businessObjectDataSearchKey.getBusinessObjectFormatFileType() != null) {
        businessObjectDataSearchKey.setBusinessObjectFormatFileType(alternateKeyHelper.validateStringParameter("business object format file type", businessObjectDataSearchKey.getBusinessObjectFormatFileType()));
    }
    // Validate partition value filters, if specified.
    if (CollectionUtils.isNotEmpty(businessObjectDataSearchKey.getPartitionValueFilters())) {
        businessObjectDataHelper.validatePartitionValueFilters(businessObjectDataSearchKey.getPartitionValueFilters(), null, false);
        // TODO: For now, we only support partition values or partition range in the filter.
        for (PartitionValueFilter partitionValueFilter : businessObjectDataSearchKey.getPartitionValueFilters()) {
            List<String> partitionValues = partitionValueFilter.getPartitionValues();
            PartitionValueRange partitionValueRange = partitionValueFilter.getPartitionValueRange();
            // as it is done above at businessObjectDataHelper.validatePartitionValueFilters().
            if (CollectionUtils.isEmpty(partitionValues) && partitionValueRange == null) {
                throw new IllegalArgumentException("Only partition values or partition range are supported in partition value filter.");
            }
        }
    }
    // Validate attribute value filters, if specified.
    if (CollectionUtils.isNotEmpty(businessObjectDataSearchKey.getAttributeValueFilters())) {
        for (AttributeValueFilter attributeValueFilter : businessObjectDataSearchKey.getAttributeValueFilters()) {
            if (attributeValueFilter.getAttributeName() != null) {
                attributeValueFilter.setAttributeName(attributeValueFilter.getAttributeName().trim());
            }
            if (StringUtils.isBlank(attributeValueFilter.getAttributeName()) && StringUtils.isEmpty(attributeValueFilter.getAttributeValue())) {
                throw new IllegalArgumentException("Either attribute name or attribute value filter must be specified.");
            }
        }
    }
}

Example 3 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class CheckBusinessObjectDataAvailability method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    // Create the request.
    BusinessObjectDataAvailabilityRequest request = new BusinessObjectDataAvailabilityRequest();
    request.setNamespace(activitiHelper.getExpressionVariableAsString(namespace, execution));
    request.setBusinessObjectDefinitionName(activitiHelper.getExpressionVariableAsString(businessObjectDefinitionName, execution));
    request.setBusinessObjectFormatUsage(activitiHelper.getExpressionVariableAsString(businessObjectFormatUsage, execution));
    request.setBusinessObjectFormatFileType(activitiHelper.getExpressionVariableAsString(businessObjectFormatFileType, execution));
    request.setBusinessObjectFormatVersion(activitiHelper.getExpressionVariableAsInteger(businessObjectFormatVersion, execution, "BusinessObjectFormatVersion", false));
    // Build the partition value filter.
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    request.setPartitionValueFilter(partitionValueFilter);
    // Set the partition key if present.
    String partitionKeyString = activitiHelper.getExpressionVariableAsString(partitionKey, execution);
    if (partitionKeyString != null) {
        partitionValueFilter.setPartitionKey(partitionKeyString);
    }
    // Set the partition values if present.
    String partitionValuesString = activitiHelper.getExpressionVariableAsString(partitionValues, execution);
    List<String> partitionValueList = daoHelper.splitStringWithDefaultDelimiterEscaped(partitionValuesString);
    if (!CollectionUtils.isEmpty(partitionValueList)) {
        partitionValueFilter.setPartitionValues(partitionValueList);
    }
    // Build the partition value range and set it on the filter if present.
    PartitionValueRange partitionValueRange = new PartitionValueRange();
    partitionValueRange.setStartPartitionValue(activitiHelper.getExpressionVariableAsString(startPartitionValue, execution));
    partitionValueRange.setEndPartitionValue(activitiHelper.getExpressionVariableAsString(endPartitionValue, execution));
    if (StringUtils.isNotBlank(partitionValueRange.getStartPartitionValue()) || StringUtils.isNotBlank(partitionValueRange.getEndPartitionValue())) {
        partitionValueFilter.setPartitionValueRange(partitionValueRange);
    }
    // Set the business object data version if present.
    request.setBusinessObjectDataVersion(activitiHelper.getExpressionVariableAsInteger(businessObjectDataVersion, execution, "BusinessObjectDataVersion", false));
    // Set the storage.
    request.setStorageName(activitiHelper.getExpressionVariableAsString(storageName, execution));
    // Call the availability service.
    BusinessObjectDataAvailability businessObjectDataAvailability = businessObjectDataService.checkBusinessObjectDataAvailability(request);
    // Set the JSON response as a workflow variable.
    setJsonResponseAsWorkflowVariable(businessObjectDataAvailability, execution);
    // Set a workflow variable for whether all the data is available or not for easy access.
    boolean allAvailable = businessObjectDataAvailability.getNotAvailableStatuses().size() == 0;
    setTaskWorkflowVariable(execution, VARIABLE_IS_ALL_DATA_AVAILABLE, allAvailable);
}

Example 4 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataDaoImpl method createPartitionValueFilters.

/**
 * Creates a predicate for partition value filters.
 *
 * @param businessDataSearchKey businessDataSearchKey
 * @param businessObjectDataEntity businessObjectDataEntity
 * @param businessObjectFormatEntity businessObjectFormatEntity
 * @param builder builder
 * @param predicatePram predicate parameter
 *
 * @return the predicate
 */
private Predicate createPartitionValueFilters(BusinessObjectDataSearchKey businessDataSearchKey, Root<BusinessObjectDataEntity> businessObjectDataEntity, Join<BusinessObjectDataEntity, BusinessObjectFormatEntity> businessObjectFormatEntity, CriteriaBuilder builder, Predicate predicatePram) {
    Predicate predicate = predicatePram;
    if (businessDataSearchKey.getPartitionValueFilters() != null && !businessDataSearchKey.getPartitionValueFilters().isEmpty()) {
        for (PartitionValueFilter partitionFilter : businessDataSearchKey.getPartitionValueFilters()) {
            Join<BusinessObjectFormatEntity, SchemaColumnEntity> schemaEntity = businessObjectFormatEntity.join(BusinessObjectFormatEntity_.schemaColumns);
            List<String> partitionValues = partitionFilter.getPartitionValues();
            predicate = builder.and(predicate, builder.equal(builder.upper(schemaEntity.get(SchemaColumnEntity_.name)), partitionFilter.getPartitionKey().toUpperCase()));
            predicate = builder.and(predicate, builder.isNotNull(schemaEntity.get(SchemaColumnEntity_.partitionLevel)));
            if (partitionValues != null && !partitionValues.isEmpty()) {
                predicate = builder.and(predicate, builder.or(builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 1), businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue).in(partitionValues)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 2), businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue2).in(partitionValues)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 3), businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue3).in(partitionValues)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 4), businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue4).in(partitionValues)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 5), businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue5).in(partitionValues))));
            } else if (partitionFilter.getPartitionValueRange() != null) {
                PartitionValueRange partitionRange = partitionFilter.getPartitionValueRange();
                String startPartitionValue = partitionRange.getStartPartitionValue();
                String endPartitionValue = partitionRange.getEndPartitionValue();
                predicate = builder.and(predicate, builder.or(builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 1), builder.greaterThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue), startPartitionValue), builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue), endPartitionValue)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 2), builder.greaterThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue2), startPartitionValue), builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue2), endPartitionValue)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 3), builder.greaterThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue3), startPartitionValue), builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue3), endPartitionValue)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 4), builder.greaterThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue4), startPartitionValue), builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue4), endPartitionValue)), builder.and(builder.equal(schemaEntity.get(SchemaColumnEntity_.partitionLevel), 5), builder.greaterThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue5), startPartitionValue), builder.lessThanOrEqualTo(businessObjectDataEntity.get(BusinessObjectDataEntity_.partitionValue5), endPartitionValue))));
            }
        }
    }
    return predicate;
}

Example 5 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceCheckBusinessObjectDataAvailabilityTest method testCheckBusinessObjectDataAvailabilityIncludeAllRegisteredSubPartitionsSecondSubPartitionValidNonAvailableStorageUnit.

@Test
public void testCheckBusinessObjectDataAvailabilityIncludeAllRegisteredSubPartitionsSecondSubPartitionValidNonAvailableStorageUnit() {
    // Create two VALID sub-partitions - the first with an "available" storage unit and the second with a "non-available" storage unit.
    storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_1), DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_2), DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLED, NO_STORAGE_DIRECTORY_PATH);
    // Check this business object data availability with "IncludeAllRegisteredSubPartitions" option enabled.
    BusinessObjectDataAvailability result = businessObjectDataService.checkBusinessObjectDataAvailability(new BusinessObjectDataAvailabilityRequest(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, INCLUDE_ALL_REGISTERED_SUBPARTITIONS));
    // Validate the response object.
    // Both sub-partitions should be listed - the first as VALID "available" and the second as NO_ENABLED_STORAGE_UNIT "non-available".
    assertEquals(new BusinessObjectDataAvailability(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, Arrays.asList(new BusinessObjectDataStatus(FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_1), DATA_VERSION, BusinessObjectDataStatusEntity.VALID)), Arrays.asList(new BusinessObjectDataStatus(FORMAT_VERSION, PARTITION_VALUE, Arrays.asList(SUB_PARTITION_VALUE_2), DATA_VERSION, StorageUnitStatusEntity.DISABLED))), result);
}