Examples with PartitionValueFilter org.finra.herd.model.api.xml.PartitionValueFilter used on opensource projects

Example 51 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getExpectedBusinessObjectDataAvailabilityCollectionResponse.

/**
 * Creates an expected business object data availability collection response using hard coded test values.
 *
 * @return the business object data availability collection response
 */
public BusinessObjectDataAvailabilityCollectionResponse getExpectedBusinessObjectDataAvailabilityCollectionResponse() {
    // Prepare a check availability collection response using hard coded test values.
    BusinessObjectDataAvailabilityCollectionResponse businessObjectDataAvailabilityCollectionResponse = new BusinessObjectDataAvailabilityCollectionResponse();
    // Create a list of check business object data availability responses.
    List<BusinessObjectDataAvailability> businessObjectDataAvailabilityResponses = new ArrayList<>();
    businessObjectDataAvailabilityCollectionResponse.setBusinessObjectDataAvailabilityResponses(businessObjectDataAvailabilityResponses);
    // Create a business object data availability response.
    BusinessObjectDataAvailability businessObjectDataAvailability = new BusinessObjectDataAvailability(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, AbstractServiceTest.FORMAT_FILE_TYPE_CODE, AbstractServiceTest.FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(AbstractServiceTest.PARTITION_KEY, Arrays.asList(AbstractServiceTest.PARTITION_VALUE), AbstractServiceTest.NO_PARTITION_VALUE_RANGE, AbstractServiceTest.NO_LATEST_BEFORE_PARTITION_VALUE, AbstractServiceTest.NO_LATEST_AFTER_PARTITION_VALUE)), null, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.NO_STORAGE_NAMES, AbstractServiceTest.STORAGE_NAME, Arrays.asList(new BusinessObjectDataStatus(AbstractServiceTest.FORMAT_VERSION, AbstractServiceTest.PARTITION_VALUE, AbstractServiceTest.SUBPARTITION_VALUES, AbstractServiceTest.DATA_VERSION, BusinessObjectDataStatusEntity.VALID)), new ArrayList<>());
    businessObjectDataAvailabilityResponses.add(businessObjectDataAvailability);
    // Set the expected values for the flags.
    businessObjectDataAvailabilityCollectionResponse.setIsAllDataAvailable(true);
    businessObjectDataAvailabilityCollectionResponse.setIsAllDataNotAvailable(false);
    return businessObjectDataAvailabilityCollectionResponse;
}

Example 52 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getTestBusinessObjectDataDdlRequest.

/**
 * Creates and returns a business object data ddl request using passed parameters along with some hard-coded test values.
 *
 * @param startPartitionValue the start partition value for the partition value range
 * @param endPartitionValue the end partition value for the partition value range
 * @param partitionValues the list of partition values
 * @param customDdlName the custom DDL name
 *
 * @return the newly created business object data ddl request
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public BusinessObjectDataDdlRequest getTestBusinessObjectDataDdlRequest(String startPartitionValue, String endPartitionValue, List<String> partitionValues, String customDdlName) {
    BusinessObjectDataDdlRequest request = new BusinessObjectDataDdlRequest();
    request.setNamespace(AbstractServiceTest.NAMESPACE);
    request.setBusinessObjectDefinitionName(AbstractServiceTest.BDEF_NAME);
    request.setBusinessObjectFormatUsage(AbstractServiceTest.FORMAT_USAGE_CODE);
    request.setBusinessObjectFormatFileType(FileTypeEntity.TXT_FILE_TYPE);
    request.setBusinessObjectFormatVersion(AbstractServiceTest.FORMAT_VERSION);
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    request.setPartitionValueFilters(Arrays.asList(partitionValueFilter));
    partitionValueFilter.setPartitionKey(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME);
    if (startPartitionValue != null || endPartitionValue != null) {
        PartitionValueRange partitionValueRange = new PartitionValueRange();
        partitionValueFilter.setPartitionValueRange(partitionValueRange);
        partitionValueRange.setStartPartitionValue(startPartitionValue);
        partitionValueRange.setEndPartitionValue(endPartitionValue);
    }
    if (partitionValues != null) {
        partitionValueFilter.setPartitionValues(new ArrayList(partitionValues));
    }
    request.setBusinessObjectDataVersion(AbstractServiceTest.DATA_VERSION);
    request.setStorageName(AbstractServiceTest.STORAGE_NAME);
    request.setOutputFormat(BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL);
    request.setTableName(AbstractServiceTest.TABLE_NAME);
    request.setCustomDdlName(customDdlName);
    request.setIncludeDropTableStatement(true);
    request.setIncludeIfNotExistsOption(true);
    request.setAllowMissingData(true);
    request.setIncludeAllRegisteredSubPartitions(AbstractServiceTest.NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS);
    return request;
}

Example 53 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method buildPartitionFilters.

/**
 * Build partition filters based on the specified partition value filters.  This method also validates the partition value filters (including partition
 * keys) against the business object format schema.  When request contains multiple partition value filters, the system will check business object data
 * availability for n-fold Cartesian product of the partition values specified, where n is a number of partition value filters (partition value sets).
 *
 * @param partitionValueFilters the list of partition value filters
 * @param standalonePartitionValueFilter the standalone partition value filter
 * @param businessObjectFormatKey the business object format key
 * @param businessObjectDataVersion the business object data version
 * @param storageNames the optional list of storage names (case-insensitive)
 * @param storagePlatformType the optional storage platform type, e.g. S3 for Hive DDL. It is ignored when the list of storages is not empty
 * @param excludedStoragePlatformType the optional storage platform type to be excluded from search. It is ignored when the list of storages is not empty or
 * the storage platform type is specified
 * @param businessObjectFormatEntity the business object format entity
 *
 * @return the list of partition filters
 */
public List<List<String>> buildPartitionFilters(List<PartitionValueFilter> partitionValueFilters, PartitionValueFilter standalonePartitionValueFilter, BusinessObjectFormatKey businessObjectFormatKey, Integer businessObjectDataVersion, List<String> storageNames, String storagePlatformType, String excludedStoragePlatformType, BusinessObjectFormatEntity businessObjectFormatEntity) {
    // Build a list of partition value filters to process based on the specified partition value filters.
    List<PartitionValueFilter> partitionValueFiltersToProcess = getPartitionValuesToProcess(partitionValueFilters, standalonePartitionValueFilter);
    // Build a map of column positions and the relative partition values.
    Map<Integer, List<String>> partitionValues = new HashMap<>();
    // Initialize the map with null partition values for all possible primary and sub-partition values. Partition column position uses one-based numbering.
    for (int i = 0; i < BusinessObjectDataEntity.MAX_SUBPARTITIONS + 1; i++) {
        partitionValues.put(i, NULL_VALUE_LIST);
    }
    // Process all partition value filters one by one and populate the relative entries in the map.
    for (PartitionValueFilter partitionValueFilter : partitionValueFiltersToProcess) {
        // Get the partition key.  If partition key is not specified, use the primary partition column.
        String partitionKey = StringUtils.isNotBlank(partitionValueFilter.getPartitionKey()) ? partitionValueFilter.getPartitionKey() : businessObjectFormatEntity.getPartitionKey();
        // Get the partition column position (one-based numbering).
        int partitionColumnPosition = getPartitionColumnPosition(partitionKey, businessObjectFormatEntity);
        // Get unique and sorted list of partition values to check the availability for.
        List<String> uniqueAndSortedPartitionValues = getPartitionValues(partitionValueFilter, partitionKey, partitionColumnPosition, businessObjectFormatKey, businessObjectDataVersion, storageNames, storagePlatformType, excludedStoragePlatformType, businessObjectFormatEntity);
        // Add this partition value filter to the map.
        List<String> previousPartitionValues = partitionValues.put(partitionColumnPosition - 1, uniqueAndSortedPartitionValues);
        // Check if this partition column has not been already added.
        if (!NULL_VALUE_LIST.equals(previousPartitionValues)) {
            throw new IllegalArgumentException("Partition value filters specify duplicate partition columns.");
        }
    }
    // When request contains multiple partition value filters, the system will check business object data availability for n-fold Cartesian product
    // of the partition values specified, where n is a number of partition value filters (partition value sets).
    List<String[]> crossProductResult = getCrossProduct(partitionValues);
    List<List<String>> partitionFilters = new ArrayList<>();
    for (String[] crossProductRow : crossProductResult) {
        partitionFilters.add(Arrays.asList(crossProductRow));
    }
    return partitionFilters;
}

Example 54 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataSearchHelperTest method testValidateBusinessObjectDataSearchKeyIllegalArgumentExceptions.

@Test
public void testValidateBusinessObjectDataSearchKeyIllegalArgumentExceptions() {
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(null);
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("A business object data search key must be specified.", e.getMessage());
    }
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(new BusinessObjectDataSearchKey(BLANK_TEXT, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NO_PARTITION_VALUE_FILTERS, NO_ATTRIBUTE_VALUE_FILTERS, NO_FILTER_ON_LATEST_VALID_VERSION, NO_FILTER_ON_RETENTION_EXPIRATION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("A namespace must be specified.", e.getMessage());
    }
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(new BusinessObjectDataSearchKey(BDEF_NAMESPACE, BLANK_TEXT, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NO_PARTITION_VALUE_FILTERS, NO_ATTRIBUTE_VALUE_FILTERS, NO_FILTER_ON_LATEST_VALID_VERSION, NO_FILTER_ON_RETENTION_EXPIRATION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("A business object definition name must be specified.", e.getMessage());
    }
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(new BusinessObjectDataSearchKey(BDEF_NAMESPACE, BDEF_NAME, BLANK_TEXT, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NO_PARTITION_VALUE_FILTERS, NO_ATTRIBUTE_VALUE_FILTERS, NO_FILTER_ON_LATEST_VALID_VERSION, NO_FILTER_ON_RETENTION_EXPIRATION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("A business object format usage must be specified.", e.getMessage());
    }
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(new BusinessObjectDataSearchKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Collections.singletonList(new PartitionValueFilter(PARTITION_KEY, NO_PARTITION_VALUES, NO_PARTITION_VALUE_RANGE, new LatestBeforePartitionValue(PARTITION_VALUE), NO_LATEST_AFTER_PARTITION_VALUE)), NO_ATTRIBUTE_VALUE_FILTERS, NO_FILTER_ON_LATEST_VALID_VERSION, NO_FILTER_ON_RETENTION_EXPIRATION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("Only partition values or partition range are supported in partition value filter.", e.getMessage());
    }
    try {
        businessObjectDataSearchHelper.validateBusinessObjectDataSearchKey(new BusinessObjectDataSearchKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, NO_PARTITION_VALUE_FILTERS, Collections.singletonList(new AttributeValueFilter(BLANK_TEXT, EMPTY_STRING)), NO_FILTER_ON_LATEST_VALID_VERSION, NO_FILTER_ON_RETENTION_EXPIRATION));
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("Either attribute name or attribute value filter must be specified.", e.getMessage());
    }
}

Example 55 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceCheckBusinessObjectDataAvailabilityTest method testCheckBusinessObjectDataAvailabilityNoStorageNamesAndMultipleValidBusinessObjectDataVersionsInDifferentStorage.

@Test
public void testCheckBusinessObjectDataAvailabilityNoStorageNamesAndMultipleValidBusinessObjectDataVersionsInDifferentStorage() {
    // Create business object data keys for two versions of business object data.
    List<BusinessObjectDataKey> businessObjectDataKeys = Arrays.asList(new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, INITIAL_DATA_VERSION), new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, SECOND_DATA_VERSION));
    // Create two storage units for the business object data instances in two different storage entities.
    storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME, businessObjectDataKeys.get(0), NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    storageUnitDaoTestHelper.createStorageUnitEntity(STORAGE_NAME_2, businessObjectDataKeys.get(1), NO_LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.ENABLED, NO_STORAGE_DIRECTORY_PATH);
    // Check business object data availability when business object data version along with storage names are not specified
    // and different valid business object data versions are registered in different storage entities.
    BusinessObjectDataAvailability resultBusinessObjectDataAvailability = businessObjectDataService.checkBusinessObjectDataAvailability(new BusinessObjectDataAvailabilityRequest(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, NO_STORAGE_NAME, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS));
    // Validate the response object.
    assertEquals(new BusinessObjectDataAvailability(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(PARTITION_KEY, Arrays.asList(PARTITION_VALUE), NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, NO_LATEST_AFTER_PARTITION_VALUE)), NO_STANDALONE_PARTITION_VALUE_FILTER, NO_DATA_VERSION, NO_STORAGE_NAMES, NO_STORAGE_NAME, Arrays.asList(new BusinessObjectDataStatus(FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, SECOND_DATA_VERSION, BusinessObjectDataStatusEntity.VALID)), new ArrayList<>()), resultBusinessObjectDataAvailability);
}