Examples with PartitionValueFilter org.finra.herd.model.api.xml.PartitionValueFilter used on opensource projects

Example 41 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceGenerateBusinessObjectDataDdlTest method testGenerateBusinessObjectDataDdlLatestAfterPartitionValueNoExists.

@Test
public void testGenerateBusinessObjectDataDdlLatestAfterPartitionValueNoExists() {
    // Prepare database entities required for testing.
    businessObjectDataServiceTestHelper.createDatabaseEntitiesForBusinessObjectDataDdlTesting(PARTITION_VALUE);
    // Try retrieve business object data ddl using a latest before partition value filter option when the latest partition value does not exist.
    try {
        businessObjectDataService.generateBusinessObjectDataDdl(new BusinessObjectDataDdlRequest(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(FIRST_PARTITION_COLUMN_NAME, NO_PARTITION_VALUES, NO_PARTITION_VALUE_RANGE, NO_LATEST_BEFORE_PARTITION_VALUE, new LatestAfterPartitionValue(PARTITION_VALUE_2))), NO_STANDALONE_PARTITION_VALUE_FILTER, DATA_VERSION, NO_STORAGE_NAMES, STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, TABLE_NAME, NO_CUSTOM_DDL_NAME, INCLUDE_DROP_TABLE_STATEMENT, INCLUDE_IF_NOT_EXISTS_OPTION, NO_INCLUDE_DROP_PARTITIONS, NO_ALLOW_MISSING_DATA, NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS));
        fail("Suppose to throw an ObjectNotFoundException when failed to find the latest after partition value.");
    } catch (ObjectNotFoundException e) {
        assertEquals(String.format("Failed to find partition value which is the latest after partition value = \"%s\" " + "for partition key = \"%s\" due to no available business object data " + "in \"%s\" storage that satisfies the search criteria. Business object data {namespace: \"%s\", " + "businessObjectDefinitionName: \"%s\", businessObjectFormatUsage: \"%s\", businessObjectFormatFileType: \"%s\", " + "businessObjectFormatVersion: %d, businessObjectDataVersion: %d}", PARTITION_VALUE_2, FIRST_PARTITION_COLUMN_NAME, STORAGE_NAME, NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, FORMAT_VERSION, DATA_VERSION), e.getMessage());
    }
}

Example 42 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getExpectedBusinessObjectDataDdlCollectionResponse.

/**
 * Creates an expected generate business object data ddl collection response using hard coded test values.
 *
 * @return the business object data ddl collection response
 */
public BusinessObjectDataDdlCollectionResponse getExpectedBusinessObjectDataDdlCollectionResponse() {
    // Prepare a generate business object data collection response using hard coded test values.
    BusinessObjectDataDdlCollectionResponse businessObjectDataDdlCollectionResponse = new BusinessObjectDataDdlCollectionResponse();
    // Create a list of business object data ddl responses.
    List<BusinessObjectDataDdl> businessObjectDataDdlResponses = new ArrayList<>();
    businessObjectDataDdlCollectionResponse.setBusinessObjectDataDdlResponses(businessObjectDataDdlResponses);
    // Get the actual HIVE DDL expected to be generated.
    String expectedDdl = getExpectedBusinessObjectDataDdl(AbstractServiceTest.PARTITION_VALUE);
    // Create a business object data ddl response.
    BusinessObjectDataDdl expectedBusinessObjectDataDdl = new BusinessObjectDataDdl(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, Arrays.asList(AbstractServiceTest.PARTITION_VALUE), AbstractServiceTest.NO_PARTITION_VALUE_RANGE, AbstractServiceTest.NO_LATEST_BEFORE_PARTITION_VALUE, AbstractServiceTest.NO_LATEST_AFTER_PARTITION_VALUE)), AbstractServiceTest.NO_STANDALONE_PARTITION_VALUE_FILTER, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.NO_STORAGE_NAMES, AbstractServiceTest.STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, AbstractServiceTest.TABLE_NAME, AbstractServiceTest.NO_CUSTOM_DDL_NAME, expectedDdl);
    // Add two business object ddl responses to the collection response.
    businessObjectDataDdlResponses.add(expectedBusinessObjectDataDdl);
    businessObjectDataDdlResponses.add(expectedBusinessObjectDataDdl);
    // Set the expected DDL collection value.
    businessObjectDataDdlCollectionResponse.setDdlCollection(String.format("%s\n\n%s", expectedDdl, expectedDdl));
    return businessObjectDataDdlCollectionResponse;
}

Example 43 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataServiceTestHelper method getTestBusinessObjectDataDdlCollectionRequest.

/**
 * Creates a generate business object data ddl collection request using hard coded test values.
 *
 * @return the business object data ddl collection request
 */
public BusinessObjectDataDdlCollectionRequest getTestBusinessObjectDataDdlCollectionRequest() {
    // Create a generate business object data ddl collection request.
    BusinessObjectDataDdlCollectionRequest businessObjectDataDdlCollectionRequest = new BusinessObjectDataDdlCollectionRequest();
    // Create a list of generate business object data ddl requests.
    List<BusinessObjectDataDdlRequest> businessObjectDataDdlRequests = new ArrayList<>();
    businessObjectDataDdlCollectionRequest.setBusinessObjectDataDdlRequests(businessObjectDataDdlRequests);
    // Create a generate business object data ddl request.
    BusinessObjectDataDdlRequest businessObjectDataDdlRequest = new BusinessObjectDataDdlRequest(AbstractServiceTest.NAMESPACE, AbstractServiceTest.BDEF_NAME, AbstractServiceTest.FORMAT_USAGE_CODE, FileTypeEntity.TXT_FILE_TYPE, AbstractServiceTest.FORMAT_VERSION, Arrays.asList(new PartitionValueFilter(AbstractServiceTest.FIRST_PARTITION_COLUMN_NAME, Arrays.asList(AbstractServiceTest.PARTITION_VALUE), AbstractServiceTest.NO_PARTITION_VALUE_RANGE, AbstractServiceTest.NO_LATEST_BEFORE_PARTITION_VALUE, AbstractServiceTest.NO_LATEST_AFTER_PARTITION_VALUE)), AbstractServiceTest.NO_STANDALONE_PARTITION_VALUE_FILTER, AbstractServiceTest.DATA_VERSION, AbstractServiceTest.NO_STORAGE_NAMES, AbstractServiceTest.STORAGE_NAME, BusinessObjectDataDdlOutputFormatEnum.HIVE_13_DDL, AbstractServiceTest.TABLE_NAME, AbstractServiceTest.NO_CUSTOM_DDL_NAME, AbstractServiceTest.INCLUDE_DROP_TABLE_STATEMENT, AbstractServiceTest.INCLUDE_IF_NOT_EXISTS_OPTION, AbstractServiceTest.INCLUDE_DROP_PARTITIONS, AbstractServiceTest.NO_ALLOW_MISSING_DATA, AbstractServiceTest.NO_INCLUDE_ALL_REGISTERED_SUBPARTITIONS, AbstractServiceTest.NO_SUPPRESS_SCAN_FOR_UNREGISTERED_SUBPARTITIONS);
    // Add two business object ddl requests to the collection request.
    businessObjectDataDdlRequests.add(businessObjectDataDdlRequest);
    businessObjectDataDdlRequests.add(businessObjectDataDdlRequest);
    return businessObjectDataDdlCollectionRequest;
}

Example 44 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataDaoTest method testBusinessObjectDataSearchWithPartitionValueFilters.

@Test
public void testBusinessObjectDataSearchWithPartitionValueFilters() {
    BusinessObjectDataEntity businessObjectDataEntity = createBusinessObjectEntityForPartitionValueFilterTest();
    String namespace = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getNamespace().getCode();
    String bDefName = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getName();
    String usage = businessObjectDataEntity.getBusinessObjectFormat().getUsage();
    String fileTypeCode = businessObjectDataEntity.getBusinessObjectFormat().getFileType().getCode();
    int formatVersion = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectFormatVersion();
    BusinessObjectDataSearchKey businessObjectDataSearchKey = new BusinessObjectDataSearchKey();
    List<PartitionValueFilter> partitionValueFilters = new ArrayList<>();
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    partitionValueFilters.add(partitionValueFilter);
    partitionValueFilter.setPartitionKey(PARTITION_KEY);
    List<String> values = new ArrayList<>();
    values.add(PARTITION_VALUE);
    partitionValueFilter.setPartitionValues(values);
    for (int i = 1; i < 5; i++) {
        PartitionValueFilter partitionValueFilter2 = new PartitionValueFilter();
        partitionValueFilters.add(partitionValueFilter2);
        partitionValueFilter2.setPartitionKey(PARTITION_KEY + i);
        List<String> values2 = new ArrayList<>();
        // add the expected sub partition value
        values2.add(SUBPARTITION_VALUES.get(i - 1));
        // add some noise, it should be ignored as the query is doing in
        values2.add("NOISE");
        partitionValueFilter2.setPartitionValues(values2);
    }
    businessObjectDataSearchKey.setPartitionValueFilters(partitionValueFilters);
    businessObjectDataSearchKey.setNamespace(namespace);
    businessObjectDataSearchKey.setBusinessObjectDefinitionName(bDefName);
    businessObjectDataSearchKey.setBusinessObjectFormatUsage(usage);
    businessObjectDataSearchKey.setBusinessObjectFormatFileType(fileTypeCode);
    businessObjectDataSearchKey.setBusinessObjectFormatVersion(formatVersion);
    List<BusinessObjectData> result = businessObjectDataDao.searchBusinessObjectData(businessObjectDataSearchKey, DEFAULT_PAGE_NUMBER, DEFAULT_PAGE_SIZE);
    assertEquals(1, result.size());
    for (BusinessObjectData data : result) {
        assertEquals(NAMESPACE, data.getNamespace());
        assertEquals(BDEF_NAME, data.getBusinessObjectDefinitionName());
        assertEquals(FORMAT_USAGE_CODE, data.getBusinessObjectFormatUsage());
        assertEquals(FORMAT_FILE_TYPE_CODE, data.getBusinessObjectFormatFileType());
        assertTrue(FORMAT_VERSION == data.getBusinessObjectFormatVersion());
    }
}

Example 45 with PartitionValueFilter

use of org.finra.herd.model.api.xml.PartitionValueFilter in project herd by FINRAOS.

the class BusinessObjectDataDaoTest method testBusinessObjectDataSearchWithPartitionValueRangeFiltersNegative.

@Test
public void testBusinessObjectDataSearchWithPartitionValueRangeFiltersNegative() {
    BusinessObjectDataEntity businessObjectDataEntity = createBusinessObjectEntityForPartitionValueFilterTest();
    String namespace = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getNamespace().getCode();
    String bDefName = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectDefinition().getName();
    String usage = businessObjectDataEntity.getBusinessObjectFormat().getUsage();
    String fileTypeCode = businessObjectDataEntity.getBusinessObjectFormat().getFileType().getCode();
    int formatVerion = businessObjectDataEntity.getBusinessObjectFormat().getBusinessObjectFormatVersion();
    BusinessObjectDataSearchKey businessObjectDataSearchKey = new BusinessObjectDataSearchKey();
    List<PartitionValueFilter> partitionValueFilters = new ArrayList<>();
    PartitionValueFilter partitionValueFilter = new PartitionValueFilter();
    partitionValueFilters.add(partitionValueFilter);
    partitionValueFilter.setPartitionKey(PARTITION_KEY + "6");
    PartitionValueRange partitionValueRange = new PartitionValueRange();
    partitionValueRange.setStartPartitionValue(PARTITION_VALUE);
    partitionValueRange.setEndPartitionValue(PARTITION_VALUE + "1");
    partitionValueFilter.setPartitionValueRange(partitionValueRange);
    businessObjectDataSearchKey.setPartitionValueFilters(partitionValueFilters);
    businessObjectDataSearchKey.setNamespace(namespace);
    businessObjectDataSearchKey.setBusinessObjectDefinitionName(bDefName);
    businessObjectDataSearchKey.setBusinessObjectFormatUsage(usage);
    businessObjectDataSearchKey.setBusinessObjectFormatFileType(fileTypeCode);
    businessObjectDataSearchKey.setBusinessObjectFormatVersion(formatVerion);
    List<BusinessObjectData> result = businessObjectDataDao.searchBusinessObjectData(businessObjectDataSearchKey, DEFAULT_PAGE_NUMBER, DEFAULT_PAGE_SIZE);
    assertEquals(0, result.size());
}