Examples with PreFederatedQueryPlugin ddf.catalog.plugin.PreFederatedQueryPlugin used on opensource projects

Example 1 with PreFederatedQueryPlugin

use of ddf.catalog.plugin.PreFederatedQueryPlugin in project ddf by codice.

the class AbstractFederationStrategy method federate.

@Override
public QueryResponse federate(List<Source> sources, final QueryRequest queryRequest) {
    final String methodName = "federate";
    LOGGER.trace("ENTERING: {}", methodName);
    if (LOGGER.isDebugEnabled()) {
        for (Source source : sources) {
            if (source != null) {
                LOGGER.debug("source to query: {}", source.getId());
            }
        }
    }
    Query originalQuery = queryRequest.getQuery();
    int offset = originalQuery.getStartIndex();
    final int pageSize = originalQuery.getPageSize();
    // limit offset to max value
    if (offset > this.maxStartIndex) {
        offset = this.maxStartIndex;
    }
    final QueryResponseImpl queryResponseQueue = new QueryResponseImpl(queryRequest, null);
    Map<Source, Future<SourceResponse>> futures = new HashMap<Source, Future<SourceResponse>>();
    Query modifiedQuery = getModifiedQuery(originalQuery, sources.size(), offset, pageSize);
    QueryRequest modifiedQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), queryRequest.getSourceIds(), queryRequest.getProperties());
    // Do NOT call source.isAvailable() when checking sources
    for (final Source source : sources) {
        if (source != null) {
            if (!futures.containsKey(source)) {
                LOGGER.debug("running query on source: {}", source.getId());
                try {
                    for (PreFederatedQueryPlugin service : preQuery) {
                        try {
                            modifiedQueryRequest = service.process(source, modifiedQueryRequest);
                        } catch (PluginExecutionException e) {
                            LOGGER.info("Error executing PreFederatedQueryPlugin: ", e);
                        }
                    }
                } catch (StopProcessingException e) {
                    LOGGER.info("Plugin stopped processing: ", e);
                }
                futures.put(source, queryExecutorService.submit(new CallableSourceResponse(source, modifiedQueryRequest.getQuery(), modifiedQueryRequest.getProperties())));
            } else {
                LOGGER.info("Duplicate source found with name {}. Ignoring second one.", source.getId());
            }
        }
    }
    QueryResponseImpl offsetResults = null;
    // OffsetResultHandler does.
    if (offset > 1 && sources.size() > 1) {
        offsetResults = new QueryResponseImpl(queryRequest, null);
        queryExecutorService.submit(new OffsetResultHandler(queryResponseQueue, offsetResults, pageSize, offset));
    }
    queryExecutorService.submit(createMonitor(queryExecutorService, futures, queryResponseQueue, modifiedQueryRequest.getQuery()));
    QueryResponse queryResponse = null;
    if (offset > 1 && sources.size() > 1) {
        queryResponse = offsetResults;
        LOGGER.debug("returning offsetResults");
    } else {
        queryResponse = queryResponseQueue;
        LOGGER.debug("returning returnResults: {}", queryResponse);
    }
    try {
        for (PostFederatedQueryPlugin service : postQuery) {
            try {
                queryResponse = service.process(queryResponse);
            } catch (PluginExecutionException e) {
                LOGGER.info("Error executing PostFederatedQueryPlugin: ", e);
            }
        }
    } catch (StopProcessingException e) {
        LOGGER.info("Plugin stopped processing: ", e);
    }
    LOGGER.debug("returning Query Results: {}", queryResponse);
    LOGGER.trace("EXITING: {}.federate", CLASS_NAME);
    return queryResponse;
}

Example 2 with PreFederatedQueryPlugin

use of ddf.catalog.plugin.PreFederatedQueryPlugin in project ddf by codice.

the class FederationStrategyTest method testFederateOneSourceOffsetTwoPageSizeTwo.

/**
     * Verify that the original query is used by the source.
     * <p>
     * No special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateOneSourceOffsetTwoPageSizeTwo() throws Exception {
    LOGGER.debug("testFederate_OneSource_OffsetTwo_PageSizeTwo()");
    // Offset of 2
    when(mockQuery.getStartIndex()).thenReturn(2);
    // Page size of 2
    when(mockQuery.getPageSize()).thenReturn(2);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    // ArgumentCaptor<QueryRequest> argument = ArgumentCaptor.forClass(QueryRequest.class);
    /**
         * When using the original query to query the source, the desired offset and page size are
         * used. So, the results returned by the source already have the offset and page size taken
         * into account.
         */
    Result mockResult1 = mock(Result.class);
    Result mockResult2 = mock(Result.class);
    SourceResponse mockSourceResponse = mock(SourceResponse.class);
    List<Result> results = Arrays.asList(mockResult1, mockResult2);
    when(mockSourceResponse.getResults()).thenReturn(results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSourceResponse);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.1 #######");
    // Only one source
    List<Source> sources = new ArrayList<Source>(1);
    sources.add(mockSource1);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Verification
    assertNotNull(federatedResponse);
    LOGGER.debug("Federated response result size: {}", federatedResponse.getResults().size());
    /**
         * Verify two results (page size) are returned. The results returned by the source already
         * have the offset and page size taken into account, so we can verify that the lists match.
         */
    assertEquals(2, federatedResponse.getResults().size());
    assertEquals(mockResult1, federatedResponse.getResults().get(0));
    assertEquals(mockResult2, federatedResponse.getResults().get(1));
    LOGGER.debug("mockResult1: {}", mockResult1);
    LOGGER.debug("mockResult2: {}", mockResult2);
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("result: {}", result);
    }
}

Example 3 with PreFederatedQueryPlugin

use of ddf.catalog.plugin.PreFederatedQueryPlugin in project ddf by codice.

the class FederationStrategyTest method testFederateOneSourceOffsetOnePageSizeTwo.

/**
     * Verify that the original query is used by the source.
     * <p>
     * No special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateOneSourceOffsetOnePageSizeTwo() throws Exception {
    LOGGER.debug("testFederate_OneSource_OffsetOne_PageSizeTwo()");
    // Offset of 1
    when(mockQuery.getStartIndex()).thenReturn(1);
    // Page size of 2
    when(mockQuery.getPageSize()).thenReturn(2);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    /**
         * When using the original query to query the source, the desired offset and page size are
         * used. So, the results returned by the source already have the offset and page size taken
         * into account.
         */
    Result mockResult1 = mock(Result.class);
    Result mockResult2 = mock(Result.class);
    SourceResponse mockSourceResponse = mock(SourceResponse.class);
    List<Result> results = Arrays.asList(mockResult1, mockResult2);
    when(mockSourceResponse.getResults()).thenReturn(results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSourceResponse);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.2 #######");
    // Only one source
    List<Source> sources = new ArrayList<Source>(1);
    sources.add(mockSource1);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Verification
    assertNotNull(federatedResponse);
    LOGGER.debug("Federated response result size: {}", federatedResponse.getResults().size());
    /**
         * Verify two results (page size) are returned. The results returned by the source already
         * have the offset and page size taken into account, so we can verify that the lists match.
         */
    assertEquals(2, federatedResponse.getResults().size());
    assertEquals(mockResult1, federatedResponse.getResults().get(0));
    assertEquals(mockResult2, federatedResponse.getResults().get(1));
    LOGGER.debug("mockResult1: {}", mockResult1);
    LOGGER.debug("mockResult2: {}", mockResult2);
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("result: {}", result);
    }
    // Check the responseProperties
    assertEquals("####### MOCK SOURCE 1.2 #######", ((List) federatedResponse.getPropertyValue(QueryResponse.SITE_LIST)).get(0));
    Map<String, Serializable> siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 1.2 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
}

Example 4 with PreFederatedQueryPlugin

use of ddf.catalog.plugin.PreFederatedQueryPlugin in project ddf by codice.

the class FederationStrategyTest method testFederateTwoSourcesOffsetOnePageSizeThree.

/**
     * Verify that the original query passed is used by the sources.
     * <p>
     * No special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateTwoSourcesOffsetOnePageSizeThree() throws Exception {
    LOGGER.debug("testFederate_TwoSources_OffsetOne_PageSizeThree()");
    // Offset of 1
    when(mockQuery.getStartIndex()).thenReturn(1);
    // Page size of 3
    when(mockQuery.getPageSize()).thenReturn(3);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    /**
         * When using the original query to query the sources, the desired offset and page size are
         * used. So, the results returned by the sources already have the offset and page size taken
         * into account.
         */
    Result mockSource1Result1 = mock(Result.class);
    Mockito.when(mockSource1Result1.getRelevanceScore()).thenReturn(0.7);
    Result mockSource1Result2 = mock(Result.class);
    Mockito.when(mockSource1Result2.getRelevanceScore()).thenReturn(0.5);
    Result mockSource1Result3 = mock(Result.class);
    Mockito.when(mockSource1Result3.getRelevanceScore()).thenReturn(0.3);
    Result mockSource1Result4 = mock(Result.class);
    Mockito.when(mockSource1Result4.getRelevanceScore()).thenReturn(0.1);
    SourceResponse mockSource1Response = mock(SourceResponse.class);
    List<Result> mockSource1Results = Arrays.asList(mockSource1Result1, mockSource1Result2, mockSource1Result3, mockSource1Result4);
    when(mockSource1Response.getResults()).thenReturn(mockSource1Results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSource1Response);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.4 #######");
    Result mockSource2Result1 = mock(Result.class);
    Mockito.when(mockSource2Result1.getRelevanceScore()).thenReturn(0.8);
    Result mockSource2Result2 = mock(Result.class);
    Mockito.when(mockSource2Result2.getRelevanceScore()).thenReturn(0.6);
    Result mockSource2Result3 = mock(Result.class);
    Mockito.when(mockSource2Result3.getRelevanceScore()).thenReturn(0.4);
    Result mockSource2Result4 = mock(Result.class);
    Mockito.when(mockSource2Result4.getRelevanceScore()).thenReturn(0.2);
    SourceResponse mockSource2Response = mock(SourceResponse.class);
    List<Result> mockSource2Results = Arrays.asList(mockSource2Result1, mockSource2Result2, mockSource2Result3, mockSource2Result4);
    when(mockSource2Response.getResults()).thenReturn(mockSource2Results);
    Source mockSource2 = mock(Source.class);
    when(mockSource2.query(any(QueryRequest.class))).thenReturn(mockSource2Response);
    when(mockSource2.getId()).thenReturn("####### MOCK SOURCE 2.4 #######");
    // Two sources
    List<Source> sources = new ArrayList<Source>(2);
    sources.add(mockSource1);
    sources.add(mockSource2);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Verification
    assertNotNull(federatedResponse);
    LOGGER.debug("Federated response result size: {}", federatedResponse.getResults().size());
    /**
         * Verify three results (page size) are returned. Since we are using mock Results, the
         * relevance score is 0.0, and the merged and sorted results of both sources is
         * mockSource2Result1, mockSource1Result1, mockSource2Result2, mockSource1Result2,
         * mockSource2Result3, mockSource1Result3, mockSource2Result4, mockSource1Result4. So, the
         * results are mockSource2Result1, mockSource1Result1, mockSource2Result2. No need to use
         * OffsetResultHander in this case.
         */
    assertEquals(3, federatedResponse.getResults().size());
    assertEquals(mockSource2Result1, federatedResponse.getResults().get(0));
    assertEquals(mockSource1Result1, federatedResponse.getResults().get(1));
    assertEquals(mockSource2Result2, federatedResponse.getResults().get(2));
    LOGGER.debug("mockSource2Result1: {}", mockSource2Result1);
    LOGGER.debug("mockSource1Result1: {}", mockSource1Result1);
    LOGGER.debug("mockSource2Result2: {}", mockSource2Result2);
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("federated response result: {}", result);
    }
    // Check the responseProperties
    List<String> siteList = (List) federatedResponse.getPropertyValue(QueryResponse.SITE_LIST);
    assertTrue(siteList.contains("####### MOCK SOURCE 2.4 #######"));
    Map<String, Serializable> siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 2.4 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
    assertTrue(siteList.contains("####### MOCK SOURCE 2.4 #######"));
    siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 1.4 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
}

Example 5 with PreFederatedQueryPlugin

use of ddf.catalog.plugin.PreFederatedQueryPlugin in project ddf by codice.

the class FederationStrategyTest method testNegativePageSizeQuery.

@Test
public void testNegativePageSizeQuery() throws Exception {
    when(mockQuery.getPageSize()).thenReturn(-1);
    QueryRequest fedQueryRequest = mock(QueryRequest.class);
    when(fedQueryRequest.getQuery()).thenReturn(mockQuery);
    Result mockResult = mock(Result.class);
    SourceResponse mockResponse = mock(SourceResponse.class);
    List<Result> results = Arrays.asList(mockResult);
    when(mockResponse.getHits()).thenReturn((long) results.size());
    when(mockResponse.getResults()).thenReturn(results);
    CatalogProvider mockProvider = mock(CatalogProvider.class);
    when(mockProvider.query(any(QueryRequest.class))).thenReturn(mockResponse);
    when(mockProvider.getId()).thenReturn("mock provider");
    List<Source> sources = new ArrayList<Source>();
    sources.add(mockProvider);
    SortedFederationStrategy sortedStrategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    QueryResponse fedResponse = sortedStrategy.federate(sources, fedQueryRequest);
    assertEquals(1, fedResponse.getResults().size());
    FifoFederationStrategy fifoStrategy = new FifoFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    fedResponse = fifoStrategy.federate(sources, fedQueryRequest);
    assertEquals(1, fedResponse.getResults().size());
}