Examples with Source ddf.catalog.source.Source used on opensource projects

Example 61 with Source

use of ddf.catalog.source.Source in project ddf by codice.

the class FederationStrategyTest method testFederateTwoSourcesOffsetTwoPageSizeThree.

/**
     * Verify that a modified version of the mockQuery passed is used by the sources.
     * <p>
     * Special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateTwoSourcesOffsetTwoPageSizeThree() throws Exception {
    LOGGER.debug("testFederate_TwoSources_OffsetTwo_PageSizeThree()");
    // Test Setup
    Query mockQuery = mock(QueryImpl.class);
    // Offset of 2
    when(mockQuery.getStartIndex()).thenReturn(2);
    // Page size of 3
    when(mockQuery.getPageSize()).thenReturn(3);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    ArgumentCaptor<QueryRequest> argument1 = ArgumentCaptor.forClass(QueryRequest.class);
    ArgumentCaptor<QueryRequest> argument2 = ArgumentCaptor.forClass(QueryRequest.class);
    /**
         * When using a modified mockQuery to mockQuery the sources, the desired offset and page size are
         * NOT used. So, the results returned by each source start at index 1 and end at (offset +
         * pageSize - 1).
         *
         * Number of results returned by each source = offset + pageSize - 1 4 = 2 + 3 - 1
         */
    Result mockSource1Result1 = mock(Result.class);
    Result mockSource1Result2 = mock(Result.class);
    Result mockSource1Result3 = mock(Result.class);
    Result mockSource1Result4 = mock(Result.class);
    SourceResponse mockSource1Response = mock(SourceResponse.class);
    List<Result> mockSource1Results = Arrays.asList(mockSource1Result1, mockSource1Result2, mockSource1Result3, mockSource1Result4);
    when(mockSource1Response.getResults()).thenReturn(mockSource1Results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSource1Response);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.3 #######");
    /**
         * When using a modified mockQuery to mockQuery the sources, the desired offset and page size are
         * NOT used. So, the results returned by each source start at index 1 and end at (offset +
         * pageSize - 1).
         *
         * Number of results returned by each source = offset + pageSize - 1 4 = 2 + 3 - 1
         */
    Result mockSource2Result1 = mock(Result.class);
    Result mockSource2Result2 = mock(Result.class);
    Result mockSource2Result3 = mock(Result.class);
    Result mockSource2Result4 = mock(Result.class);
    SourceResponse mockSource2Response = mock(SourceResponse.class);
    List<Result> mockSource2Results = Arrays.asList(mockSource2Result1, mockSource2Result2, mockSource2Result3, mockSource2Result4);
    when(mockSource2Response.getResults()).thenReturn(mockSource2Results);
    Source mockSource2 = mock(Source.class);
    when(mockSource2.query(any(QueryRequest.class))).thenReturn(mockSource2Response);
    when(mockSource2.getId()).thenReturn("####### MOCK SOURCE 2.3 #######");
    // Two sources
    List<Source> sources = new ArrayList<Source>(2);
    sources.add(mockSource1);
    sources.add(mockSource2);
    Result mockSortedResult1 = mock(Result.class);
    Result mockSortedResult2 = mock(Result.class);
    Result mockSortedResult3 = mock(Result.class);
    Result mockSortedResult4 = mock(Result.class);
    Result mockSortedResult5 = mock(Result.class);
    Result mockSortedResult6 = mock(Result.class);
    Result mockSortedResult7 = mock(Result.class);
    Result mockSortedResult8 = mock(Result.class);
    List<Result> mockSortedResults = Arrays.asList(mockSortedResult1, mockSortedResult2, mockSortedResult3, mockSortedResult4, mockSortedResult5, mockSortedResult6, mockSortedResult7, mockSortedResult8);
    QueryResponseImpl mockOriginalResults = Mockito.mock(QueryResponseImpl.class);
    // Return true for the number of mockSortedResults
    Mockito.when(mockOriginalResults.hasMoreResults()).thenReturn(true, true, true, true, true, true, true, true, false);
    Mockito.when(mockOriginalResults.getResults()).thenReturn(mockSortedResults);
    // Returns the sorted results from both sources (4 + 4 = 8)
    Mockito.when(mockOriginalResults.take()).thenReturn(mockSortedResult1, mockSortedResult2, mockSortedResult3, mockSortedResult4, mockSortedResult5, mockSortedResult6, mockSortedResult7, mockSortedResult8);
    QueryResponseImpl offsetResultQueue = new QueryResponseImpl(queryRequest, null);
    PowerMockito.whenNew(QueryResponseImpl.class).withArguments(queryRequest, (Map<String, Serializable>) null).thenReturn(mockOriginalResults, offsetResultQueue);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Make sure we've finished running the test
    killAndWaitForExecutor();
    // Verification
    assertNotNull(federatedResponse);
    verify(mockSource1).query(argument1.capture());
    // The modified query should have a start index of 1 and an end index of offset + pageSize -
    // 1
    assertEquals(1, argument1.getValue().getQuery().getStartIndex());
    assertEquals(4, argument1.getValue().getQuery().getPageSize());
    verify(mockSource2).query(argument2.capture());
    assertThat(mockQuery, not(argument2.getValue().getQuery()));
    // The modified query should have a start index of 1 and an end index of offset + pageSize -
    // 1
    assertEquals(1, argument2.getValue().getQuery().getStartIndex());
    assertEquals(4, argument2.getValue().getQuery().getPageSize());
    /**
         * Verify three results (page size) are returned. The sorted results returned by the sources
         * do NOT have the offset and page size taken into account, so the offset and page size are
         * applied to the sorted results in the OffsetResultHandler.
         *
         * Offset of 2 (start at result 2) and page size of 3 (end at result 4).
         */
    LOGGER.debug("mockSortedResult1: {}", mockSortedResult1);
    LOGGER.debug("mockSortedResult2: {}", mockSortedResult2);
    LOGGER.debug("mockSortedResult3: {}", mockSortedResult3);
    LOGGER.debug("mockSortedResult4: {}", mockSortedResult4);
    assertEquals(3, federatedResponse.getResults().size());
    assertEquals(mockSortedResult2, federatedResponse.getResults().get(0));
    assertEquals(mockSortedResult3, federatedResponse.getResults().get(1));
    assertEquals(mockSortedResult4, federatedResponse.getResults().get(2));
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("federated response result: {}", result);
    }
}

Example 62 with Source

use of ddf.catalog.source.Source in project ddf by codice.

the class FifoFederationStrategy method federate.

@Override
public QueryResponse federate(List<Source> sources, final QueryRequest queryRequest) {
    Query originalQuery = queryRequest.getQuery();
    int offset = originalQuery.getStartIndex();
    // limit offset to max value
    if (offset > this.maxStartIndex) {
        offset = this.maxStartIndex;
    }
    final int pageSize = originalQuery.getPageSize();
    QueryResponseImpl queryResponse = new QueryResponseImpl(queryRequest, null);
    Map<Source, Future<SourceResponse>> futures = new HashMap<Source, Future<SourceResponse>>();
    Query modifiedQuery = getModifiedQuery(originalQuery, sources.size(), offset, pageSize);
    QueryRequest modifiedQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), queryRequest.getSourceIds(), queryRequest.getProperties());
    executeSourceQueries(sources, futures, modifiedQueryRequest);
    int resultsToSkip = 0;
    if (offset > 1 && sources.size() > 1) {
        resultsToSkip = offset - 1;
    }
    queryExecutorService.submit(new FifoQueryMonitor(queryExecutorService, futures, queryResponse, modifiedQueryRequest.getQuery(), resultsToSkip));
    return executePostFederationPlugins(queryResponse);
}

Example 63 with Source

use of ddf.catalog.source.Source in project ddf by codice.

the class CachingFederationStrategyTest method testStartIndexGreaterThanMaxStartIndex.

@Test
public void testStartIndexGreaterThanMaxStartIndex() throws Exception {
    // startIndex and pageSize must be > 1
    Query mockQ = new QueryImpl(mock(NullFilterImpl.class), CachingFederationStrategy.DEFAULT_MAX_START_INDEX + 5, 1, mock(SortBy.class), true, LONG_TIMEOUT);
    QueryRequest fedQueryRequest = new QueryRequestImpl(mockQ, properties);
    List<Source> sources = new ArrayList<>();
    // Multiple sources needed for OffsetResultHandler to be used
    for (int i = 0; i < 2; i++) {
        Source mockSource = mock(Source.class);
        when(mockSource.getId()).thenReturn("mock source " + i);
        sources.add(mockSource);
    }
    QueryResponse federateResponse = strategy.federate(sources, fedQueryRequest);
    assertThat(federateResponse.getRequest().getQuery().getStartIndex(), is(CachingFederationStrategy.DEFAULT_MAX_START_INDEX + 5));
    // Modified offset is set to 1
    assertThat(requestArgumentCaptor.getValue().getQuery().getStartIndex(), is(1));
}

Example 64 with Source

use of ddf.catalog.source.Source in project ddf by codice.

the class CachingFederationStrategyTest method testFederateQueryCache.

@Test
public void testFederateQueryCache() throws Exception {
    properties.put(QUERY_MODE, CACHE_QUERY_MODE);
    QueryRequest fedQueryRequest = new QueryRequestImpl(mockQuery, properties);
    Source mockSource = mock(Source.class);
    when(mockSource.query(any(QueryRequest.class))).thenReturn(mockResponse);
    QueryResponse federateResponse = strategy.federate(Arrays.asList(mockSource), fedQueryRequest);
    assertThat(requestArgumentCaptor.getValue().getPropertyValue(QUERY_MODE), is(CACHE_QUERY_MODE));
    verify(cache).query(any(QueryRequest.class));
    verify(mockSource, times(0)).query(any(QueryRequest.class));
    verifyCacheNotUpdated();
    assertThat(federateResponse.getRequest().getQuery(), is(requestArgumentCaptor.getValue().getQuery()));
}

Example 65 with Source

use of ddf.catalog.source.Source in project ddf by codice.

the class CachingFederationStrategy method sourceFederate.

private QueryResponse sourceFederate(List<Source> sources, final QueryRequest queryRequest) {
    if (LOGGER.isDebugEnabled()) {
        for (Source source : sources) {
            if (source != null) {
                LOGGER.debug("source to query: {}", source.getId());
            }
        }
    }
    Query originalQuery = queryRequest.getQuery();
    int offset = originalQuery.getStartIndex();
    final int pageSize = originalQuery.getPageSize();
    // limit offset to max value
    if (offset > this.maxStartIndex) {
        offset = this.maxStartIndex;
    }
    final QueryResponseImpl queryResponseQueue = new QueryResponseImpl(queryRequest, null);
    Map<Future<SourceResponse>, QueryRequest> futures = new HashMap<>();
    Query modifiedQuery = getModifiedQuery(originalQuery, sources.size(), offset, pageSize);
    QueryRequest modifiedQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), queryRequest.getSourceIds(), queryRequest.getProperties());
    CompletionService<SourceResponse> queryCompletion = new ExecutorCompletionService<>(queryExecutorService);
    // Do NOT call source.isAvailable() when checking sources
    for (final Source source : sources) {
        if (source != null) {
            if (!futuresContainsSource(source, futures)) {
                LOGGER.debug("running query on source: {}", source.getId());
                QueryRequest sourceQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), Collections.singleton(source.getId()), new HashMap<>(queryRequest.getProperties()));
                try {
                    for (PreFederatedQueryPlugin service : preQuery) {
                        try {
                            sourceQueryRequest = service.process(source, sourceQueryRequest);
                        } catch (PluginExecutionException e) {
                            LOGGER.info("Error executing PreFederatedQueryPlugin", e);
                        }
                    }
                } catch (StopProcessingException e) {
                    LOGGER.info("Plugin stopped processing", e);
                }
                if (source instanceof CatalogProvider && SystemInfo.getSiteName().equals(source.getId())) {
                    // TODO RAP 12 Jul 16: DDF-2294 - Extract into a new PreFederatedQueryPlugin
                    sourceQueryRequest = validationQueryFactory.getQueryRequestWithValidationFilter(sourceQueryRequest, showErrors, showWarnings);
                }
                futures.put(queryCompletion.submit(new CallableSourceResponse(source, sourceQueryRequest)), sourceQueryRequest);
            } else {
                LOGGER.info("Duplicate source found with name {}. Ignoring second one.", source.getId());
            }
        }
    }
    QueryResponseImpl offsetResults = null;
    // OffsetResultHandler does.
    if (offset > 1 && sources.size() > 1) {
        offsetResults = new QueryResponseImpl(queryRequest, null);
        queryExecutorService.submit(new OffsetResultHandler(queryResponseQueue, offsetResults, pageSize, offset));
    }
    queryExecutorService.submit(sortedQueryMonitorFactory.createMonitor(queryCompletion, futures, queryResponseQueue, modifiedQueryRequest, postQuery));
    QueryResponse queryResponse;
    if (offset > 1 && sources.size() > 1) {
        queryResponse = offsetResults;
        LOGGER.debug("returning offsetResults");
    } else {
        queryResponse = queryResponseQueue;
        LOGGER.debug("returning returnResults: {}", queryResponse);
    }
    LOGGER.debug("returning Query Results: {}", queryResponse);
    return queryResponse;
}