Examples with QueryRequest ddf.catalog.operation.QueryRequest used on opensource projects

Example 11 with QueryRequest

use of ddf.catalog.operation.QueryRequest in project ddf by codice.

the class FederationStrategyTest method testQueryTimeout.

/**
     * Tests that the framework properly times out using the default federation strategy.
     */
@Test
public void testQueryTimeout() throws Exception {
    long queryDelay = 100;
    UuidGenerator uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    MockDelayProvider provider = new MockDelayProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<ContentType>(), true, new Date());
    provider.setQueryDelayMillis(queryDelay);
    // Mock register the provider in the container
    SourcePollerRunner runner = new SourcePollerRunner();
    SourcePoller poller = new SourcePoller(runner);
    runner.bind(provider);
    // Must have more than one thread or sleeps will block the monitor
    SortedFederationStrategy fedStrategy = new SortedFederationStrategy(executor, new ArrayList<>(), new ArrayList<>());
    FrameworkProperties props = new FrameworkProperties();
    props.setCatalogProviders(Collections.singletonList(provider));
    props.setFederationStrategy(fedStrategy);
    props.setSourcePoller(poller);
    props.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
    props.setFilterBuilder(new GeotoolsFilterBuilder());
    props.setDefaultAttributeValueRegistry(new DefaultAttributeValueRegistryImpl());
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(props.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(props, metacardFactory);
    Historian historian = new Historian();
    historian.setHistoryEnabled(false);
    SourceOperations sourceOperations = new SourceOperations(props);
    QueryOperations queryOperations = new QueryOperations(props, sourceOperations, opsSecurity, opsMetacard);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(props, sourceOperations);
    CreateOperations createOperations = new CreateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    UpdateOperations updateOperations = new UpdateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    DeleteOperations deleteOperations = new DeleteOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard);
    opsStorage.setHistorian(historian);
    updateOperations.setHistorian(historian);
    deleteOperations.setHistorian(historian);
    deleteOperations.setOpsCatStoreSupport(opsCatStore);
    CatalogFrameworkImpl framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, null, null, null);
    sourceOperations.bind(provider);
    List<Metacard> metacards = new ArrayList<Metacard>();
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    metacards.add(newCard);
    CreateResponse createResponse = null;
    try {
        try {
            createResponse = framework.create(new CreateRequestImpl(metacards, null));
        } catch (SourceUnavailableException e) {
            long timeout = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
            //this is a hack because the unit test is flaky and should be removed once a better test is possible
            while (System.currentTimeMillis() < timeout) {
                Thread.sleep(1000);
                try {
                    createResponse = framework.create(new CreateRequestImpl(metacards, null));
                    break;
                } catch (SourceUnavailableException e1) {
                //ignore
                }
            }
        }
        if (createResponse == null) {
            fail();
        }
    } catch (IngestException e) {
        fail();
    }
    assertEquals(createResponse.getCreatedMetacards().size(), provider.size());
    for (Metacard curCard : createResponse.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
    }
    QueryImpl query = new QueryImpl(filterFactory.equals(filterFactory.property(Metacard.ID), filterFactory.literal(createResponse.getCreatedMetacards().get(0).getId())));
    query.setTimeoutMillis(SHORT_TIMEOUT);
    query.setSortBy(new FilterFactoryImpl().sort(Result.RELEVANCE, SortOrder.ASCENDING));
    QueryRequest fedQueryRequest = new QueryRequestImpl(query);
    try {
        QueryResponse response = framework.query(fedQueryRequest);
        assertEquals("Timeout should happen before results return", 0, response.getHits());
    } catch (UnsupportedQueryException e) {
        fail();
    } catch (FederationException e) {
        LOGGER.error("Unexpected federation exception during test", e);
        fail();
    }
}

Example 12 with QueryRequest

use of ddf.catalog.operation.QueryRequest in project ddf by codice.

the class FederationStrategyTest method testFederateOneSourceOffsetOnePageSizeTwo.

/**
     * Verify that the original query is used by the source.
     * <p>
     * No special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateOneSourceOffsetOnePageSizeTwo() throws Exception {
    LOGGER.debug("testFederate_OneSource_OffsetOne_PageSizeTwo()");
    // Offset of 1
    when(mockQuery.getStartIndex()).thenReturn(1);
    // Page size of 2
    when(mockQuery.getPageSize()).thenReturn(2);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    /**
         * When using the original query to query the source, the desired offset and page size are
         * used. So, the results returned by the source already have the offset and page size taken
         * into account.
         */
    Result mockResult1 = mock(Result.class);
    Result mockResult2 = mock(Result.class);
    SourceResponse mockSourceResponse = mock(SourceResponse.class);
    List<Result> results = Arrays.asList(mockResult1, mockResult2);
    when(mockSourceResponse.getResults()).thenReturn(results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSourceResponse);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.2 #######");
    // Only one source
    List<Source> sources = new ArrayList<Source>(1);
    sources.add(mockSource1);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Verification
    assertNotNull(federatedResponse);
    LOGGER.debug("Federated response result size: {}", federatedResponse.getResults().size());
    /**
         * Verify two results (page size) are returned. The results returned by the source already
         * have the offset and page size taken into account, so we can verify that the lists match.
         */
    assertEquals(2, federatedResponse.getResults().size());
    assertEquals(mockResult1, federatedResponse.getResults().get(0));
    assertEquals(mockResult2, federatedResponse.getResults().get(1));
    LOGGER.debug("mockResult1: {}", mockResult1);
    LOGGER.debug("mockResult2: {}", mockResult2);
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("result: {}", result);
    }
    // Check the responseProperties
    assertEquals("####### MOCK SOURCE 1.2 #######", ((List) federatedResponse.getPropertyValue(QueryResponse.SITE_LIST)).get(0));
    Map<String, Serializable> siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 1.2 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
}

Example 13 with QueryRequest

use of ddf.catalog.operation.QueryRequest in project ddf by codice.

the class FederationStrategyTest method testFederateTwoSourcesOffsetOnePageSizeThree.

/**
     * Verify that the original query passed is used by the sources.
     * <p>
     * No special results handling done by OffsetResultsHandler.
     */
@Test
public void testFederateTwoSourcesOffsetOnePageSizeThree() throws Exception {
    LOGGER.debug("testFederate_TwoSources_OffsetOne_PageSizeThree()");
    // Offset of 1
    when(mockQuery.getStartIndex()).thenReturn(1);
    // Page size of 3
    when(mockQuery.getPageSize()).thenReturn(3);
    QueryRequest queryRequest = mock(QueryRequest.class);
    when(queryRequest.getQuery()).thenReturn(mockQuery);
    /**
         * When using the original query to query the sources, the desired offset and page size are
         * used. So, the results returned by the sources already have the offset and page size taken
         * into account.
         */
    Result mockSource1Result1 = mock(Result.class);
    Mockito.when(mockSource1Result1.getRelevanceScore()).thenReturn(0.7);
    Result mockSource1Result2 = mock(Result.class);
    Mockito.when(mockSource1Result2.getRelevanceScore()).thenReturn(0.5);
    Result mockSource1Result3 = mock(Result.class);
    Mockito.when(mockSource1Result3.getRelevanceScore()).thenReturn(0.3);
    Result mockSource1Result4 = mock(Result.class);
    Mockito.when(mockSource1Result4.getRelevanceScore()).thenReturn(0.1);
    SourceResponse mockSource1Response = mock(SourceResponse.class);
    List<Result> mockSource1Results = Arrays.asList(mockSource1Result1, mockSource1Result2, mockSource1Result3, mockSource1Result4);
    when(mockSource1Response.getResults()).thenReturn(mockSource1Results);
    Source mockSource1 = mock(Source.class);
    when(mockSource1.query(any(QueryRequest.class))).thenReturn(mockSource1Response);
    when(mockSource1.getId()).thenReturn("####### MOCK SOURCE 1.4 #######");
    Result mockSource2Result1 = mock(Result.class);
    Mockito.when(mockSource2Result1.getRelevanceScore()).thenReturn(0.8);
    Result mockSource2Result2 = mock(Result.class);
    Mockito.when(mockSource2Result2.getRelevanceScore()).thenReturn(0.6);
    Result mockSource2Result3 = mock(Result.class);
    Mockito.when(mockSource2Result3.getRelevanceScore()).thenReturn(0.4);
    Result mockSource2Result4 = mock(Result.class);
    Mockito.when(mockSource2Result4.getRelevanceScore()).thenReturn(0.2);
    SourceResponse mockSource2Response = mock(SourceResponse.class);
    List<Result> mockSource2Results = Arrays.asList(mockSource2Result1, mockSource2Result2, mockSource2Result3, mockSource2Result4);
    when(mockSource2Response.getResults()).thenReturn(mockSource2Results);
    Source mockSource2 = mock(Source.class);
    when(mockSource2.query(any(QueryRequest.class))).thenReturn(mockSource2Response);
    when(mockSource2.getId()).thenReturn("####### MOCK SOURCE 2.4 #######");
    // Two sources
    List<Source> sources = new ArrayList<Source>(2);
    sources.add(mockSource1);
    sources.add(mockSource2);
    SortedFederationStrategy strategy = new SortedFederationStrategy(executor, new ArrayList<PreFederatedQueryPlugin>(), new ArrayList<PostFederatedQueryPlugin>());
    // Run Test
    QueryResponse federatedResponse = strategy.federate(sources, queryRequest);
    // Verification
    assertNotNull(federatedResponse);
    LOGGER.debug("Federated response result size: {}", federatedResponse.getResults().size());
    /**
         * Verify three results (page size) are returned. Since we are using mock Results, the
         * relevance score is 0.0, and the merged and sorted results of both sources is
         * mockSource2Result1, mockSource1Result1, mockSource2Result2, mockSource1Result2,
         * mockSource2Result3, mockSource1Result3, mockSource2Result4, mockSource1Result4. So, the
         * results are mockSource2Result1, mockSource1Result1, mockSource2Result2. No need to use
         * OffsetResultHander in this case.
         */
    assertEquals(3, federatedResponse.getResults().size());
    assertEquals(mockSource2Result1, federatedResponse.getResults().get(0));
    assertEquals(mockSource1Result1, federatedResponse.getResults().get(1));
    assertEquals(mockSource2Result2, federatedResponse.getResults().get(2));
    LOGGER.debug("mockSource2Result1: {}", mockSource2Result1);
    LOGGER.debug("mockSource1Result1: {}", mockSource1Result1);
    LOGGER.debug("mockSource2Result2: {}", mockSource2Result2);
    for (Result result : federatedResponse.getResults()) {
        LOGGER.debug("federated response result: {}", result);
    }
    // Check the responseProperties
    List<String> siteList = (List) federatedResponse.getPropertyValue(QueryResponse.SITE_LIST);
    assertTrue(siteList.contains("####### MOCK SOURCE 2.4 #######"));
    Map<String, Serializable> siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 2.4 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
    assertTrue(siteList.contains("####### MOCK SOURCE 2.4 #######"));
    siteProperties = (Map) federatedResponse.getPropertyValue("####### MOCK SOURCE 1.4 #######");
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_HITS));
    assertNotNull(siteProperties.get(QueryResponse.TOTAL_RESULTS_RETURNED));
}

Example 14 with QueryRequest

use of ddf.catalog.operation.QueryRequest in project ddf by codice.

the class MetacardsMigratable method export.

/**
     * Exports all the metacards currently stored in the catalog framework.
     * <p>
     * {@inheritDoc}
     */
@Override
@NotNull
public MigrationMetadata export(@NotNull Path exportPath) throws MigrationException {
    config.setExportPath(exportPath.resolve(this.getId()));
    fileWriter.createExportDirectory(config.getExportPath());
    Collection<MigrationWarning> warnings = new ArrayList<>();
    Map<String, Serializable> props = createMapWithNativeQueryMode();
    Filter dumpFilter = filterBuilder.attribute(Metacard.ANY_TEXT).is().like().text("*");
    QueryImpl exportQuery = new QueryImpl(dumpFilter);
    exportQuery.setPageSize(config.getExportQueryPageSize());
    exportQuery.setRequestsTotalResultsCount(false);
    QueryRequest exportQueryRequest = new QueryRequestImpl(exportQuery, props);
    try {
        executeQueryLoop(exportQuery, exportQueryRequest);
    } catch (Exception e) {
        LOGGER.info("Internal error occurred when exporting catalog: {}", e);
        throw new ExportMigrationException(DEFAULT_FAILURE_MESSAGE);
    } finally {
        cleanup();
    }
    return new MigrationMetadata(warnings);
}

Example 15 with QueryRequest

use of ddf.catalog.operation.QueryRequest in project ddf by codice.

the class CatalogMetricsTest method catalogTemporalQueryMetric.

@Test
public void catalogTemporalQueryMetric() throws Exception {
    Filter temporalFilter = filterBuilder.attribute(Metacard.ANY_DATE).before().date(new Date());
    QueryRequest query = new QueryRequestImpl(new QueryImpl(temporalFilter));
    underTest.process(query);
    assertThat(underTest.temporalQueries.getCount(), is(1L));
}