Examples with SourcePoller ddf.catalog.util.impl.SourcePoller used on opensource projects

Example 1 with SourcePoller

use of ddf.catalog.util.impl.SourcePoller in project ddf by codice.

the class FederationStrategyTest method testQueryTimeout.

/**
     * Tests that the framework properly times out using the default federation strategy.
     */
@Test
public void testQueryTimeout() throws Exception {
    long queryDelay = 100;
    UuidGenerator uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    MockDelayProvider provider = new MockDelayProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<ContentType>(), true, new Date());
    provider.setQueryDelayMillis(queryDelay);
    // Mock register the provider in the container
    SourcePollerRunner runner = new SourcePollerRunner();
    SourcePoller poller = new SourcePoller(runner);
    runner.bind(provider);
    // Must have more than one thread or sleeps will block the monitor
    SortedFederationStrategy fedStrategy = new SortedFederationStrategy(executor, new ArrayList<>(), new ArrayList<>());
    FrameworkProperties props = new FrameworkProperties();
    props.setCatalogProviders(Collections.singletonList(provider));
    props.setFederationStrategy(fedStrategy);
    props.setSourcePoller(poller);
    props.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
    props.setFilterBuilder(new GeotoolsFilterBuilder());
    props.setDefaultAttributeValueRegistry(new DefaultAttributeValueRegistryImpl());
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(props.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(props, metacardFactory);
    Historian historian = new Historian();
    historian.setHistoryEnabled(false);
    SourceOperations sourceOperations = new SourceOperations(props);
    QueryOperations queryOperations = new QueryOperations(props, sourceOperations, opsSecurity, opsMetacard);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(props, sourceOperations);
    CreateOperations createOperations = new CreateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    UpdateOperations updateOperations = new UpdateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    DeleteOperations deleteOperations = new DeleteOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard);
    opsStorage.setHistorian(historian);
    updateOperations.setHistorian(historian);
    deleteOperations.setHistorian(historian);
    deleteOperations.setOpsCatStoreSupport(opsCatStore);
    CatalogFrameworkImpl framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, null, null, null);
    sourceOperations.bind(provider);
    List<Metacard> metacards = new ArrayList<Metacard>();
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    metacards.add(newCard);
    CreateResponse createResponse = null;
    try {
        try {
            createResponse = framework.create(new CreateRequestImpl(metacards, null));
        } catch (SourceUnavailableException e) {
            long timeout = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
            //this is a hack because the unit test is flaky and should be removed once a better test is possible
            while (System.currentTimeMillis() < timeout) {
                Thread.sleep(1000);
                try {
                    createResponse = framework.create(new CreateRequestImpl(metacards, null));
                    break;
                } catch (SourceUnavailableException e1) {
                //ignore
                }
            }
        }
        if (createResponse == null) {
            fail();
        }
    } catch (IngestException e) {
        fail();
    }
    assertEquals(createResponse.getCreatedMetacards().size(), provider.size());
    for (Metacard curCard : createResponse.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
    }
    QueryImpl query = new QueryImpl(filterFactory.equals(filterFactory.property(Metacard.ID), filterFactory.literal(createResponse.getCreatedMetacards().get(0).getId())));
    query.setTimeoutMillis(SHORT_TIMEOUT);
    query.setSortBy(new FilterFactoryImpl().sort(Result.RELEVANCE, SortOrder.ASCENDING));
    QueryRequest fedQueryRequest = new QueryRequestImpl(query);
    try {
        QueryResponse response = framework.query(fedQueryRequest);
        assertEquals("Timeout should happen before results return", 0, response.getHits());
    } catch (UnsupportedQueryException e) {
        fail();
    } catch (FederationException e) {
        LOGGER.error("Unexpected federation exception during test", e);
        fail();
    }
}

Example 2 with SourcePoller

use of ddf.catalog.util.impl.SourcePoller in project ddf by codice.

the class CatalogFrameworkImplTest method createDummyCatalogFramework.

private CatalogFramework createDummyCatalogFramework(CatalogProvider provider, StorageProvider storageProvider, BundleContext context, MockEventProcessor admin, boolean sourceAvailability) {
    // Mock register the provider in the container
    // Mock the source poller
    SourcePoller mockPoller = mock(SourcePoller.class);
    CachedSource mockSource = mock(CachedSource.class);
    when(mockSource.isAvailable()).thenReturn(sourceAvailability);
    when(mockPoller.getCachedSource(isA(Source.class))).thenReturn(mockSource);
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setCatalogProviders(Collections.singletonList(provider));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setSourcePoller(mockPoller);
    frameworkProperties.setBundleContext(context);
    frameworkProperties.setDefaultAttributeValueRegistry(defaultAttributeValueRegistry);
    return createFramework(frameworkProperties);
}

Example 3 with SourcePoller

use of ddf.catalog.util.impl.SourcePoller in project ddf by codice.

the class FanoutCatalogFrameworkTest method testNullContentTypesInGetSourceInfo.

/**
     * This test is to verify that an NPE will not be thrown if {@code source.getContentTypes}
     * returns null.
     *
     * @throws SourceUnavailableException
     */
@Test
public void testNullContentTypesInGetSourceInfo() throws SourceUnavailableException {
    SourcePollerRunner runner = new SourcePollerRunner();
    SourcePoller poller = new SourcePoller(runner);
    ArrayList<PostIngestPlugin> postIngestPlugins = new ArrayList<PostIngestPlugin>();
    SourceInfoRequest request = new SourceInfoRequestEnterprise(true);
    List<FederatedSource> fedSources = new ArrayList<FederatedSource>();
    FederatedSource mockFederatedSource = mock(FederatedSource.class);
    when(mockFederatedSource.isAvailable()).thenReturn(true);
    // Mockito would not accept Collections.emptySet() as the parameter for
    // thenReturn for mockFederatedSource.getContentTypes()
    when(mockFederatedSource.getContentTypes()).thenReturn(null);
    fedSources.add(mockFederatedSource);
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setSourcePoller(poller);
    frameworkProperties.setFederationStrategy(new MockFederationStrategy());
    frameworkProperties.setPostIngest(postIngestPlugins);
    Map<String, FederatedSource> sourceMap = new HashMap<>();
    for (FederatedSource federatedSource : fedSources) {
        sourceMap.put(federatedSource.getId(), federatedSource);
    }
    frameworkProperties.setFederatedSources(sourceMap);
    CatalogFrameworkImpl framework = createCatalogFramework(frameworkProperties);
    // Assert not null simply to prove that we returned an object.
    assertNotNull(framework.getSourceInfo(request));
}

Example 4 with SourcePoller

use of ddf.catalog.util.impl.SourcePoller in project ddf by codice.

the class CatalogFrameworkImplTest method testPreQueryStopExecution.

@Test(expected = FederationException.class)
public void testPreQueryStopExecution() throws UnsupportedQueryException, FederationException, SourceUnavailableException {
    SourcePoller poller = mock(SourcePoller.class);
    when(poller.getCachedSource(isA(Source.class))).thenReturn(null);
    MockMemoryProvider provider = new MockMemoryProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<ContentType>(), true, new Date());
    FederationStrategy federationStrategy = mock(FederationStrategy.class);
    QueryRequest request = mock(QueryRequest.class);
    when(request.getQuery()).thenReturn(mock(Query.class));
    PreQueryPlugin stopQueryPlugin = new PreQueryPlugin() {

        @Override
        public QueryRequest process(QueryRequest input) throws PluginExecutionException, StopProcessingException {
            throw new StopProcessingException("Testing that the framework will stop the query.");
        }
    };
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setSourcePoller(poller);
    frameworkProperties.setPreQuery(Arrays.asList(stopQueryPlugin));
    frameworkProperties.setFederationStrategy(federationStrategy);
    frameworkProperties.setCatalogProviders(Collections.singletonList(provider));
    CatalogFrameworkImpl framework = createFramework(frameworkProperties);
    framework.query(request);
}

Example 5 with SourcePoller

use of ddf.catalog.util.impl.SourcePoller in project ddf by codice.

the class CatalogFrameworkImplTest method testGetResourceToTestSecondResourceReaderWithSameSchemeGetsCalledIfFirstDoesNotReturnAnything.

/**
     * Tests that multiple ResourceReaders with the same scheme will be invoked if the first one did
     * not return a Response.
     *
     * @throws Exception
     */
@Test
//CACHE
@Ignore
public void testGetResourceToTestSecondResourceReaderWithSameSchemeGetsCalledIfFirstDoesNotReturnAnything() throws Exception {
    String localProviderName = "ddf";
    final String EXPECTED = "result from mockResourceResponse2";
    final String DDF = "ddf";
    // Mock a Catalog Provider
    CatalogProvider provider = mock(CatalogProvider.class);
    when(provider.getId()).thenReturn(localProviderName);
    when(provider.isAvailable(isA(SourceMonitor.class))).thenReturn(true);
    when(provider.isAvailable()).thenReturn(true);
    // Mock register the provider in the container
    // Mock the source poller
    SourcePoller mockPoller = mock(SourcePoller.class);
    when(mockPoller.getCachedSource(isA(Source.class))).thenReturn(null);
    // Create two ResourceReaders. The first should not return anything
    // and the second should.
    ResourceReader resourceReader1 = mock(ResourceReader.class);
    ResourceReader resourceReader2 = mock(ResourceReader.class);
    // Set the supported Schemes so that both ResourceReaders use
    // the same scheme ("DAD")
    Set<String> supportedSchemes = new HashSet<String>();
    supportedSchemes.add("DAD");
    when(resourceReader1.getSupportedSchemes()).thenReturn(supportedSchemes);
    when(resourceReader2.getSupportedSchemes()).thenReturn(supportedSchemes);
    List<ResourceReader> resourceReaders = new ArrayList<ResourceReader>();
    resourceReaders.add(resourceReader1);
    resourceReaders.add(resourceReader2);
    // Set up the requests and responses. The first ResourceReader will return null
    // and the second one will retrieve a value, showing that if more than one
    // ResourceReader with the same scheme are used, they will be called until a
    // response is returned
    ResourceRequest mockResourceRequest = mock(ResourceRequest.class);
    URI myURI = new URI("DAD", "host", "/path", "fragment");
    when(mockResourceRequest.getAttributeValue()).thenReturn(myURI);
    when(mockResourceRequest.getAttributeName()).thenReturn(new String(ResourceRequest.GET_RESOURCE_BY_PRODUCT_URI));
    Result result = mock(Result.class);
    Metacard metacard = mock(Metacard.class);
    when(metacard.getResourceURI()).thenReturn(myURI);
    when(result.getMetacard()).thenReturn(metacard);
    List<Result> results = new ArrayList<Result>();
    results.add(result);
    QueryResponse queryResponse = mock(QueryResponse.class);
    when(queryResponse.getResults()).thenReturn(results);
    List<Source> federatedSources = new ArrayList<Source>();
    FederationStrategy strategy = mock(FederationStrategy.class);
    when(strategy.federate(isA(federatedSources.getClass()), isA(QueryRequest.class))).thenReturn(queryResponse);
    ResourceResponse mockResourceResponse1 = mock(ResourceResponse.class);
    when(mockResourceResponse1.getRequest()).thenReturn(mockResourceRequest);
    when(mockResourceResponse1.getResource()).thenReturn(null);
    when(resourceReader1.retrieveResource(any(URI.class), anyMap())).thenReturn(null);
    Resource mockResource = mock(Resource.class);
    when(mockResource.getName()).thenReturn(EXPECTED);
    ResourceResponse mockResourceResponse2 = mock(ResourceResponse.class);
    when(mockResourceResponse2.getResource()).thenReturn(mockResource);
    when(resourceReader2.retrieveResource(any(URI.class), anyMap())).thenReturn(mockResourceResponse2);
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setSourcePoller(mockPoller);
    frameworkProperties.setResourceReaders(resourceReaders);
    frameworkProperties.setFederationStrategy(strategy);
    frameworkProperties.setCatalogProviders(Collections.singletonList(provider));
    SourceOperations sourceOps = new SourceOperations(frameworkProperties);
    QueryOperations queryOps = new QueryOperations(frameworkProperties, sourceOps, null, null);
    ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOps, null);
    resOps.setId(DDF);
    CatalogFrameworkImpl catalogFramework = new CatalogFrameworkImpl(null, null, null, null, resOps, null, null);
    sourceOps.bind(provider);
    ResourceResponse response = catalogFramework.getResource(mockResourceRequest, DDF);
    // Verify that the Response is as expected
    org.junit.Assert.assertEquals(EXPECTED, response.getResource().getName());
    // Verify that resourceReader1 was called 1 time
    // This line is equivalent to verify(resourceReader1,
    // times(1)).retrieveResource(any(URI.class), anyMap());
    verify(resourceReader1).retrieveResource(any(URI.class), anyMap());
}