use of ddf.catalog.source.Source in project ddf by codice.
the class CachingFederationStrategy method sourceFederate.
private QueryResponse sourceFederate(List<Source> sources, final QueryRequest queryRequest) {
if (LOGGER.isDebugEnabled()) {
for (Source source : sources) {
if (source != null) {
LOGGER.debug("source to query: {}", source.getId());
}
}
}
Query originalQuery = queryRequest.getQuery();
int offset = originalQuery.getStartIndex();
final int pageSize = originalQuery.getPageSize();
// limit offset to max value
if (offset > this.maxStartIndex) {
offset = this.maxStartIndex;
}
final QueryResponseImpl queryResponseQueue = new QueryResponseImpl(queryRequest, null);
Map<Future<SourceResponse>, QueryRequest> futures = new HashMap<>();
Query modifiedQuery = getModifiedQuery(originalQuery, sources.size(), offset, pageSize);
QueryRequest modifiedQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), queryRequest.getSourceIds(), queryRequest.getProperties());
CompletionService<SourceResponse> queryCompletion = new ExecutorCompletionService<>(queryExecutorService);
// Do NOT call source.isAvailable() when checking sources
for (final Source source : sources) {
if (source != null) {
if (!futuresContainsSource(source, futures)) {
LOGGER.debug("running query on source: {}", source.getId());
QueryRequest sourceQueryRequest = new QueryRequestImpl(modifiedQuery, queryRequest.isEnterprise(), Collections.singleton(source.getId()), new HashMap<>(queryRequest.getProperties()));
try {
for (PreFederatedQueryPlugin service : preQuery) {
try {
sourceQueryRequest = service.process(source, sourceQueryRequest);
} catch (PluginExecutionException e) {
LOGGER.info("Error executing PreFederatedQueryPlugin", e);
}
}
} catch (StopProcessingException e) {
LOGGER.info("Plugin stopped processing", e);
}
if (source instanceof CatalogProvider && SystemInfo.getSiteName().equals(source.getId())) {
// TODO RAP 12 Jul 16: DDF-2294 - Extract into a new PreFederatedQueryPlugin
sourceQueryRequest = validationQueryFactory.getQueryRequestWithValidationFilter(sourceQueryRequest, showErrors, showWarnings);
}
futures.put(queryCompletion.submit(new CallableSourceResponse(source, sourceQueryRequest)), sourceQueryRequest);
} else {
LOGGER.info("Duplicate source found with name {}. Ignoring second one.", source.getId());
}
}
}
QueryResponseImpl offsetResults = null;
// OffsetResultHandler does.
if (offset > 1 && sources.size() > 1) {
offsetResults = new QueryResponseImpl(queryRequest, null);
queryExecutorService.submit(new OffsetResultHandler(queryResponseQueue, offsetResults, pageSize, offset));
}
queryExecutorService.submit(sortedQueryMonitorFactory.createMonitor(queryCompletion, futures, queryResponseQueue, modifiedQueryRequest, postQuery));
QueryResponse queryResponse;
if (offset > 1 && sources.size() > 1) {
queryResponse = offsetResults;
LOGGER.debug("returning offsetResults");
} else {
queryResponse = queryResponseQueue;
LOGGER.debug("returning returnResults: {}", queryResponse);
}
LOGGER.debug("returning Query Results: {}", queryResponse);
return queryResponse;
}
use of ddf.catalog.source.Source in project ddf by codice.
the class CachingFederationStrategyTest method testFederateQueryWithOffsetHandler.
@Test
public void testFederateQueryWithOffsetHandler() throws Exception {
properties.put(QUERY_MODE, NATIVE_QUERY_MODE);
// startIndex and pageSize must be > 1
Query mockQ = new QueryImpl(mock(NullFilterImpl.class), 2, 2, mock(SortBy.class), true, LONG_TIMEOUT);
QueryRequest fedQueryRequest = new QueryRequestImpl(mockQ, properties);
List<Source> sources = new ArrayList<>();
// Multiple sources needed for OffsetResultHandler to be used
for (int i = 0; i < 2; i++) {
Source mockSource = mock(Source.class);
when(mockSource.getId()).thenReturn("mock source " + i);
sources.add(mockSource);
}
strategy.federate(sources, fedQueryRequest);
// Verify the sources were queried
for (Source source : sources) {
verify(source).query(any(QueryRequest.class));
}
}
use of ddf.catalog.source.Source in project ddf by codice.
the class CatalogFrameworkImplTest method testGetResourceToTestSecondResourceReaderWithSameSchemeGetsCalledIfFirstDoesNotReturnAnything.
/**
* Tests that multiple ResourceReaders with the same scheme will be invoked if the first one did
* not return a Response.
*
* @throws Exception
*/
@Test
//CACHE
@Ignore
public void testGetResourceToTestSecondResourceReaderWithSameSchemeGetsCalledIfFirstDoesNotReturnAnything() throws Exception {
String localProviderName = "ddf";
final String EXPECTED = "result from mockResourceResponse2";
final String DDF = "ddf";
// Mock a Catalog Provider
CatalogProvider provider = mock(CatalogProvider.class);
when(provider.getId()).thenReturn(localProviderName);
when(provider.isAvailable(isA(SourceMonitor.class))).thenReturn(true);
when(provider.isAvailable()).thenReturn(true);
// Mock register the provider in the container
// Mock the source poller
SourcePoller mockPoller = mock(SourcePoller.class);
when(mockPoller.getCachedSource(isA(Source.class))).thenReturn(null);
// Create two ResourceReaders. The first should not return anything
// and the second should.
ResourceReader resourceReader1 = mock(ResourceReader.class);
ResourceReader resourceReader2 = mock(ResourceReader.class);
// Set the supported Schemes so that both ResourceReaders use
// the same scheme ("DAD")
Set<String> supportedSchemes = new HashSet<String>();
supportedSchemes.add("DAD");
when(resourceReader1.getSupportedSchemes()).thenReturn(supportedSchemes);
when(resourceReader2.getSupportedSchemes()).thenReturn(supportedSchemes);
List<ResourceReader> resourceReaders = new ArrayList<ResourceReader>();
resourceReaders.add(resourceReader1);
resourceReaders.add(resourceReader2);
// Set up the requests and responses. The first ResourceReader will return null
// and the second one will retrieve a value, showing that if more than one
// ResourceReader with the same scheme are used, they will be called until a
// response is returned
ResourceRequest mockResourceRequest = mock(ResourceRequest.class);
URI myURI = new URI("DAD", "host", "/path", "fragment");
when(mockResourceRequest.getAttributeValue()).thenReturn(myURI);
when(mockResourceRequest.getAttributeName()).thenReturn(new String(ResourceRequest.GET_RESOURCE_BY_PRODUCT_URI));
Result result = mock(Result.class);
Metacard metacard = mock(Metacard.class);
when(metacard.getResourceURI()).thenReturn(myURI);
when(result.getMetacard()).thenReturn(metacard);
List<Result> results = new ArrayList<Result>();
results.add(result);
QueryResponse queryResponse = mock(QueryResponse.class);
when(queryResponse.getResults()).thenReturn(results);
List<Source> federatedSources = new ArrayList<Source>();
FederationStrategy strategy = mock(FederationStrategy.class);
when(strategy.federate(isA(federatedSources.getClass()), isA(QueryRequest.class))).thenReturn(queryResponse);
ResourceResponse mockResourceResponse1 = mock(ResourceResponse.class);
when(mockResourceResponse1.getRequest()).thenReturn(mockResourceRequest);
when(mockResourceResponse1.getResource()).thenReturn(null);
when(resourceReader1.retrieveResource(any(URI.class), anyMap())).thenReturn(null);
Resource mockResource = mock(Resource.class);
when(mockResource.getName()).thenReturn(EXPECTED);
ResourceResponse mockResourceResponse2 = mock(ResourceResponse.class);
when(mockResourceResponse2.getResource()).thenReturn(mockResource);
when(resourceReader2.retrieveResource(any(URI.class), anyMap())).thenReturn(mockResourceResponse2);
FrameworkProperties frameworkProperties = new FrameworkProperties();
frameworkProperties.setSourcePoller(mockPoller);
frameworkProperties.setResourceReaders(resourceReaders);
frameworkProperties.setFederationStrategy(strategy);
frameworkProperties.setCatalogProviders(Collections.singletonList(provider));
SourceOperations sourceOps = new SourceOperations(frameworkProperties);
QueryOperations queryOps = new QueryOperations(frameworkProperties, sourceOps, null, null);
ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOps, null);
resOps.setId(DDF);
CatalogFrameworkImpl catalogFramework = new CatalogFrameworkImpl(null, null, null, null, resOps, null, null);
sourceOps.bind(provider);
ResourceResponse response = catalogFramework.getResource(mockResourceRequest, DDF);
// Verify that the Response is as expected
org.junit.Assert.assertEquals(EXPECTED, response.getResource().getName());
// Verify that resourceReader1 was called 1 time
// This line is equivalent to verify(resourceReader1,
// times(1)).retrieveResource(any(URI.class), anyMap());
verify(resourceReader1).retrieveResource(any(URI.class), anyMap());
}
use of ddf.catalog.source.Source in project ddf by codice.
the class CatalogFrameworkImplTest method testGetFederatedSourcesDuplicates.
@Test
public void testGetFederatedSourcesDuplicates() {
List<FederatedSource> federatedSources = createDefaultFederatedSourceList(true);
// Duplicate Site
FederatedSource siteC2 = new MockSource("C", "Site C2", "v1.0", "DDF", null, true, new Date());
federatedSources.add(siteC2);
// Expected Sites
List<FederatedSource> expectedSources = createDefaultFederatedSourceList(true);
// Mock register the federated sources in the container
// Mock the source poller
SourcePoller mockPoller = mock(SourcePoller.class);
when(mockPoller.getCachedSource(isA(Source.class))).thenReturn(null);
FrameworkProperties frameworkProperties = new FrameworkProperties();
frameworkProperties.setSourcePoller(mockPoller);
Map<String, FederatedSource> sources = new HashMap<>();
for (FederatedSource federatedSource : expectedSources) {
sources.put(federatedSource.getId(), federatedSource);
}
frameworkProperties.setFederatedSources(sources);
CatalogFrameworkImpl framework = createFramework(frameworkProperties);
// Returned Sites
SourceInfoRequest request = new SourceInfoRequestEnterprise(true);
SourceInfoResponse response = null;
try {
response = framework.getSourceInfo(request);
} catch (SourceUnavailableException e) {
LOGGER.debug("SourceUnavilable", e);
fail();
}
Set<SourceDescriptor> sourceDescriptors = response.getSourceInfo();
// should contain ONLY the original federated sites and the catalog framework's
// site info (even though it has no local catalog provider configured) - hence,
// the "+1"
assertEquals(expectedSources.size(), sourceDescriptors.size());
}
use of ddf.catalog.source.Source in project ddf by codice.
the class CatalogFrameworkImplTest method setup.
@Before
public void setup() throws StopProcessingException, PluginExecutionException, URISyntaxException, FederationException, IOException, CatalogTransformerException, InterruptedException {
System.setProperty("bad.files", "crossdomain.xml,clientaccesspolicy.xml,.htaccess,.htpasswd,hosts,passwd,group,resolv.conf,nfs.conf,ftpd.conf,ntp.conf,web.config,robots.txt");
System.setProperty("bad.file.extensions", ".exe,.jsp,.html,.js,.php,.phtml,.php3,.php4,.php5,.phps,.shtml,.jhtml,.pl,.py,.cgi,.msi,.com,.scr,.gadget,.application,.pif,.hta,.cpl,.msc,.jar,.kar,.bat,.cmd,.vb,.vbs,.vbe,.jse,.ws,.wsf,.wsc,.wsh,.ps1,.ps1xml,.ps2,.ps2xml,.psc1,.psc2,.msh,.msh1,.msh2,.mshxml,.msh1xml,.msh2xml,.scf,.lnk,.inf,.reg,.dll,.vxd,.cpl,.cfg,.config,.crt,.cert,.pem,.jks,.p12,.p7b,.key,.der,.csr,.jsb,.mhtml,.mht,.xhtml,.xht");
System.setProperty("bad.mime.types", "text/html,text/javascript,text/x-javascript,application/x-shellscript,text/scriptlet,application/x-msdownload,application/x-msmetafile");
// Setup
/*
* Prepare to capture the ResourceResponse argument passed into
* PostResourcePlugin.process(). We will verify that it contains a non-null ResourceRequest
* in the verification section of this test.
*/
argument = ArgumentCaptor.forClass(ResourceResponse.class);
Resource mockResource = mock(Resource.class);
mockResourceRequest = mock(ResourceRequest.class);
when(mockResourceRequest.getAttributeValue()).thenReturn(new URI("myURI"));
when(mockResourceRequest.getAttributeName()).thenReturn(new String("myName"));
mockResourceResponse = mock(ResourceResponse.class);
when(mockResourceResponse.getRequest()).thenReturn(mockResourceRequest);
when(mockResourceResponse.getResource()).thenReturn(mockResource);
mockPostResourcePlugin = mock(PostResourcePlugin.class);
/*
* We verify (see verification section of test) that PostResourcePlugin.process() receives a
* ResourceResponse with a non-null ResourceRequest. We assume that it works correctly and
* returns a ResourceResponse with a non-null ResourceRequest, so we return our
* mockResouceResponse that contains a non-null ResourceRequest.
*/
when(mockPostResourcePlugin.process(isA(ResourceResponse.class))).thenReturn(mockResourceResponse);
List<PostResourcePlugin> mockPostResourcePlugins = new ArrayList<PostResourcePlugin>();
mockPostResourcePlugins.add(mockPostResourcePlugin);
eventAdmin = new MockEventProcessor();
provider = new MockMemoryProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<>(), true, new Date());
storageProvider = new MockMemoryStorageProvider();
// Mock register the provider in the container
// Mock the source poller
SourcePoller mockPoller = mock(SourcePoller.class);
when(mockPoller.getCachedSource(isA(Source.class))).thenReturn(null);
ArrayList<PostIngestPlugin> postIngestPlugins = new ArrayList<PostIngestPlugin>();
postIngestPlugins.add(eventAdmin);
mockFederationStrategy = mock(FederationStrategy.class);
Result mockFederationResult = mock(Result.class);
when(mockFederationResult.getMetacard()).thenReturn(new MetacardImpl());
QueryRequest mockQueryRequest = mock(QueryRequest.class);
Query mockQuery = mock(Query.class);
when(mockQuery.getTimeoutMillis()).thenReturn(1L);
when(mockQueryRequest.getQuery()).thenReturn(mockQuery);
QueryResponseImpl queryResponse = new QueryResponseImpl(mockQueryRequest, Collections.singletonList(mockFederationResult), 1);
when(mockFederationStrategy.federate(anyList(), anyObject())).thenReturn(queryResponse);
federatedSources = createDefaultFederatedSourceList(true);
MimeTypeResolver mimeTypeResolver = mock(MimeTypeResolver.class);
MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
InputTransformer inputTransformer = mock(InputTransformer.class);
when(inputTransformer.transform(any(InputStream.class))).thenReturn(new MetacardImpl());
when(mimeTypeToTransformerMapper.findMatches(any(Class.class), any(MimeType.class))).thenReturn(Collections.singletonList(inputTransformer));
mockRemoteDeleteOperations = mock(RemoteDeleteOperations.class);
FrameworkProperties frameworkProperties = new FrameworkProperties();
frameworkProperties.setAccessPlugins(new ArrayList<>());
frameworkProperties.setPolicyPlugins(new ArrayList<>());
frameworkProperties.setSourcePoller(mockPoller);
frameworkProperties.setCatalogProviders(Collections.singletonList((CatalogProvider) provider));
frameworkProperties.setPostResource(mockPostResourcePlugins);
frameworkProperties.setFederationStrategy(mockFederationStrategy);
frameworkProperties.setFilterBuilder(new GeotoolsFilterBuilder());
frameworkProperties.setPreIngest(new ArrayList<>());
frameworkProperties.setPostIngest(postIngestPlugins);
frameworkProperties.setPreQuery(new ArrayList<>());
frameworkProperties.setPostQuery(new ArrayList<>());
frameworkProperties.setPreResource(new ArrayList<>());
frameworkProperties.setPostResource(new ArrayList<>());
frameworkProperties.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
frameworkProperties.setMimeTypeMapper(new MimeTypeMapperImpl(Collections.singletonList(mimeTypeResolver)));
frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
frameworkProperties.setValidationQueryFactory(new ValidationQueryFactory(new GeotoolsFilterAdapterImpl(), new GeotoolsFilterBuilder()));
Map<String, FederatedSource> federatedSourceMap = new HashMap<>();
if (federatedSources != null) {
for (FederatedSource source : federatedSources) {
federatedSourceMap.put(source.getId(), source);
}
}
SourcePollerRunner runner = new SourcePollerRunner();
SourcePoller poller = new SourcePoller(runner);
for (FederatedSource source : federatedSources) {
runner.bind(source);
}
runner.bind(provider);
int wait = 0;
while (wait < 5) {
for (FederatedSource source : federatedSources) {
CachedSource cachedSource = poller.getCachedSource(source);
if (cachedSource == null || !cachedSource.isAvailable()) {
Thread.sleep(100);
wait++;
break;
}
}
CachedSource cachedProvider = poller.getCachedSource(provider);
if (cachedProvider == null || !cachedProvider.isAvailable()) {
Thread.sleep(100);
}
wait++;
}
frameworkProperties.setSourcePoller(poller);
frameworkProperties.setFederatedSources(federatedSourceMap);
defaultAttributeValueRegistry = new DefaultAttributeValueRegistryImpl();
frameworkProperties.setDefaultAttributeValueRegistry(defaultAttributeValueRegistry);
attributeInjector = spy(new AttributeInjectorImpl(new AttributeRegistryImpl()));
frameworkProperties.setAttributeInjectors(Collections.singletonList(attributeInjector));
uuidGenerator = mock(UuidGenerator.class);
when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(mimeTypeToTransformerMapper, uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
SourceOperations sourceOperations = new SourceOperations(frameworkProperties);
TransformOperations transformOperations = new TransformOperations(frameworkProperties);
Historian historian = new Historian();
historian.setHistoryEnabled(false);
QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
opsStorage.setHistorian(historian);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard);
deleteOperations.setOpsCatStoreSupport(opsCatStore);
ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity) {
@Override
protected ResourceInfo getResourceInfo(ResourceRequest resourceRequest, String site, boolean isEnterprise, StringBuilder federatedSite, Map<String, Serializable> requestProperties, boolean fanoutEnabled) throws ResourceNotSupportedException, ResourceNotFoundException {
URI uri = null;
Metacard metacard = new MetacardImpl();
try {
uri = new URI("myURI");
} catch (URISyntaxException e) {
}
return new ResourceInfo(metacard, uri);
}
};
updateOperations.setHistorian(historian);
deleteOperations.setHistorian(historian);
framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
// Conditionally bind objects if framework properties are setup
if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
sourceOperations.bind(provider);
}
sourceOperations.bind(storageProvider);
resourceFramework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
// Conditionally bind objects if framework properties are setup
if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
sourceOperations.bind(provider);
}
sourceOperations.bind(storageProvider);
ThreadContext.bind(mock(Subject.class));
}
Aggregations