use of ddf.catalog.impl.operations.QueryOperations in project ddf by codice.
the class FederationStrategyTest method testQueryTimeout.
/**
* Tests that the framework properly times out using the default federation strategy.
*/
@Test
public void testQueryTimeout() throws Exception {
long queryDelay = 100;
UuidGenerator uuidGenerator = mock(UuidGenerator.class);
when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
MockDelayProvider provider = new MockDelayProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<ContentType>(), true, new Date());
provider.setQueryDelayMillis(queryDelay);
// Mock register the provider in the container
SourcePollerRunner runner = new SourcePollerRunner();
SourcePoller poller = new SourcePoller(runner);
runner.bind(provider);
// Must have more than one thread or sleeps will block the monitor
SortedFederationStrategy fedStrategy = new SortedFederationStrategy(executor, new ArrayList<>(), new ArrayList<>());
FrameworkProperties props = new FrameworkProperties();
props.setCatalogProviders(Collections.singletonList(provider));
props.setFederationStrategy(fedStrategy);
props.setSourcePoller(poller);
props.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
props.setFilterBuilder(new GeotoolsFilterBuilder());
props.setDefaultAttributeValueRegistry(new DefaultAttributeValueRegistryImpl());
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(props.getMimeTypeToTransformerMapper(), uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(props, metacardFactory);
Historian historian = new Historian();
historian.setHistoryEnabled(false);
SourceOperations sourceOperations = new SourceOperations(props);
QueryOperations queryOperations = new QueryOperations(props, sourceOperations, opsSecurity, opsMetacard);
OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(props, sourceOperations);
CreateOperations createOperations = new CreateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
UpdateOperations updateOperations = new UpdateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
DeleteOperations deleteOperations = new DeleteOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard);
opsStorage.setHistorian(historian);
updateOperations.setHistorian(historian);
deleteOperations.setHistorian(historian);
deleteOperations.setOpsCatStoreSupport(opsCatStore);
CatalogFrameworkImpl framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, null, null, null);
sourceOperations.bind(provider);
List<Metacard> metacards = new ArrayList<Metacard>();
MetacardImpl newCard = new MetacardImpl();
newCard.setId(null);
metacards.add(newCard);
CreateResponse createResponse = null;
try {
try {
createResponse = framework.create(new CreateRequestImpl(metacards, null));
} catch (SourceUnavailableException e) {
long timeout = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
//this is a hack because the unit test is flaky and should be removed once a better test is possible
while (System.currentTimeMillis() < timeout) {
Thread.sleep(1000);
try {
createResponse = framework.create(new CreateRequestImpl(metacards, null));
break;
} catch (SourceUnavailableException e1) {
//ignore
}
}
}
if (createResponse == null) {
fail();
}
} catch (IngestException e) {
fail();
}
assertEquals(createResponse.getCreatedMetacards().size(), provider.size());
for (Metacard curCard : createResponse.getCreatedMetacards()) {
assertNotNull(curCard.getId());
}
QueryImpl query = new QueryImpl(filterFactory.equals(filterFactory.property(Metacard.ID), filterFactory.literal(createResponse.getCreatedMetacards().get(0).getId())));
query.setTimeoutMillis(SHORT_TIMEOUT);
query.setSortBy(new FilterFactoryImpl().sort(Result.RELEVANCE, SortOrder.ASCENDING));
QueryRequest fedQueryRequest = new QueryRequestImpl(query);
try {
QueryResponse response = framework.query(fedQueryRequest);
assertEquals("Timeout should happen before results return", 0, response.getHits());
} catch (UnsupportedQueryException e) {
fail();
} catch (FederationException e) {
LOGGER.error("Unexpected federation exception during test", e);
fail();
}
}
use of ddf.catalog.impl.operations.QueryOperations in project ddf by codice.
the class FanoutCatalogFrameworkTest method createCatalogFramework.
private CatalogFrameworkImpl createCatalogFramework(FrameworkProperties frameworkProperties) {
final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
final SourcePoller<Set<ContentType>> mockContentTypesSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).getContentTypes())).when(mockContentTypesSourcePoller).getCachedValueForSource(any(Source.class));
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mockContentTypesSourcePoller);
TransformOperations transformOperations = new TransformOperations(frameworkProperties);
QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, null, opsCatStore, opsStorage);
UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, null, opsCatStore, opsStorage);
DeleteOperations deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, null);
framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resourceOperations, sourceOperations, transformOperations);
framework.setId(NEW_SOURCE_ID);
framework.setFanoutEnabled(true);
return framework;
}
use of ddf.catalog.impl.operations.QueryOperations in project ddf by codice.
the class CatalogFrameworkImplTest method createFramework.
private CatalogFrameworkImpl createFramework(FrameworkProperties frameworkProperties) {
final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
final SourcePoller<Set<ContentType>> mockContentTypesSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).getContentTypes())).when(mockContentTypesSourcePoller).getCachedValueForSource(any(Source.class));
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
SourceOperations sourceOperations = new SourceOperations(frameworkProperties, mock(ActionRegistry.class), mockStatusSourcePoller, mockContentTypesSourcePoller);
QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
queryOperations.setSecurityLogger(mock(SecurityLogger.class));
queryOperations.setPermissions(new PermissionsImpl());
OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
DeleteOperations deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard);
ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
TransformOperations transformOperations = new TransformOperations(frameworkProperties);
Historian historian = new Historian();
historian.setHistoryEnabled(false);
updateOperations.setHistorian(historian);
deleteOperations.setHistorian(historian);
CatalogFrameworkImpl catalogFramework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resourceOperations, sourceOperations, transformOperations);
// Conditionally bind objects if framework properties are setup
if (CollectionUtils.isNotEmpty(frameworkProperties.getCatalogProviders())) {
sourceOperations.bind(provider);
}
if (CollectionUtils.isNotEmpty(frameworkProperties.getStorageProviders())) {
sourceOperations.bind(storageProvider);
}
return catalogFramework;
}
use of ddf.catalog.impl.operations.QueryOperations in project ddf by codice.
the class CatalogFrameworkImplTest method setup.
@Before
public void setup() throws StopProcessingException, PluginExecutionException, URISyntaxException, FederationException, IOException, CatalogTransformerException, InterruptedException {
System.setProperty("bad.files", "crossdomain.xml,clientaccesspolicy.xml,.htaccess,.htpasswd,hosts,passwd,group,resolv.conf,nfs.conf,ftpd.conf,ntp.conf,web.config,robots.txt");
System.setProperty("bad.file.extensions", ".exe,.jsp,.html,.js,.php,.phtml,.php3,.php4,.php5,.phps,.shtml,.jhtml,.pl,.py,.cgi,.msi,.com,.scr,.gadget,.application,.pif,.hta,.cpl,.msc,.jar,.kar,.bat,.cmd,.vb,.vbs,.vbe,.jse,.ws,.wsf,.wsc,.wsh,.ps1,.ps1xml,.ps2,.ps2xml,.psc1,.psc2,.msh,.msh1,.msh2,.mshxml,.msh1xml,.msh2xml,.scf,.lnk,.inf,.reg,.dll,.vxd,.cpl,.cfg,.config,.crt,.cert,.pem,.jks,.p12,.p7b,.key,.der,.csr,.jsb,.mhtml,.mht,.xhtml,.xht");
System.setProperty("bad.mime.types", "text/html,text/javascript,text/x-javascript,application/x-shellscript,text/scriptlet,application/x-msdownload,application/x-msmetafile");
System.setProperty("ignore.files", ".DS_Store,Thumbs.db");
// Setup
/*
* Prepare to capture the ResourceResponse argument passed into
* PostResourcePlugin.process(). We will verify that it contains a non-null ResourceRequest
* in the verification section of this test.
*/
argument = ArgumentCaptor.forClass(ResourceResponse.class);
Resource mockResource = mock(Resource.class);
mockResourceRequest = mock(ResourceRequest.class);
when(mockResourceRequest.getAttributeValue()).thenReturn(new URI("myURI"));
when(mockResourceRequest.getAttributeName()).thenReturn(new String("myName"));
mockResourceResponse = mock(ResourceResponse.class);
when(mockResourceResponse.getRequest()).thenReturn(mockResourceRequest);
when(mockResourceResponse.getResource()).thenReturn(mockResource);
mockPostResourcePlugin = mock(PostResourcePlugin.class);
/*
* We verify (see verification section of test) that PostResourcePlugin.process() receives a
* ResourceResponse with a non-null ResourceRequest. We assume that it works correctly and
* returns a ResourceResponse with a non-null ResourceRequest, so we return our
* mockResouceResponse that contains a non-null ResourceRequest.
*/
when(mockPostResourcePlugin.process(isA(ResourceResponse.class))).thenReturn(mockResourceResponse);
List<PostResourcePlugin> mockPostResourcePlugins = new ArrayList<PostResourcePlugin>();
mockPostResourcePlugins.add(mockPostResourcePlugin);
eventAdmin = new MockEventProcessor();
provider = new MockMemoryProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<>(), true, new Date());
storageProvider = new MockMemoryStorageProvider();
ArrayList<PostIngestPlugin> postIngestPlugins = new ArrayList<PostIngestPlugin>();
postIngestPlugins.add(eventAdmin);
mockFederationStrategy = mock(FederationStrategy.class);
Result mockFederationResult = mock(Result.class);
when(mockFederationResult.getMetacard()).thenReturn(new MetacardImpl());
QueryRequest mockQueryRequest = mock(QueryRequest.class);
Query mockQuery = mock(Query.class);
when(mockQuery.getTimeoutMillis()).thenReturn(1L);
when(mockQueryRequest.getQuery()).thenReturn(mockQuery);
QueryResponseImpl queryResponse = new QueryResponseImpl(mockQueryRequest, Collections.singletonList(mockFederationResult), 1);
when(mockFederationStrategy.federate(anyList(), any())).thenReturn(queryResponse);
federatedSources = createDefaultFederatedSourceList(true);
MimeTypeResolver mimeTypeResolver = mock(MimeTypeResolver.class);
MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
InputTransformer inputTransformer = mock(InputTransformer.class);
when(inputTransformer.transform(any(InputStream.class))).thenReturn(new MetacardImpl());
when(mimeTypeToTransformerMapper.findMatches(any(Class.class), any(MimeType.class))).thenReturn(Collections.singletonList(inputTransformer));
mockRemoteDeleteOperations = mock(RemoteDeleteOperations.class);
FrameworkProperties frameworkProperties = new FrameworkProperties();
frameworkProperties.setAccessPlugins(new ArrayList<>());
frameworkProperties.setPolicyPlugins(new ArrayList<>());
frameworkProperties.setCatalogProviders(Collections.singletonList((CatalogProvider) provider));
frameworkProperties.setPostResource(mockPostResourcePlugins);
frameworkProperties.setFederationStrategy(mockFederationStrategy);
frameworkProperties.setFilterBuilder(new GeotoolsFilterBuilder());
frameworkProperties.setPreIngest(new ArrayList<>());
frameworkProperties.setPostIngest(postIngestPlugins);
frameworkProperties.setPreQuery(new ArrayList<>());
frameworkProperties.setPostQuery(new ArrayList<>());
frameworkProperties.setPreResource(new ArrayList<>());
frameworkProperties.setPostResource(new ArrayList<>());
frameworkProperties.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
frameworkProperties.setMimeTypeMapper(new MimeTypeMapperImpl(Collections.singletonList(mimeTypeResolver)));
frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
List<FederatedSource> federatedSourceList = new ArrayList<>();
if (federatedSources != null) {
for (FederatedSource source : federatedSources) {
federatedSourceList.add(source);
}
}
frameworkProperties.setFederatedSources(federatedSourceList);
defaultAttributeValueRegistry = new DefaultAttributeValueRegistryImpl();
frameworkProperties.setDefaultAttributeValueRegistry(defaultAttributeValueRegistry);
attributeInjector = spy(new AttributeInjectorImpl(new AttributeRegistryImpl()));
frameworkProperties.setAttributeInjectors(Collections.singletonList(attributeInjector));
uuidGenerator = mock(UuidGenerator.class);
when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
sourceActionRegistry = mock(ActionRegistry.class);
when(sourceActionRegistry.list(any())).thenReturn(Collections.emptyList());
final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
final SourcePoller<Set<ContentType>> mockContentTypesSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).getContentTypes())).when(mockContentTypesSourcePoller).getCachedValueForSource(any(Source.class));
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(mimeTypeToTransformerMapper, uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mockContentTypesSourcePoller);
TransformOperations transformOperations = new TransformOperations(frameworkProperties);
Historian historian = new Historian();
historian.setHistoryEnabled(false);
QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
queryOperations.setSecurityLogger(mock(SecurityLogger.class));
queryOperations.setPermissions(new PermissionsImpl());
OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard);
ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity) {
@Override
protected ResourceInfo getResourceInfo(ResourceRequest resourceRequest, String site, boolean isEnterprise, StringBuilder federatedSite, Map<String, Serializable> requestProperties, boolean fanoutEnabled) throws ResourceNotSupportedException, ResourceNotFoundException {
URI uri = null;
Metacard metacard = new MetacardImpl();
try {
uri = new URI("myURI");
} catch (URISyntaxException e) {
}
return new ResourceInfo(metacard, uri);
}
};
updateOperations.setHistorian(historian);
deleteOperations.setHistorian(historian);
framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
// Conditionally bind objects if framework properties are setup
if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
sourceOperations.bind(provider);
}
sourceOperations.bind(storageProvider);
resourceFramework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
// Conditionally bind objects if framework properties are setup
if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
sourceOperations.bind(provider);
}
sourceOperations.bind(storageProvider);
ThreadContext.bind(mock(Subject.class));
}
use of ddf.catalog.impl.operations.QueryOperations in project ddf by codice.
the class FanoutCatalogFrameworkTest method testBlacklistedTagDeleteRequestFails.
@Test(expected = IngestException.class)
public void testBlacklistedTagDeleteRequestFails() throws Exception {
Metacard metacard = new MetacardImpl();
metacard.setAttribute(new AttributeImpl(Metacard.ID, "metacardId"));
metacard.setAttribute(new AttributeImpl(Metacard.TAGS, "blacklisted"));
CatalogProvider catalogProvider = mock(CatalogProvider.class);
doReturn(true).when(catalogProvider).isAvailable();
StorageProvider storageProvider = new MockMemoryStorageProvider();
FilterBuilder filterBuilder = new GeotoolsFilterBuilder();
QueryRequestImpl queryRequest = new QueryRequestImpl(mock(Query.class));
ResultImpl result = new ResultImpl(metacard);
List<Result> results = new ArrayList<>();
results.add(result);
QueryResponseImpl queryResponse = new QueryResponseImpl(queryRequest, results, 1);
FederationStrategy strategy = mock(FederationStrategy.class);
when(strategy.federate(anyList(), any())).thenReturn(queryResponse);
QueryResponsePostProcessor queryResponsePostProcessor = mock(QueryResponsePostProcessor.class);
doNothing().when(queryResponsePostProcessor).processResponse(any());
frameworkProperties.setCatalogProviders(Collections.singletonList(catalogProvider));
frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
frameworkProperties.setFilterBuilder(filterBuilder);
frameworkProperties.setFederationStrategy(strategy);
frameworkProperties.setQueryResponsePostProcessor(queryResponsePostProcessor);
OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mock(SourcePoller.class));
sourceOperations.bind(catalogProvider);
sourceOperations.bind(storageProvider);
TransformOperations transformOperations = new TransformOperations(frameworkProperties);
QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
DeleteOperations deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, null);
framework = new CatalogFrameworkImpl(null, null, deleteOperations, queryOperations, resourceOperations, sourceOperations, transformOperations);
framework.setId(NEW_SOURCE_ID);
framework.setFanoutEnabled(true);
framework.setFanoutTagBlacklist(Collections.singletonList("blacklisted"));
DeleteRequest request = new DeleteRequestImpl(metacard.getId());
framework.delete(request);
}
Aggregations