Examples with ResourceResponse ddf.catalog.operation.ResourceResponse used on opensource projects

Example 71 with ResourceResponse

use of ddf.catalog.operation.ResourceResponse in project ddf by codice.

the class SeedCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (resourceLimit <= 0) {
        printErrorMessage("A limit of " + resourceLimit + " was supplied. The limit must be greater than 0.");
        return null;
    }
    final long start = System.currentTimeMillis();
    int resourceDownloads = 0;
    int downloadErrors = 0;
    int pageCount = 0;
    while (resourceDownloads < resourceLimit) {
        final QueryImpl query = new QueryImpl(getFilter());
        query.setPageSize(resourceLimit);
        query.setStartIndex(pageCount * resourceLimit + 1);
        ++pageCount;
        final QueryRequestImpl queryRequest;
        if (isNotEmpty(sources)) {
            queryRequest = new QueryRequestImpl(query, sources);
        } else {
            queryRequest = new QueryRequestImpl(query, true);
        }
        final QueryResponse queryResponse = catalogFramework.query(queryRequest);
        if (queryResponse.getResults().isEmpty()) {
            break;
        }
        final List<Entry<? extends ResourceRequest, String>> resourceRequests = queryResponse.getResults().stream().map(Result::getMetacard).filter(isNonCachedResource).map(metacard -> new SimpleEntry<>(new ResourceRequestById(metacard.getId()), metacard.getSourceId())).collect(toList());
        for (Entry<? extends ResourceRequest, String> requestAndSourceId : resourceRequests) {
            final ResourceRequest request = requestAndSourceId.getKey();
            try {
                ResourceResponse response = catalogFramework.getResource(request, requestAndSourceId.getValue());
                ++resourceDownloads;
                EXECUTOR.execute(new ResourceCloseHandler(response));
            } catch (IOException | ResourceNotFoundException | ResourceNotSupportedException e) {
                ++downloadErrors;
                LOGGER.debug("Could not download resource for metacard [id={}]", request.getAttributeValue(), e);
            }
            printProgressAndFlush(start, resourceLimit, resourceDownloads);
            if (resourceDownloads == resourceLimit) {
                break;
            }
        }
    }
    printProgressAndFlush(start, resourceDownloads, resourceDownloads);
    console.println();
    if (downloadErrors > 0) {
        printErrorMessage(downloadErrors + " resource download(s) had errors. Check the logs for details.");
    }
    printSuccessMessage("Done seeding. " + resourceDownloads + " resource download(s) started.");
    return null;
}

Example 72 with ResourceResponse

use of ddf.catalog.operation.ResourceResponse in project ddf by codice.

the class ResourceOperations method getResource.

// 
// 
// 
@SuppressWarnings("javadoc")
ResourceResponse getResource(ResourceRequest resourceRequest, boolean isEnterprise, String resourceSiteName, boolean fanoutEnabled) throws IOException, ResourceNotFoundException, ResourceNotSupportedException {
    ResourceResponse resourceResponse = null;
    ResourceRequest resourceReq = resourceRequest;
    ResourceRetriever retriever = null;
    if (fanoutEnabled) {
        isEnterprise = true;
    }
    if (resourceSiteName == null && !isEnterprise) {
        throw new ResourceNotFoundException("resourceSiteName cannot be null when obtaining resource.");
    }
    validateGetResourceRequest(resourceReq);
    try {
        resourceReq = preProcessPreAuthorizationPlugins(resourceReq);
        resourceReq = processPreResourcePolicyPlugins(resourceReq);
        resourceReq = processPreResourceAccessPlugins(resourceReq);
        resourceReq = processPreResourcePlugins(resourceReq);
        Map<String, Serializable> requestProperties = resourceReq.getProperties();
        LOGGER.debug("Attempting to get resource from siteName: {}", resourceSiteName);
        // At this point we pull out the properties and use them.
        Serializable sourceIdProperty = requestProperties.get(ResourceRequest.SOURCE_ID);
        final String namedSource = (sourceIdProperty != null) ? sourceIdProperty.toString() : resourceSiteName;
        Serializable enterpriseProperty = requestProperties.get(ResourceRequest.IS_ENTERPRISE);
        if (enterpriseProperty != null && Boolean.parseBoolean(enterpriseProperty.toString())) {
            isEnterprise = true;
        }
        // check if the resourceRequest has an ID only
        // If so, the metacard needs to be found and the Resource URI
        StringBuilder resolvedSourceIdHolder = new StringBuilder();
        ResourceInfo resourceInfo = getResourceInfo(resourceReq, namedSource, isEnterprise, resolvedSourceIdHolder, requestProperties, fanoutEnabled);
        if (resourceInfo == null) {
            throw new ResourceNotFoundException("Resource could not be found for the given attribute value: " + resourceReq.getAttributeValue());
        }
        final URI responseURI = resourceInfo.getResourceUri();
        final Metacard metacard = resourceInfo.getMetacard();
        final String resolvedSourceId = resolvedSourceIdHolder.toString();
        LOGGER.debug("resolvedSourceId = {}", resolvedSourceId);
        LOGGER.debug("ID = {}", getId());
        // since resolvedSourceId specifies what source the product
        // metacard resides on, we can just
        // change resourceSiteName to be that value, and then the
        // following if-else statements will
        // handle retrieving the product on the correct source
        final String resourceSourceName = (isEnterprise) ? resolvedSourceId : namedSource;
        // retrieve product from specified federated site if not in cache
        if (!resourceSourceName.equals(getId())) {
            LOGGER.debug("Searching federatedSource {} for resource.", resourceSourceName);
            LOGGER.debug("metacard for product found on source: {}", resolvedSourceId);
            final List<FederatedSource> sources = frameworkProperties.getFederatedSources().stream().filter(e -> e.getId().equals(resourceSourceName)).collect(Collectors.toList());
            if (!sources.isEmpty()) {
                if (sources.size() != 1) {
                    LOGGER.debug("Found multiple federatedSources for id: {}", resourceSourceName);
                }
                FederatedSource source = sources.get(0);
                LOGGER.debug("Adding federated site to federated query: {}", source.getId());
                LOGGER.debug("Retrieving product from remote source {}", source.getId());
                retriever = new RemoteResourceRetriever(source, responseURI, requestProperties);
            } else {
                LOGGER.debug("Could not find federatedSource: {}", resourceSourceName);
            }
        } else {
            LOGGER.debug("Retrieving product from local source {}", resourceSourceName);
            retriever = new LocalResourceRetriever(frameworkProperties.getResourceReaders(), responseURI, metacard, requestProperties);
        }
        try {
            resourceResponse = frameworkProperties.getDownloadManager().download(resourceRequest, metacard, retriever);
        } catch (DownloadException e) {
            LOGGER.info("Unable to download resource", e);
        }
        resourceResponse = putPropertiesInResponse(resourceRequest, resourceResponse);
        resourceResponse = validateFixGetResourceResponse(resourceResponse, resourceReq);
        resourceResponse = postProcessPreAuthorizationPlugins(resourceResponse, metacard);
        resourceResponse = processPostResourcePolicyPlugins(resourceResponse, metacard);
        resourceResponse = processPostResourceAccessPlugins(resourceResponse, metacard);
        resourceResponse = processPostResourcePlugins(resourceResponse);
        resourceResponse.getProperties().put(Constants.METACARD_PROPERTY, metacard);
    } catch (DataUsageLimitExceededException e) {
        LOGGER.info("RuntimeException caused by: ", e);
        throw e;
    } catch (RuntimeException e) {
        LOGGER.info("RuntimeException caused by: ", e);
        throw new ResourceNotFoundException("Unable to find resource");
    } catch (StopProcessingException e) {
        LOGGER.info("Resource not supported", e);
        throw new ResourceNotSupportedException(FAILED_BY_GET_RESOURCE_PLUGIN + e.getMessage());
    }
    return resourceResponse;
}

Example 73 with ResourceResponse

use of ddf.catalog.operation.ResourceResponse in project ddf by codice.

the class CatalogMetricsTest method catalogPostResourceLatencyMetric.

@Test
public void catalogPostResourceLatencyMetric() throws Exception {
    Iterable<Tag> tags = Tags.of("successful", "true");
    ResourceRequest request = mock(ResourceRequest.class);
    ResourceResponse response = mock(ResourceResponse.class);
    when(response.getRequest()).thenReturn(request);
    when(request.getPropertyValue("metrics.catalog.operation.start")).thenReturn(System.currentTimeMillis() - 1000);
    catalogMetrics.process(response);
    assertThat(meterRegistry.summary("ddf.catalog.resource.latency", tags).count(), is(1L));
    assertThat(meterRegistry.summary("ddf.catalog.resource.latency", tags).max(), greaterThanOrEqualTo(1000.0));
}

Example 74 with ResourceResponse

use of ddf.catalog.operation.ResourceResponse in project ddf by codice.

the class ResourceMetacardTransformerTest method getResourceResponse.

private ResourceResponse getResourceResponse(Resource resource) {
    ResourceResponse resourceResponse = mock(ResourceResponse.class);
    when(resourceResponse.getResource()).thenReturn(resource);
    return resourceResponse;
}

Example 75 with ResourceResponse

use of ddf.catalog.operation.ResourceResponse in project ddf by codice.

the class CatalogFrameworkImplTest method setup.

@Before
public void setup() throws StopProcessingException, PluginExecutionException, URISyntaxException, FederationException, IOException, CatalogTransformerException, InterruptedException {
    System.setProperty("bad.files", "crossdomain.xml,clientaccesspolicy.xml,.htaccess,.htpasswd,hosts,passwd,group,resolv.conf,nfs.conf,ftpd.conf,ntp.conf,web.config,robots.txt");
    System.setProperty("bad.file.extensions", ".exe,.jsp,.html,.js,.php,.phtml,.php3,.php4,.php5,.phps,.shtml,.jhtml,.pl,.py,.cgi,.msi,.com,.scr,.gadget,.application,.pif,.hta,.cpl,.msc,.jar,.kar,.bat,.cmd,.vb,.vbs,.vbe,.jse,.ws,.wsf,.wsc,.wsh,.ps1,.ps1xml,.ps2,.ps2xml,.psc1,.psc2,.msh,.msh1,.msh2,.mshxml,.msh1xml,.msh2xml,.scf,.lnk,.inf,.reg,.dll,.vxd,.cpl,.cfg,.config,.crt,.cert,.pem,.jks,.p12,.p7b,.key,.der,.csr,.jsb,.mhtml,.mht,.xhtml,.xht");
    System.setProperty("bad.mime.types", "text/html,text/javascript,text/x-javascript,application/x-shellscript,text/scriptlet,application/x-msdownload,application/x-msmetafile");
    System.setProperty("ignore.files", ".DS_Store,Thumbs.db");
    // Setup
    /*
     * Prepare to capture the ResourceResponse argument passed into
     * PostResourcePlugin.process(). We will verify that it contains a non-null ResourceRequest
     * in the verification section of this test.
     */
    argument = ArgumentCaptor.forClass(ResourceResponse.class);
    Resource mockResource = mock(Resource.class);
    mockResourceRequest = mock(ResourceRequest.class);
    when(mockResourceRequest.getAttributeValue()).thenReturn(new URI("myURI"));
    when(mockResourceRequest.getAttributeName()).thenReturn(new String("myName"));
    mockResourceResponse = mock(ResourceResponse.class);
    when(mockResourceResponse.getRequest()).thenReturn(mockResourceRequest);
    when(mockResourceResponse.getResource()).thenReturn(mockResource);
    mockPostResourcePlugin = mock(PostResourcePlugin.class);
    /*
     * We verify (see verification section of test) that PostResourcePlugin.process() receives a
     * ResourceResponse with a non-null ResourceRequest. We assume that it works correctly and
     * returns a ResourceResponse with a non-null ResourceRequest, so we return our
     * mockResouceResponse that contains a non-null ResourceRequest.
     */
    when(mockPostResourcePlugin.process(isA(ResourceResponse.class))).thenReturn(mockResourceResponse);
    List<PostResourcePlugin> mockPostResourcePlugins = new ArrayList<PostResourcePlugin>();
    mockPostResourcePlugins.add(mockPostResourcePlugin);
    eventAdmin = new MockEventProcessor();
    provider = new MockMemoryProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<>(), true, new Date());
    storageProvider = new MockMemoryStorageProvider();
    ArrayList<PostIngestPlugin> postIngestPlugins = new ArrayList<PostIngestPlugin>();
    postIngestPlugins.add(eventAdmin);
    mockFederationStrategy = mock(FederationStrategy.class);
    Result mockFederationResult = mock(Result.class);
    when(mockFederationResult.getMetacard()).thenReturn(new MetacardImpl());
    QueryRequest mockQueryRequest = mock(QueryRequest.class);
    Query mockQuery = mock(Query.class);
    when(mockQuery.getTimeoutMillis()).thenReturn(1L);
    when(mockQueryRequest.getQuery()).thenReturn(mockQuery);
    QueryResponseImpl queryResponse = new QueryResponseImpl(mockQueryRequest, Collections.singletonList(mockFederationResult), 1);
    when(mockFederationStrategy.federate(anyList(), any())).thenReturn(queryResponse);
    federatedSources = createDefaultFederatedSourceList(true);
    MimeTypeResolver mimeTypeResolver = mock(MimeTypeResolver.class);
    MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
    InputTransformer inputTransformer = mock(InputTransformer.class);
    when(inputTransformer.transform(any(InputStream.class))).thenReturn(new MetacardImpl());
    when(mimeTypeToTransformerMapper.findMatches(any(Class.class), any(MimeType.class))).thenReturn(Collections.singletonList(inputTransformer));
    mockRemoteDeleteOperations = mock(RemoteDeleteOperations.class);
    FrameworkProperties frameworkProperties = new FrameworkProperties();
    frameworkProperties.setAccessPlugins(new ArrayList<>());
    frameworkProperties.setPolicyPlugins(new ArrayList<>());
    frameworkProperties.setCatalogProviders(Collections.singletonList((CatalogProvider) provider));
    frameworkProperties.setPostResource(mockPostResourcePlugins);
    frameworkProperties.setFederationStrategy(mockFederationStrategy);
    frameworkProperties.setFilterBuilder(new GeotoolsFilterBuilder());
    frameworkProperties.setPreIngest(new ArrayList<>());
    frameworkProperties.setPostIngest(postIngestPlugins);
    frameworkProperties.setPreQuery(new ArrayList<>());
    frameworkProperties.setPostQuery(new ArrayList<>());
    frameworkProperties.setPreResource(new ArrayList<>());
    frameworkProperties.setPostResource(new ArrayList<>());
    frameworkProperties.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setMimeTypeMapper(new MimeTypeMapperImpl(Collections.singletonList(mimeTypeResolver)));
    frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
    List<FederatedSource> federatedSourceList = new ArrayList<>();
    if (federatedSources != null) {
        for (FederatedSource source : federatedSources) {
            federatedSourceList.add(source);
        }
    }
    frameworkProperties.setFederatedSources(federatedSourceList);
    defaultAttributeValueRegistry = new DefaultAttributeValueRegistryImpl();
    frameworkProperties.setDefaultAttributeValueRegistry(defaultAttributeValueRegistry);
    attributeInjector = spy(new AttributeInjectorImpl(new AttributeRegistryImpl()));
    frameworkProperties.setAttributeInjectors(Collections.singletonList(attributeInjector));
    uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    sourceActionRegistry = mock(ActionRegistry.class);
    when(sourceActionRegistry.list(any())).thenReturn(Collections.emptyList());
    final SourcePoller<SourceStatus> mockStatusSourcePoller = mock(SourcePoller.class);
    doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).isAvailable() ? SourceStatus.AVAILABLE : SourceStatus.UNAVAILABLE)).when(mockStatusSourcePoller).getCachedValueForSource(any(Source.class));
    final SourcePoller<Set<ContentType>> mockContentTypesSourcePoller = mock(SourcePoller.class);
    doAnswer(invocationOnMock -> Optional.of(((Source) invocationOnMock.getArguments()[0]).getContentTypes())).when(mockContentTypesSourcePoller).getCachedValueForSource(any(Source.class));
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(mimeTypeToTransformerMapper, uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    SourceOperations sourceOperations = new SourceOperations(frameworkProperties, sourceActionRegistry, mockStatusSourcePoller, mockContentTypesSourcePoller);
    TransformOperations transformOperations = new TransformOperations(frameworkProperties);
    Historian historian = new Historian();
    historian.setHistoryEnabled(false);
    QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
    queryOperations.setSecurityLogger(mock(SecurityLogger.class));
    queryOperations.setPermissions(new PermissionsImpl());
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
    CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    UpdateOperations updateOperations = new UpdateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    deleteOperations = new DeleteOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacard);
    ResourceOperations resOps = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity) {

        @Override
        protected ResourceInfo getResourceInfo(ResourceRequest resourceRequest, String site, boolean isEnterprise, StringBuilder federatedSite, Map<String, Serializable> requestProperties, boolean fanoutEnabled) throws ResourceNotSupportedException, ResourceNotFoundException {
            URI uri = null;
            Metacard metacard = new MetacardImpl();
            try {
                uri = new URI("myURI");
            } catch (URISyntaxException e) {
            }
            return new ResourceInfo(metacard, uri);
        }
    };
    updateOperations.setHistorian(historian);
    deleteOperations.setHistorian(historian);
    framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
    // Conditionally bind objects if framework properties are setup
    if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
        sourceOperations.bind(provider);
    }
    sourceOperations.bind(storageProvider);
    resourceFramework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, resOps, sourceOperations, transformOperations);
    // Conditionally bind objects if framework properties are setup
    if (!CollectionUtils.isEmpty(frameworkProperties.getCatalogProviders())) {
        sourceOperations.bind(provider);
    }
    sourceOperations.bind(storageProvider);
    ThreadContext.bind(mock(Subject.class));
}