Search in sources :

Example 1 with ResourceNotSupportedException

use of ddf.catalog.resource.ResourceNotSupportedException in project ddf by codice.

the class ProcessingPostIngestPlugin method getProcessResource.

private ProcessResource getProcessResource(Metacard metacard, Subject subject) {
    LOGGER.trace("Attempting to retrieve process resource metacard with id \"{}\" and sourceId \"{}\".", metacard.getId(), metacard.getSourceId());
    ResourceRequest request = new ResourceRequestById(metacard.getId());
    if (subject == null) {
        LOGGER.debug("No available subject to fetch metacard resource. Returning null");
        return null;
    }
    return subject.execute(() -> {
        try {
            ResourceResponse response = catalogFramework.getResource(request, metacard.getSourceId());
            Resource resource = response.getResource();
            ProcessResource processResource = new ProcessResourceImpl(metacard.getId(), resource.getInputStream(), resource.getMimeTypeValue(), resource.getName(), resource.getSize(), false);
            return processResource;
        } catch (IOException | ResourceNotFoundException | ResourceNotSupportedException | RuntimeException e) {
            LOGGER.debug("Unable to get resource id:{}, sourceId:{}. Returning null", metacard.getId(), metacard.getSourceId(), e);
        }
        return null;
    });
}
Also used : ResourceResponse(ddf.catalog.operation.ResourceResponse) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) ResourceRequestById(ddf.catalog.operation.impl.ResourceRequestById) Resource(ddf.catalog.resource.Resource) ProcessResource(org.codice.ddf.catalog.async.data.api.internal.ProcessResource) ProcessResource(org.codice.ddf.catalog.async.data.api.internal.ProcessResource) ResourceRequest(ddf.catalog.operation.ResourceRequest) IOException(java.io.IOException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException) ProcessResourceImpl(org.codice.ddf.catalog.async.data.impl.ProcessResourceImpl)

Example 2 with ResourceNotSupportedException

use of ddf.catalog.resource.ResourceNotSupportedException in project ddf by codice.

the class ExportCommand method doContentExport.

private List<ExportItem> doContentExport(/*Mutable,IO*/
ZipFile zipFile, List<ExportItem> exportedItems) throws ZipException {
    List<ExportItem> contentItemsToExport = exportedItems.stream().filter(ei -> ei.getResourceUri() != null).filter(ei -> ei.getResourceUri().getScheme() != null).filter(ei -> ei.getResourceUri().getScheme().startsWith(ContentItem.CONTENT_SCHEME)).filter(ei -> !ei.getMetacardTag().equals("deleted")).filter(ei -> !ei.getMetacardTag().equals("revision") || ei.getResourceUri().getSchemeSpecificPart().equals(ei.getId())).filter(distinctByKey(ei -> ei.getResourceUri().getSchemeSpecificPart())).collect(Collectors.toList());
    List<ExportItem> exportedContentItems = new ArrayList<>();
    for (ExportItem contentItem : contentItemsToExport) {
        ResourceResponse resource;
        try {
            resource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(contentItem.getResourceUri()));
        } catch (IOException | ResourceNotSupportedException e) {
            throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
        } catch (ResourceNotFoundException e) {
            continue;
        }
        writeToZip(zipFile, contentItem, resource);
        exportedContentItems.add(contentItem);
        if (!contentItem.getMetacardTag().equals("revision")) {
            for (String derivedUri : contentItem.getDerivedUris()) {
                URI uri;
                try {
                    uri = new URI(derivedUri);
                } catch (URISyntaxException e) {
                    LOGGER.debug("Uri [{}] is not a valid URI. Derived content will not be included in export", derivedUri);
                    continue;
                }
                ResourceResponse derivedResource;
                try {
                    derivedResource = catalogFramework.getLocalResource(new ResourceRequestByProductUri(uri));
                } catch (IOException e) {
                    throw new CatalogCommandRuntimeException("Unable to retrieve resource for " + contentItem.getId(), e);
                } catch (ResourceNotFoundException | ResourceNotSupportedException e) {
                    LOGGER.warn("Could not retreive resource [{}]", uri, e);
                    console.printf("%sUnable to retrieve resource for export : %s%s%n", Ansi.ansi().fg(Ansi.Color.RED).toString(), uri, Ansi.ansi().reset().toString());
                    continue;
                }
                writeToZip(zipFile, contentItem, derivedResource);
            }
        }
    }
    return exportedContentItems;
}
Also used : Ansi(org.fusesource.jansi.Ansi) StringUtils(org.apache.commons.lang.StringUtils) DeleteStorageRequestImpl(ddf.catalog.content.operation.impl.DeleteStorageRequestImpl) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) BinaryContent(ddf.catalog.data.BinaryContent) LoggerFactory(org.slf4j.LoggerFactory) SecurityLogger(ddf.security.common.audit.SecurityLogger) Command(org.apache.karaf.shell.api.action.Command) MetacardTransformer(ddf.catalog.transform.MetacardTransformer) MetacardVersion(ddf.catalog.core.versioning.MetacardVersion) Map(java.util.Map) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) URI(java.net.URI) ParseException(java.text.ParseException) ZipFile(net.lingala.zip4j.core.ZipFile) TimeZone(java.util.TimeZone) Predicate(java.util.function.Predicate) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) DeletedMetacard(ddf.catalog.core.versioning.DeletedMetacard) Set(java.util.Set) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException) CatalogTransformerException(ddf.catalog.transform.CatalogTransformerException) Objects(java.util.Objects) ZipException(net.lingala.zip4j.exception.ZipException) StorageException(ddf.catalog.content.StorageException) List(java.util.List) FilenameUtils(org.apache.commons.io.FilenameUtils) ResourceResponse(ddf.catalog.operation.ResourceResponse) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) SimpleDateFormat(java.text.SimpleDateFormat) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) HashMap(java.util.HashMap) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Reference(org.apache.karaf.shell.api.action.lifecycle.Reference) SortBy(org.opengis.filter.sort.SortBy) ContentItem(ddf.catalog.content.data.ContentItem) Metacard(ddf.catalog.data.Metacard) CQLException(org.geotools.filter.text.cql2.CQLException) StorageProvider(ddf.catalog.content.StorageProvider) Result(ddf.catalog.data.Result) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Logger(org.slf4j.Logger) ZipParameters(net.lingala.zip4j.model.ZipParameters) QueryResultIterable(org.codice.ddf.commands.util.QueryResultIterable) IngestException(ddf.catalog.source.IngestException) ResourceRequestByProductUri(ddf.catalog.operation.impl.ResourceRequestByProductUri) IOException(java.io.IOException) IdAndUriMetacard(org.codice.ddf.commands.catalog.export.IdAndUriMetacard) File(java.io.File) JarSigner(org.codice.ddf.catalog.transformer.zip.JarSigner) TimeUnit(java.util.concurrent.TimeUnit) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) Paths(java.nio.file.Paths) Service(org.apache.karaf.shell.api.action.lifecycle.Service) Filter(org.opengis.filter.Filter) Option(org.apache.karaf.shell.api.action.Option) Collections(java.util.Collections) ExportItem(org.codice.ddf.commands.catalog.export.ExportItem) ArrayList(java.util.ArrayList) CatalogCommandRuntimeException(org.codice.ddf.commands.util.CatalogCommandRuntimeException) IOException(java.io.IOException) URISyntaxException(java.net.URISyntaxException) ResourceRequestByProductUri(ddf.catalog.operation.impl.ResourceRequestByProductUri) URI(java.net.URI) ResourceResponse(ddf.catalog.operation.ResourceResponse) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException)

Example 3 with ResourceNotSupportedException

use of ddf.catalog.resource.ResourceNotSupportedException in project ddf by codice.

the class ReliableResourceDownloader method retrieveResource.

private ReliableResourceCallable retrieveResource(long bytesRead) {
    ReliableResourceCallable reliableResourceCallable = null;
    try {
        LOGGER.debug("Attempting to re-retrieve resource, skipping {} bytes", bytesRead);
        // Re-fetch product from the Source after setting up values to indicate the number of
        // bytes to skip. This prevents the same bytes being read again and put in the
        // PipedOutputStream that the client is still reading from and in the file being cached
        // to. It also allows for range headers to be used in the request so that already read
        // bytes do not need to be re-retrieved.
        ResourceResponse resourceResponse = retriever.retrieveResource(bytesRead);
        LOGGER.debug("Name of re-retrieved resource = {}", resourceResponse.getResource().getName());
        resourceInputStream = resourceResponse.getResource().getInputStream();
        reliableResourceCallable = new ReliableResourceCallable(resourceInputStream, countingFbos, fos, downloaderConfig.getChunkSize(), lock);
        // So that Callable can account for bytes read in previous download attempt(s)
        reliableResourceCallable.setBytesRead(bytesRead);
    } catch (ResourceNotFoundException | ResourceNotSupportedException | IOException e) {
        LOGGER.info("Unable to re-retrieve product; cannot download product file {}", filePath);
    }
    return reliableResourceCallable;
}
Also used : ResourceResponse(ddf.catalog.operation.ResourceResponse) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) IOException(java.io.IOException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException)

Example 4 with ResourceNotSupportedException

use of ddf.catalog.resource.ResourceNotSupportedException in project ddf by codice.

the class ReliableResourceDownloadManagerTest method testDownloadResourceNotSupported.

@Test(expected = DownloadException.class)
public void testDownloadResourceNotSupported() throws Exception {
    Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
    resourceRequest = mock(ResourceRequest.class);
    ResourceRetriever retriever = mock(ResourceRetriever.class);
    when(retriever.retrieveResource()).thenThrow(new ResourceNotSupportedException());
    downloadMgr.download(resourceRequest, metacard, retriever);
}
Also used : Metacard(ddf.catalog.data.Metacard) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) ResourceRetriever(ddf.catalog.resourceretriever.ResourceRetriever) ResourceRequest(ddf.catalog.operation.ResourceRequest) Test(org.junit.Test)

Example 5 with ResourceNotSupportedException

use of ddf.catalog.resource.ResourceNotSupportedException in project ddf by codice.

the class ReliableResourceDownloadManager method download.

/**
     * @param resourceRequest the original @ResourceRequest to retrieve the resource
     * @param metacard        the @Metacard associated with the resource being downloaded
     * @param retriever       the @ResourceRetriever to be used to get the resource
     * @return the modified @ResourceResponse with the @ReliableResourceInputStream that the client
     * should read from
     * @throws DownloadException
     */
public ResourceResponse download(ResourceRequest resourceRequest, Metacard metacard, ResourceRetriever retriever) throws DownloadException {
    ResourceResponse resourceResponse = null;
    String downloadIdentifier = UUID.randomUUID().toString();
    if (metacard == null) {
        throw new DownloadException("Cannot download resource if metacard is null");
    } else if (StringUtils.isBlank(metacard.getId())) {
        throw new DownloadException("Metacard must have unique id.");
    } else if (retriever == null) {
        throw new DownloadException("Cannot download resource if retriever is null");
    } else if (resourceRequest == null) {
        throw new DownloadException("Cannot download resource if request is null");
    }
    if (downloaderConfig.isCacheEnabled()) {
        Resource cachedResource = downloaderConfig.getResourceCache().getValid(new CacheKey(metacard, resourceRequest).generateKey(), metacard);
        if (cachedResource != null) {
            resourceResponse = new ResourceResponseImpl(resourceRequest, resourceRequest.getProperties(), cachedResource);
            LOGGER.debug("Successfully retrieved product from cache for metacard ID = {}", metacard.getId());
        } else {
            LOGGER.debug("Unable to get resource from cache. Have to retrieve it from source");
        }
    }
    if (resourceResponse == null) {
        try {
            resourceResponse = retriever.retrieveResource();
        } catch (ResourceNotFoundException | ResourceNotSupportedException | IOException e) {
            throw new DownloadException("Cannot download resource", e);
        }
        resourceResponse.getProperties().put(Metacard.ID, metacard.getId());
        // Sources do not create ResourceResponses with the original ResourceRequest, hence
        // it is added here because it will be needed for caching
        resourceResponse = new ResourceResponseImpl(resourceRequest, resourceResponse.getProperties(), resourceResponse.getResource());
        resourceResponse = startDownload(downloadIdentifier, resourceResponse, retriever, metacard);
    }
    return resourceResponse;
}
Also used : ResourceResponse(ddf.catalog.operation.ResourceResponse) ResourceNotSupportedException(ddf.catalog.resource.ResourceNotSupportedException) Resource(ddf.catalog.resource.Resource) ResourceResponseImpl(ddf.catalog.operation.impl.ResourceResponseImpl) IOException(java.io.IOException) ResourceNotFoundException(ddf.catalog.resource.ResourceNotFoundException) CacheKey(ddf.catalog.cache.impl.CacheKey)

Aggregations

ResourceNotSupportedException (ddf.catalog.resource.ResourceNotSupportedException)14 ResourceNotFoundException (ddf.catalog.resource.ResourceNotFoundException)11 ResourceResponse (ddf.catalog.operation.ResourceResponse)10 IOException (java.io.IOException)10 ResourceRequest (ddf.catalog.operation.ResourceRequest)6 ResourceRequestById (ddf.catalog.operation.impl.ResourceRequestById)6 Metacard (ddf.catalog.data.Metacard)5 Resource (ddf.catalog.resource.Resource)5 QueryRequestImpl (ddf.catalog.operation.impl.QueryRequestImpl)3 URI (java.net.URI)3 HashMap (java.util.HashMap)3 Result (ddf.catalog.data.Result)2 QueryResponse (ddf.catalog.operation.QueryResponse)2 QueryImpl (ddf.catalog.operation.impl.QueryImpl)2 ResourceImpl (ddf.catalog.resource.impl.ResourceImpl)2 CatalogTransformerException (ddf.catalog.transform.CatalogTransformerException)2 Serializable (java.io.Serializable)2 URISyntaxException (java.net.URISyntaxException)2 Collections (java.util.Collections)2 MimeType (javax.activation.MimeType)2