Examples with ReliableResource ddf.catalog.resource.data.ReliableResource used on opensource projects

Example 31 with ReliableResource

use of ddf.catalog.resource.data.ReliableResource in project ddf by codice.

the class ReliableResourceDownloadManagerTest method testStreamToClientExceptionDuringProductDownloadCachingEnabled.

/**
 * Tests that if exception with the FileBackedOutputStream being written to and concurrently read
 * by the client occurs during a product retrieval, then the product download to the client is
 * stopped, but the caching of the file continues.
 *
 * @throws Exception
 */
@Test
@Ignore
public // does not seem to detect this and continues to stream successfully to the client.
void testStreamToClientExceptionDuringProductDownloadCachingEnabled() throws Exception {
    mis = new MockInputStream(productInputFilename);
    Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
    resourceResponse = getMockResourceResponse();
    downloadMgr = new ReliableResourceDownloadManager(getDownloaderConfig(), downloadStatusInfo, Executors.newSingleThreadExecutor());
    // Use small chunk size so download takes long enough for client
    // to have time to simulate FileBackedOutputStream exception
    int chunkSize = 2;
    downloadMgr.setChunkSize(chunkSize);
    ResourceRetriever retriever = mock(ResourceRetriever.class);
    when(retriever.retrieveResource()).thenReturn(resourceResponse);
    ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
    ResourceResponse newResourceResponse = downloadMgr.download(resourceRequest, metacard, retriever);
    assertThat(newResourceResponse, is(notNullValue()));
    productInputStream = newResourceResponse.getResource().getInputStream();
    assertThat(productInputStream, is(instanceOf(ReliableResourceInputStream.class)));
    // On second chunk read by client it will close the download manager's cache file output stream
    // to simulate a cache file exception that should be detected by the ReliableResourceCallable
    executor = Executors.newCachedThreadPool();
    ProductDownloadClient productDownloadClient = new ProductDownloadClient(productInputStream, chunkSize);
    productDownloadClient.setSimulateFbosException(chunkSize, downloadMgr);
    future = executor.submit(productDownloadClient);
    ByteArrayOutputStream clientBytesRead = future.get();
    // Verify client did not receive entire product download
    assertTrue(clientBytesRead.size() < expectedFileSize);
    // Captures the ReliableResource object that should have been put in the ResourceCacheImpl's map
    verify(resourceCache, timeout(3000)).put(argument.capture());
    verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);
    cleanup();
}

Example 32 with ReliableResource

use of ddf.catalog.resource.data.ReliableResource in project ddf by codice.

the class ReliableResourceDownloadManagerTest method testStoreWithInputStreamRecoverableErrorCachingEnabled.

/**
 * Test that if an Exception is thrown while reading the product's InputStream that download and
 * caching is interrupted, retried and both successfully complete on the second attempt.
 *
 * @throws Exception
 */
@Test
@Ignore
public void testStoreWithInputStreamRecoverableErrorCachingEnabled() throws Exception {
    mis = new MockInputStream(productInputFilename);
    Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
    resourceResponse = getMockResourceResponse();
    ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(RetryType.INPUT_STREAM_IO_EXCEPTION);
    int chunkSize = 50;
    startDownload(true, chunkSize, false, metacard, retriever);
    ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);
    // Captures the ReliableResource object that should have been put in the ResourceCacheImpl's map
    ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
    verify(resourceCache).put(argument.capture());
    verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);
    // Verifies client read same contents as product input file
    verifyClientBytesRead(clientBytesRead);
    cleanup();
}

Example 33 with ReliableResource

use of ddf.catalog.resource.data.ReliableResource in project ddf by codice.

the class ReliableResourceDownloadManagerTest method testClientCancelProductDownloadCachingContinues.

/**
 * Tests that if user/client cancels a product retrieval that is in progress and actively being
 * cached, and the admin has configured caching to continue, that the caching continues and cached
 * file is placed in the cache map.
 *
 * @throws Exception
 */
@Test
@Ignore
public void testClientCancelProductDownloadCachingContinues() throws Exception {
    mis = new MockInputStream(productInputFilename);
    Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
    resourceResponse = getMockResourceResponse();
    ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(RetryType.CLIENT_CANCELS_DOWNLOAD);
    int chunkSize = 50;
    startDownload(true, chunkSize, true, metacard, retriever);
    // On second read of ReliableResourceInputStream, client will close the stream simulating a
    // cancel
    // of the product download
    clientRead(chunkSize, productInputStream, 2);
    // Captures the ReliableResource object that should have been put in the ResourceCacheImpl's map
    ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
    verify(resourceCache, timeout(3000)).put(argument.capture());
    verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);
    cleanup();
}

Example 34 with ReliableResource

use of ddf.catalog.resource.data.ReliableResource in project ddf by codice.

the class ReliableResourceDownloader method setupDownload.

public ResourceResponse setupDownload(Metacard metacard, DownloadStatusInfo downloadStatusInfo) {
    Resource resource = resourceResponse.getResource();
    MimeType mimeType = resource.getMimeType();
    String resourceName = resource.getName();
    fbos = new FileBackedOutputStream(DEFAULT_FILE_BACKED_OUTPUT_STREAM_THRESHOLD);
    countingFbos = new CountingOutputStream(fbos);
    streamReadByClient = new ReliableResourceInputStream(fbos, countingFbos, downloadState, downloadIdentifier, resourceResponse);
    this.metacard = metacard;
    // Create new ResourceResponse to return that will encapsulate the
    // ReliableResourceInputStream that will be read by the client simultaneously as the product
    // is cached to disk (if caching is enabled)
    ResourceImpl newResource = new ResourceImpl(streamReadByClient, mimeType, resourceName);
    resourceResponse = new ResourceResponseImpl(resourceResponse.getRequest(), resourceResponse.getProperties(), newResource);
    // Get handle to retrieved product's InputStream
    resourceInputStream = resource.getInputStream();
    eventListener.setDownloadMap(downloadIdentifier, resourceResponse);
    downloadStatusInfo.addDownloadInfo(downloadIdentifier, this, resourceResponse);
    if (downloaderConfig.isCacheEnabled()) {
        CacheKey keyMaker = null;
        String key = null;
        try {
            keyMaker = new CacheKey(metacard, resourceResponse.getRequest());
            key = keyMaker.generateKey();
        } catch (IllegalArgumentException e) {
            LOGGER.info("Cannot create cache key for resource with metacard ID = {}", metacard.getId());
            return resourceResponse;
        }
        if (!resourceCache.isPending(key)) {
            // Fully qualified path to cache file that will be written to.
            // Example:
            // <INSTALL-DIR>/data/product-cache/<source-id>-<metacard-id>
            // <INSTALL-DIR>/data/product-cache/ddf.distribution-abc123
            filePath = FilenameUtils.concat(resourceCache.getProductCacheDirectory(), key);
            if (filePath == null) {
                LOGGER.info("Unable to create cache for cache directory {} and key {} - no caching will be done.", resourceCache.getProductCacheDirectory(), key);
                return resourceResponse;
            }
            reliableResource = new ReliableResource(key, filePath, mimeType, resourceName, metacard);
            resourceCache.addPendingCacheEntry(reliableResource);
            try {
                fos = FileUtils.openOutputStream(new File(filePath));
                doCaching = true;
                this.downloadState.setCacheEnabled(true);
            } catch (IOException e) {
                LOGGER.info("Unable to open cache file {} - no caching will be done.", filePath);
            }
        } else {
            LOGGER.debug("Cache key {} is already pending caching", key);
        }
    }
    return resourceResponse;
}

Example 35 with ReliableResource

use of ddf.catalog.resource.data.ReliableResource in project ddf by codice.

the class MetacardResourceSizePlugin method process.

@Override
public QueryResponse process(QueryResponse input) throws PluginExecutionException, StopProcessingException {
    List<Result> results = input.getResults();
    for (Result result : results) {
        Metacard metacard = result.getMetacard();
        if (metacard != null) {
            // Can only search cache based on Metacard - no way to generate ResourceRequest with
            // any properties for use in generating the CacheKey
            final ResourceRequest resourceRequest = new ResourceRequestById(metacard.getId());
            CacheKey cacheKey;
            String key = null;
            ReliableResource cachedResource = null;
            try {
                cacheKey = new CacheKey(metacard, resourceRequest);
                ClassLoader tccl = Thread.currentThread().getContextClassLoader();
                key = cacheKey.generateKey();
                try {
                    Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
                    cachedResource = (ReliableResource) cache.getValid(key, metacard);
                } finally {
                    Thread.currentThread().setContextClassLoader(tccl);
                }
            } catch (IllegalArgumentException e) {
                LOGGER.debug("Unable to retrieve cached resource for metacard id = {}", metacard.getId());
            }
            if (cachedResource != null) {
                long resourceSize = cachedResource.getSize();
                if (resourceSize > 0 && cachedResource.hasProduct()) {
                    LOGGER.debug("Setting resourceSize = {} for metacard ID = {}", resourceSize, metacard.getId());
                    Attribute resourceSizeAttribute = new AttributeImpl(Metacard.RESOURCE_SIZE, String.valueOf(resourceSize));
                    metacard.setAttribute(resourceSizeAttribute);
                } else {
                    LOGGER.debug("resourceSize <= 0 for metacard ID = {}", metacard.getId());
                }
            } else {
                LOGGER.debug("No cached resource for cache key = {}", key);
            }
        }
    }
    return input;
}