Examples with IngestException ddf.catalog.source.IngestException used on opensource projects

Example 51 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class OperationsMetacardSupport method generateMetacardAndContentItems.

void generateMetacardAndContentItems(List<ContentItem> incomingContentItems, Map<String, Metacard> metacardMap, List<ContentItem> contentItems, Map<String, Map<String, Path>> tmpContentPaths) throws IngestException {
    for (ContentItem contentItem : incomingContentItems) {
        try {
            Path tmpPath = null;
            long size;
            try (InputStream inputStream = contentItem.getInputStream()) {
                if (inputStream == null) {
                    throw new IngestException("Could not copy bytes of content message.  Message was NULL.");
                }
                String sanitizedFilename = InputValidation.sanitizeFilename(contentItem.getFilename());
                tmpPath = Files.createTempFile(FilenameUtils.getBaseName(sanitizedFilename), FilenameUtils.getExtension(sanitizedFilename));
                Files.copy(inputStream, tmpPath, StandardCopyOption.REPLACE_EXISTING);
                size = Files.size(tmpPath);
                final String key = contentItem.getId();
                Map<String, Path> pathAndQualifiers = tmpContentPaths.get(key);
                if (pathAndQualifiers == null) {
                    pathAndQualifiers = new HashMap<>();
                    pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                    tmpContentPaths.put(key, pathAndQualifiers);
                } else {
                    pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                }
            } catch (IOException e) {
                if (tmpPath != null) {
                    FileUtils.deleteQuietly(tmpPath.toFile());
                }
                throw new IngestException("Could not copy bytes of content message.", e);
            }
            String mimeTypeRaw = contentItem.getMimeTypeRawData();
            mimeTypeRaw = guessMimeType(mimeTypeRaw, contentItem.getFilename(), tmpPath);
            if (!InputValidation.checkForClientSideVulnerableMimeType(mimeTypeRaw)) {
                throw new IngestException("Unsupported mime type.");
            }
            String fileName = updateFileExtension(mimeTypeRaw, contentItem.getFilename());
            Metacard metacard = metacardFactory.generateMetacard(mimeTypeRaw, contentItem.getId(), fileName, tmpPath);
            metacardMap.put(metacard.getId(), metacard);
            ContentItem generatedContentItem = new ContentItemImpl(metacard.getId(), StringUtils.isNotEmpty(contentItem.getQualifier()) ? contentItem.getQualifier() : "", com.google.common.io.Files.asByteSource(tmpPath.toFile()), mimeTypeRaw, fileName, size, metacard);
            contentItems.add(generatedContentItem);
        } catch (Exception e) {
            tmpContentPaths.values().stream().flatMap(id -> id.values().stream()).forEach(path -> FileUtils.deleteQuietly(path.toFile()));
            tmpContentPaths.clear();
            throw new IngestException("Could not create metacard.", e);
        }
    }
}

Example 52 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class RemoteDeleteOperations method doRemoteDelete.

private DeleteResponse doRemoteDelete(DeleteRequest deleteRequest) {
    HashSet<ProcessingDetails> exceptions = new HashSet<>();
    Map<String, Serializable> properties = new HashMap<>();
    List<CatalogStore> stores = opsCatStoreSupport.getCatalogStoresForRequest(deleteRequest, exceptions);
    List<Metacard> metacards = new ArrayList<>();
    for (CatalogStore store : stores) {
        try {
            if (!store.isAvailable()) {
                exceptions.add(new ProcessingDetailsImpl(store.getId(), null, "CatalogStore is not available"));
            } else {
                // TODO: 4/27/17 Address bug in DDF-2970 for overwriting deleted metacards
                DeleteResponse response = store.delete(deleteRequest);
                properties.put(store.getId(), new ArrayList<>(response.getDeletedMetacards()));
                metacards = response.getDeletedMetacards();
            }
        } catch (IngestException e) {
            INGEST_LOGGER.error("Error deleting metacards for CatalogStore {}", store.getId(), e);
            exceptions.add(new ProcessingDetailsImpl(store.getId(), e));
        }
    }
    return new DeleteResponseImpl(deleteRequest, properties, metacards, exceptions);
}

Example 53 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CatalogComponentFrameworkTest method testCreateWithIngestException.

@Test
public /**
     * Operation: CREATE
     * Body contains:  Metacard
     */
void testCreateWithIngestException() throws Exception {
    resetMocks();
    // Setup expectations to verify
    final MockEndpoint mockVerifierEndpoint = getMockEndpoint("mock:result");
    mockVerifierEndpoint.expectedMessageCount(1);
    final List<Metacard> metacards = new ArrayList<Metacard>();
    metacards.add(metacard1);
    // Mock catalog framework
    final CreateRequest createRequest = new CreateRequestImpl(metacards);
    final CreateResponse createResponse = new CreateResponseImpl(createRequest, new HashMap(), metacards);
    when(catalogFramework.create(any(CreateRequest.class))).thenThrow(new IngestException());
    // Exercise the route with a CREATE operation
    template.sendBodyAndHeader("direct:sampleInput", metacard1, "Operation", "CREATE");
    // Verify that the number of metacards in the exchange after the records
    // is identical to the input
    assertListSize(mockVerifierEndpoint.getExchanges(), 1);
    final Exchange exchange = mockVerifierEndpoint.getExchanges().get(0);
    final List<Metacard> cardsCreated = (List<Metacard>) exchange.getIn().getBody();
    assertListSize(cardsCreated, 0);
    mockVerifierEndpoint.assertIsSatisfied();
}

Example 54 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class DeleteOperations method performLocalDelete.

private DeleteResponse performLocalDelete(DeleteRequest deleteRequest, DeleteStorageRequest deleteStorageRequest) throws IngestException {
    if (!Requests.isLocal(deleteRequest)) {
        return null;
    }
    try {
        sourceOperations.getStorage().delete(deleteStorageRequest);
    } catch (StorageException e) {
        LOGGER.info("Unable to delete stored content items. Not removing stored metacards", e);
        throw new InternalIngestException("Unable to delete stored content items. Not removing stored metacards.", e);
    }
    DeleteResponse deleteResponse = sourceOperations.getCatalog().delete(deleteRequest);
    deleteResponse = injectAttributes(deleteResponse);
    try {
        historian.version(deleteResponse);
    } catch (SourceUnavailableException e) {
        LOGGER.debug("Could not version deleted item!", e);
        throw new IngestException("Could not version deleted Item!");
    }
    return deleteResponse;
}

Example 55 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class DeleteOperations method doDelete.

//
// Private helper methods
//
public DeleteResponse doDelete(DeleteRequest deleteRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    DeleteStorageRequest deleteStorageRequest = null;
    DeleteResponse deleteResponse = null;
    deleteRequest = queryOperations.setFlagsOnRequest(deleteRequest);
    deleteRequest = validateDeleteRequest(deleteRequest);
    deleteRequest = validateLocalSource(deleteRequest);
    try {
        deleteRequest = populateMetacards(deleteRequest, fanoutTagBlacklist);
        deleteRequest = preProcessPreAuthorizationPlugins(deleteRequest);
        deleteStorageRequest = new DeleteStorageRequestImpl(getDeleteMetacards(deleteRequest), deleteRequest.getProperties());
        deleteRequest = processPreDeletePolicyPlugins(deleteRequest);
        deleteRequest = processPreDeleteAccessPlugins(deleteRequest);
        deleteRequest = processPreIngestPlugins(deleteRequest);
        deleteRequest = validateDeleteRequest(deleteRequest);
        // Call the Provider delete method
        LOGGER.debug("Calling catalog.delete() with {} entries.", deleteRequest.getAttributeValues().size());
        deleteResponse = performLocalDelete(deleteRequest, deleteStorageRequest);
        deleteResponse = remoteDeleteOperations.performRemoteDelete(deleteRequest, deleteResponse);
        deleteResponse = postProcessPreAuthorizationPlugins(deleteResponse);
        deleteRequest = populateDeleteRequestPolicyMap(deleteRequest, deleteResponse);
        deleteResponse = processPostDeleteAccessPlugins(deleteResponse);
        // Post results to be available for pubsub
        deleteResponse = validateFixDeleteResponse(deleteResponse, deleteRequest);
    } catch (StopProcessingException see) {
        LOGGER.debug(PRE_INGEST_ERROR + see.getMessage(), see);
        throw new IngestException(PRE_INGEST_ERROR + see.getMessage());
    } catch (RuntimeException re) {
        LOGGER.info("Exception during runtime while performing delete", re);
        throw new InternalIngestException("Exception during runtime while performing delete");
    } finally {
        if (deleteStorageRequest != null) {
            try {
                sourceOperations.getStorage().commit(deleteStorageRequest);
            } catch (StorageException e) {
                LOGGER.info("Unable to remove stored content items.", e);
            }
        }
    }
    deleteResponse = doPostIngest(deleteResponse);
    return deleteResponse;
}