Examples with IngestException ddf.catalog.source.IngestException used on opensource projects

Example 26 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class UpdateOperations method populateMetacards.

private UpdateRequest populateMetacards(UpdateRequest updateRequest) throws IngestException {
    final String attributeName = updateRequest.getAttributeName();
    QueryRequestImpl queryRequest = createQueryRequest(updateRequest);
    QueryResponse query;
    try {
        query = queryOperations.doQuery(queryRequest, frameworkProperties.getFederationStrategy());
    } catch (FederationException e) {
        LOGGER.debug("Unable to complete query for updated metacards.", e);
        throw new IngestException("Exception during runtime while performing update");
    }
    if (!foundAllUpdateRequestMetacards(updateRequest, query)) {
        logFailedQueryInfo(updateRequest, query);
        throw new IngestException("Could not find all metacards specified in request");
    }
    updateRequest = rewriteRequestToAvoidHistoryConflicts(updateRequest, query);
    HashMap<String, Metacard> metacardMap = new HashMap<>(query.getResults().stream().map(Result::getMetacard).collect(Collectors.toMap(metacard -> getAttributeStringValue(metacard, attributeName), Function.identity())));
    updateRequest.getProperties().put(Constants.ATTRIBUTE_UPDATE_MAP_KEY, metacardMap);
    updateRequest.getProperties().put(Constants.OPERATION_TRANSACTION_KEY, new OperationTransactionImpl(OperationTransaction.OperationType.UPDATE, metacardMap.values()));
    return updateRequest;
}

Example 27 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class UpdateOperations method update.

public UpdateResponse update(UpdateStorageRequest streamUpdateRequest) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamUpdateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    UpdateResponse updateResponse = null;
    UpdateStorageRequest updateStorageRequest = null;
    UpdateStorageResponse updateStorageResponse = null;
    streamUpdateRequest = opsStorageSupport.prepareStorageRequest(streamUpdateRequest, streamUpdateRequest::getContentItems);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamUpdateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    streamUpdateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    streamUpdateRequest = applyAttributeOverrides(streamUpdateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            updateStorageRequest = new UpdateStorageRequestImpl(contentItems, streamUpdateRequest.getId(), streamUpdateRequest.getProperties());
            updateStorageRequest = processPreUpdateStoragePlugins(updateStorageRequest);
            try {
                updateStorageResponse = sourceOperations.getStorage().update(updateStorageRequest);
                updateStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                throw new IngestException("Could not store content items. Removed created metacards.", e);
            }
            updateStorageResponse = processPostUpdateStoragePlugins(updateStorageResponse);
            for (ContentItem contentItem : updateStorageResponse.getUpdatedContentItems()) {
                if (StringUtils.isBlank(contentItem.getQualifier())) {
                    Metacard metacard = metacardMap.get(contentItem.getId());
                    Metacard overrideMetacard = contentItem.getMetacard();
                    Metacard updatedMetacard = OverrideAttributesSupport.overrideMetacard(metacard, overrideMetacard, true, true);
                    updatedMetacard.setAttribute(new AttributeImpl(Metacard.RESOURCE_SIZE, String.valueOf(contentItem.getSize())));
                    metacardMap.put(contentItem.getId(), updatedMetacard);
                }
            }
        }
        UpdateRequestImpl updateRequest = new UpdateRequestImpl(Iterables.toArray(metacardMap.values().stream().map(Metacard::getId).collect(Collectors.toList()), String.class), new ArrayList<>(metacardMap.values()));
        updateRequest.setProperties(streamUpdateRequest.getProperties());
        historian.setSkipFlag(updateRequest);
        updateResponse = doUpdate(updateRequest);
        historian.version(streamUpdateRequest, updateStorageResponse, updateResponse);
    } catch (Exception e) {
        if (updateStorageRequest != null) {
            try {
                sourceOperations.getStorage().rollback(updateStorageRequest);
            } catch (StorageException e1) {
                LOGGER.info("Unable to remove temporary content for id: {}", updateStorageRequest.getId(), e1);
            }
        }
        throw new IngestException("Unable to store products for request: " + streamUpdateRequest.getId(), e);
    } finally {
        opsStorageSupport.commitAndCleanup(updateStorageRequest, tmpContentPaths);
    }
    updateResponse = doPostIngest(updateResponse);
    return updateResponse;
}

Example 28 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CatalogFrameworkQueryTest method testBeforeQuery.

@Test
public void testBeforeQuery() {
    Calendar beforeCal = Calendar.getInstance();
    beforeCal.add(Calendar.YEAR, 4);
    Calendar card1Exp = Calendar.getInstance();
    card1Exp.add(Calendar.YEAR, 1);
    Calendar card2Exp = Calendar.getInstance();
    card2Exp.add(Calendar.YEAR, 3);
    List<Metacard> metacards = new ArrayList<Metacard>();
    MetacardImpl newCard1 = new MetacardImpl();
    newCard1.setId(null);
    newCard1.setExpirationDate(card1Exp.getTime());
    metacards.add(newCard1);
    MetacardImpl newCard2 = new MetacardImpl();
    newCard2.setId(null);
    newCard2.setExpirationDate(card2Exp.getTime());
    metacards.add(newCard2);
    String mcId1 = null;
    String mcId2 = null;
    CreateResponse createResponse = null;
    try {
        createResponse = framework.create(new CreateRequestImpl(metacards, null));
    } catch (IngestException e1) {
        fail();
    } catch (SourceUnavailableException e1) {
        fail();
    }
    assertEquals(createResponse.getCreatedMetacards().size(), metacards.size());
    for (Metacard curCard : createResponse.getCreatedMetacards()) {
        if (curCard.getExpirationDate().equals(card1Exp.getTime())) {
            mcId1 = curCard.getId();
        } else {
            mcId2 = curCard.getId();
        }
        assertNotNull(curCard.getId());
    }
    FilterFactory filterFactory = new FilterFactoryImpl();
    Instant beforeInstant = new DefaultInstant(new DefaultPosition(beforeCal.getTime()));
    QueryImpl query = new QueryImpl(filterFactory.before(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beforeInstant)));
    QueryRequest queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Expecting return 2 results.", 2, response.getHits());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
    beforeInstant = new DefaultInstant(new DefaultPosition(card2Exp.getTime()));
    query = new QueryImpl(filterFactory.before(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beforeInstant)));
    queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Before filter should return 1 result", 1, response.getHits());
        assertEquals("Before filter should return metacard[" + mcId1 + "]", mcId1, response.getResults().get(0).getMetacard().getId());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
    beforeInstant = new DefaultInstant(new DefaultPosition(card1Exp.getTime()));
    query = new QueryImpl(filterFactory.before(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beforeInstant)));
    queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Before filter should return 0 results.", 0, response.getHits());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
}

Example 29 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CreateOperations method doCreate.

//
// Private helper methods
//
private CreateResponse doCreate(CreateRequest createRequest) throws IngestException, SourceUnavailableException {
    CreateResponse createResponse = null;
    Exception ingestError = null;
    createRequest = queryOperations.setFlagsOnRequest(createRequest);
    createRequest = validateCreateRequest(createRequest);
    createRequest = validateLocalSource(createRequest);
    try {
        createRequest = injectAttributes(createRequest);
        createRequest = setDefaultValues(createRequest);
        createRequest = processPreAuthorizationPlugins(createRequest);
        createRequest = updateCreateRequestPolicyMap(createRequest);
        createRequest = processPrecreateAccessPlugins(createRequest);
        createRequest.getProperties().put(Constants.OPERATION_TRANSACTION_KEY, new OperationTransactionImpl(OperationTransaction.OperationType.CREATE, Collections.emptyList()));
        createRequest = processPreIngestPlugins(createRequest);
        createRequest = validateCreateRequest(createRequest);
        createResponse = getCreateResponse(createRequest);
        createResponse = performRemoteCreate(createRequest, createResponse);
    } catch (IngestException iee) {
        INGEST_LOGGER.debug("Ingest error", iee);
        ingestError = iee;
        throw iee;
    } catch (StopProcessingException see) {
        ingestError = see;
        throw new IngestException(PRE_INGEST_ERROR, see);
    } catch (RuntimeException re) {
        ingestError = re;
        throw new InternalIngestException("Exception during runtime while performing create", re);
    } finally {
        if (ingestError != null && INGEST_LOGGER.isInfoEnabled()) {
            INGEST_LOGGER.info("Error on create operation. {} metacards failed to ingest. {}", createRequest.getMetacards().size(), buildIngestLog(createRequest), ingestError);
        }
    }
    try {
        createResponse = validateFixCreateResponse(createResponse, createRequest);
    } catch (RuntimeException re) {
        LOGGER.info("Exception during runtime while performing doing post create operations (plugins and pubsub)", re);
    }
    return createResponse;
}

Example 30 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CreateOperations method create.

public CreateResponse create(CreateStorageRequest streamCreateRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamCreateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    CreateResponse createResponse = null;
    CreateStorageRequest createStorageRequest = null;
    CreateStorageResponse createStorageResponse;
    streamCreateRequest = opsStorageSupport.prepareStorageRequest(streamCreateRequest, streamCreateRequest::getContentItems);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamCreateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    if (blockCreateMetacards(metacardMap.values(), fanoutTagBlacklist)) {
        String message = "Fanout proxy does not support create operations with blacklisted metacard tag";
        LOGGER.debug("{}. Tags blacklist: {}", message, fanoutTagBlacklist);
        throw new IngestException(message);
    }
    streamCreateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    injectAttributes(metacardMap);
    setDefaultValues(metacardMap);
    streamCreateRequest = applyAttributeOverrides(streamCreateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            createStorageRequest = new CreateStorageRequestImpl(contentItems, streamCreateRequest.getId(), streamCreateRequest.getProperties());
            createStorageRequest = processPreCreateStoragePlugins(createStorageRequest);
            try {
                createStorageResponse = sourceOperations.getStorage().create(createStorageRequest);
                createStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                throw new IngestException("Could not store content items.", e);
            }
            createStorageResponse = processPostCreateStoragePlugins(createStorageResponse);
            populateMetacardMap(metacardMap, createStorageResponse);
        }
        CreateRequest createRequest = new CreateRequestImpl(new ArrayList<>(metacardMap.values()), Optional.ofNullable(createStorageRequest).map(StorageRequest::getProperties).orElseGet(HashMap::new));
        createResponse = doCreate(createRequest);
    } catch (IOException | RuntimeException e) {
        if (createStorageRequest != null) {
            try {
                sourceOperations.getStorage().rollback(createStorageRequest);
            } catch (StorageException e1) {
                LOGGER.info("Unable to remove temporary content for id: {}", createStorageRequest.getId(), e1);
            }
        }
        throw new IngestException("Unable to store products for request: " + streamCreateRequest.getId(), e);
    } finally {
        opsStorageSupport.commitAndCleanup(createStorageRequest, tmpContentPaths);
    }
    createResponse = doPostIngest(createResponse);
    return createResponse;
}