Examples with IngestException ddf.catalog.source.IngestException used on opensource projects

Example 11 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class SolrCatalogProvider method create.

@Override
public CreateResponse create(CreateRequest request) throws IngestException {
    if (request == null) {
        throw new IngestException(REQUEST_MUST_NOT_BE_NULL_MESSAGE);
    }
    List<Metacard> metacards = request.getMetacards();
    List<Metacard> output = new ArrayList<>();
    if (metacards == null) {
        return new CreateResponseImpl(request, null, output);
    }
    for (Metacard metacard : metacards) {
        boolean isSourceIdSet = (metacard.getSourceId() != null && !"".equals(metacard.getSourceId()));
        /*
             * If an ID is not provided, then one is generated so that documents are unique. Solr
             * will not accept documents unless the id is unique.
             */
        if (metacard.getId() == null || metacard.getId().equals("")) {
            if (isSourceIdSet) {
                throw new IngestException("Metacard from a separate distribution must have ID");
            }
            metacard.setAttribute(new AttributeImpl(Metacard.ID, generatePrimaryKey()));
        }
        if (!isSourceIdSet) {
            metacard.setSourceId(getId());
        }
        output.add(metacard);
    }
    try {
        client.add(output, isForcedAutoCommit());
    } catch (SolrServerException | SolrException | IOException | MetacardCreationException e) {
        LOGGER.info("Solr could not ingest metacard(s) during create.", e);
        throw new IngestException("Could not ingest metacard(s).");
    }
    pendingNrtIndex.putAll(output.stream().collect(Collectors.toMap(Metacard::getId, m -> copyMetacard(m))));
    return new CreateResponseImpl(request, request.getProperties(), output);
}

Example 12 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class SolrCatalogProvider method update.

@Override
public UpdateResponse update(UpdateRequest updateRequest) throws IngestException {
    if (updateRequest == null) {
        throw new IngestException(REQUEST_MUST_NOT_BE_NULL_MESSAGE);
    }
    List<Entry<Serializable, Metacard>> updates = updateRequest.getUpdates();
    // the list of updates, both new and old metacards
    ArrayList<Update> updateList = new ArrayList<>();
    String attributeName = updateRequest.getAttributeName();
    // need an attribute name in order to do query
    if (attributeName == null) {
        throw new IngestException("Attribute name cannot be null. " + "Please provide the name of the attribute.");
    }
    List<String> identifiers = new ArrayList<>();
    // if we have nothing to update, send the empty list
    if (updates == null || updates.size() == 0) {
        return new UpdateResponseImpl(updateRequest, null, new ArrayList<>());
    }
    // Loop to get all identifiers
    for (Entry<Serializable, Metacard> updateEntry : updates) {
        identifiers.add(updateEntry.getKey().toString());
    }
    /* 1a. Create the old Metacard Query */
    String attributeQuery = getQuery(attributeName, identifiers);
    SolrQuery query = new SolrQuery(attributeQuery);
    // Set number of rows to the result size + 1.  The default row size in Solr is 10, so this
    // needs to be set in situations where the number of metacards to update is > 10.  Since there
    // could be more results in the query response than the number of metacards in the update request,
    // 1 is added to the row size, so we can still determine whether we found more metacards than
    // updated metacards provided
    query.setRows(updates.size() + 1);
    QueryResponse idResults = null;
    /* 1b. Execute Query */
    try {
        idResults = solr.query(query, METHOD.POST);
    } catch (SolrServerException | IOException e) {
        LOGGER.info("Failed to query for metacard(s) before update.", e);
    }
    // map of old metacards to be populated
    Map<Serializable, Metacard> idToMetacardMap = new HashMap<>();
    /* 1c. Populate list of old metacards */
    if (idResults != null && idResults.getResults() != null && idResults.getResults().size() != 0) {
        LOGGER.debug("Found {} current metacard(s).", idResults.getResults().size());
        // CHECK updates size assertion
        if (idResults.getResults().size() > updates.size()) {
            throw new IngestException("Found more metacards than updated metacards provided. Please ensure your attribute values match unique records.");
        }
        for (SolrDocument doc : idResults.getResults()) {
            Metacard old;
            try {
                old = client.createMetacard(doc);
            } catch (MetacardCreationException e) {
                LOGGER.info("Unable to create metacard(s) from Solr responses during update.", e);
                throw new IngestException("Could not create metacard(s).");
            }
            if (!idToMetacardMap.containsKey(old.getAttribute(attributeName).getValue())) {
                idToMetacardMap.put(old.getAttribute(attributeName).getValue(), old);
            } else {
                throw new IngestException("The attribute value given [" + old.getAttribute(attributeName).getValue() + "] matched multiple records. Attribute values must at most match only one unique Metacard.");
            }
        }
    }
    if (Metacard.ID.equals(attributeName)) {
        idToMetacardMap.putAll(pendingNrtIndex.getAllPresent(identifiers));
    }
    if (idToMetacardMap.size() == 0) {
        LOGGER.debug("No results found for given attribute values.");
        // return an empty list
        return new UpdateResponseImpl(updateRequest, null, new ArrayList<>());
    }
    /* 2. Update the cards */
    List<Metacard> newMetacards = new ArrayList<>();
    for (Entry<Serializable, Metacard> updateEntry : updates) {
        String localKey = updateEntry.getKey().toString();
        /* 2a. Prepare new Metacard */
        MetacardImpl newMetacard = new MetacardImpl(updateEntry.getValue());
        // Find the exact oldMetacard that corresponds with this newMetacard
        Metacard oldMetacard = idToMetacardMap.get(localKey);
        // matched but another did not
        if (oldMetacard != null) {
            // overwrite the id, in case it has not been done properly/already
            newMetacard.setId(oldMetacard.getId());
            newMetacard.setSourceId(getId());
            newMetacards.add(newMetacard);
            updateList.add(new UpdateImpl(newMetacard, oldMetacard));
        }
    }
    try {
        client.add(newMetacards, isForcedAutoCommit());
    } catch (SolrServerException | SolrException | IOException | MetacardCreationException e) {
        LOGGER.info("Failed to update metacard(s) with Solr.", e);
        throw new IngestException("Failed to update metacard(s).");
    }
    pendingNrtIndex.putAll(updateList.stream().collect(Collectors.toMap(u -> u.getNewMetacard().getId(), u -> copyMetacard(u.getNewMetacard()))));
    return new UpdateResponseImpl(updateRequest, updateRequest.getProperties(), updateList);
}

Example 13 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class ReuterSolrImport method ingest.

public void ingest() throws IOException {
    List<Metacard> metacards = new ArrayList<Metacard>();
    for (File localFile : arrayOfFile) {
        Metacard mc = readFile(localFile);
        if (mc != null) {
            metacards.add(mc);
        }
    }
    try {
        solrProvider.create(new CreateRequestImpl(metacards));
    } catch (IngestException e) {
        LOGGER.info("Unexpected IngestException", e);
    }
    solr.close();
}

Example 14 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class RegistryStoreImpl method update.

@Override
public UpdateResponse update(UpdateRequest request) throws IngestException {
    if (request.getUpdates().stream().map(e -> RegistryUtility.getRegistryId(e.getValue())).anyMatch(Objects::isNull)) {
        throw new IngestException("One or more of the metacards is not a registry metacard");
    }
    Map<String, Metacard> updatedMetacards = request.getUpdates().stream().collect(Collectors.toMap(e -> RegistryUtility.getRegistryId(e.getValue()), Map.Entry::getValue));
    Map<String, Metacard> origMetacards = ((OperationTransaction) request.getPropertyValue(Constants.OPERATION_TRANSACTION_KEY)).getPreviousStateMetacards().stream().collect(Collectors.toMap(RegistryUtility::getRegistryId, e -> e));
    //update the new metacards with the id from the orig so that they can be found on the remote system
    try {
        for (Map.Entry<String, Metacard> entry : updatedMetacards.entrySet()) {
            setMetacardExtID(entry.getValue(), origMetacards.get(entry.getKey()).getId());
        }
    } catch (ParserException e) {
        throw new IngestException("Could not update metacards id", e);
    }
    return super.update(request);
}

Example 15 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class RegistryStoreImpl method create.

@Override
public CreateResponse create(CreateRequest request) throws IngestException {
    if (request.getMetacards().stream().map(RegistryUtility::getRegistryId).anyMatch(Objects::isNull)) {
        throw new IngestException("One or more of the metacards is not a registry metacard");
    }
    validateOperation();
    List<Filter> regIdFilters = request.getMetacards().stream().map(e -> filterBuilder.attribute(RegistryObjectMetacardType.REMOTE_METACARD_ID).is().equalTo().text(e.getId())).collect(Collectors.toList());
    Filter tagFilter = filterBuilder.attribute(Metacard.TAGS).is().equalTo().text(RegistryConstants.REGISTRY_TAG_INTERNAL);
    Map<String, Serializable> queryProps = new HashMap<>();
    queryProps.put(SecurityConstants.SECURITY_SUBJECT, request.getPropertyValue(SecurityConstants.SECURITY_SUBJECT));
    QueryImpl query = new QueryImpl(filterBuilder.allOf(tagFilter, filterBuilder.attribute(RegistryObjectMetacardType.REGISTRY_LOCAL_NODE).empty(), filterBuilder.anyOf(regIdFilters)));
    QueryRequest queryRequest = new QueryRequestImpl(query, queryProps);
    try {
        SourceResponse queryResponse = super.query(queryRequest);
        Map<String, Metacard> responseMap = queryResponse.getResults().stream().collect(Collectors.toMap(e -> RegistryUtility.getRegistryId(e.getMetacard()), Result::getMetacard));
        List<Metacard> metacardsToCreate = request.getMetacards().stream().filter(e -> !responseMap.containsKey(RegistryUtility.getRegistryId(e))).collect(Collectors.toList());
        List<Metacard> allMetacards = new ArrayList<>(responseMap.values());
        if (CollectionUtils.isNotEmpty(metacardsToCreate)) {
            CreateResponse createResponse = super.create(new CreateRequestImpl(metacardsToCreate, request.getProperties()));
            allMetacards.addAll(createResponse.getCreatedMetacards());
        }
        return new CreateResponseImpl(request, request.getProperties(), allMetacards);
    } catch (UnsupportedQueryException e) {
        LOGGER.warn("Unable to perform pre-create remote query. Proceeding with original query. Error was {}", e.getMessage());
    }
    return super.create(request);
}