Examples with IngestException ddf.catalog.source.IngestException used on opensource projects

Example 16 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class UpdateOperations method doRemoteUpdate.

private UpdateResponse doRemoteUpdate(UpdateRequest updateRequest) {
    HashSet<ProcessingDetails> exceptions = new HashSet<>();
    Map<String, Serializable> properties = new HashMap<>();
    List<CatalogStore> stores = opsCatStoreSupport.getCatalogStoresForRequest(updateRequest, exceptions);
    List<Update> updates = new ArrayList<>();
    for (CatalogStore store : stores) {
        try {
            if (!store.isAvailable()) {
                exceptions.add(new ProcessingDetailsImpl(store.getId(), null, "CatalogStore is not available"));
            } else {
                UpdateResponse response = store.update(updateRequest);
                properties.put(store.getId(), new ArrayList<>(response.getUpdatedMetacards()));
                updates = response.getUpdatedMetacards();
            }
        } catch (IngestException e) {
            INGEST_LOGGER.error("Error updating metacards for CatalogStore {}", store.getId(), e);
            exceptions.add(new ProcessingDetailsImpl(store.getId(), e));
        }
    }
    return new UpdateResponseImpl(updateRequest, properties, updates, exceptions);
}

Example 17 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class UpdateOperations method doUpdate.

//
// Private helper methods
//
private UpdateResponse doUpdate(UpdateRequest updateRequest) throws IngestException, SourceUnavailableException {
    updateRequest = queryOperations.setFlagsOnRequest(updateRequest);
    updateRequest = validateUpdateRequest(updateRequest);
    updateRequest = validateLocalSource(updateRequest);
    try {
        updateRequest = injectAttributes(updateRequest);
        updateRequest = setDefaultValues(updateRequest);
        updateRequest = populateMetacards(updateRequest);
        updateRequest = processPreAuthorizationPlugins(updateRequest);
        updateRequest = populateUpdateRequestPolicyMap(updateRequest);
        updateRequest = processPreUpdateAccessPlugins(updateRequest);
        updateRequest = processPreIngestPlugins(updateRequest);
        updateRequest = validateUpdateRequest(updateRequest);
        // Call the update on the catalog
        LOGGER.debug("Calling catalog.update() with {} updates.", updateRequest.getUpdates().size());
        UpdateResponse updateResponse = performLocalUpdate(updateRequest);
        updateResponse = performRemoteUpdate(updateRequest, updateResponse);
        // Handle the posting of messages to pubsub
        updateResponse = validateFixUpdateResponse(updateResponse, updateRequest);
        return updateResponse;
    } catch (StopProcessingException see) {
        throw new IngestException(PRE_INGEST_ERROR, see);
    } catch (RuntimeException re) {
        throw new InternalIngestException("Exception during runtime while performing update", re);
    }
}

Example 18 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class FtpRequestHandlerTest method testCreateStorageRequestFail.

@SuppressWarnings("unchecked")
@Test(expected = FtpException.class)
public void testCreateStorageRequestFail() throws Exception {
    Subject subject = mock(Subject.class);
    FtpFile ftpFile = mock(FtpFile.class);
    when(session.getAttribute(SUBJECT)).thenReturn(subject);
    when(request.getArgument()).thenReturn(FILE_NAME);
    when(session.getFileSystemView().getFile(FILE_NAME)).thenReturn(ftpFile);
    when(ftpFile.isWritable()).thenReturn(true);
    when(ftpFile.getAbsolutePath()).thenReturn(FILE_NAME);
    when(subject.execute(any(Callable.class))).thenAnswer(invocationOnMock -> ((Callable) invocationOnMock.getArguments()[0]).call());
    when(catalogFramework.create((CreateStorageRequest) anyObject())).thenThrow(new IngestException());
    ftplet.onUploadStart(session, request);
}

Example 19 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CatalogFrameworkQueryTest method testBeginsQuery.

@Test
public void testBeginsQuery() {
    Calendar beginsStart = Calendar.getInstance();
    Calendar card1Exp = Calendar.getInstance();
    if (beginsStart.equals(card1Exp)) {
        card1Exp.add(Calendar.MILLISECOND, 1);
    }
    Calendar card2Exp = Calendar.getInstance();
    card2Exp.add(Calendar.YEAR, 3);
    Calendar beginsEnd = Calendar.getInstance();
    beginsEnd.add(Calendar.YEAR, 4);
    List<Metacard> metacards = new ArrayList<Metacard>();
    MetacardImpl newCard1 = new MetacardImpl();
    newCard1.setId(null);
    newCard1.setExpirationDate(card1Exp.getTime());
    metacards.add(newCard1);
    MetacardImpl newCard2 = new MetacardImpl();
    newCard2.setId(null);
    newCard2.setExpirationDate(card2Exp.getTime());
    metacards.add(newCard2);
    String mcId1 = null;
    String mcId2 = null;
    CreateResponse createResponse = null;
    try {
        createResponse = framework.create(new CreateRequestImpl(metacards, null));
    } catch (IngestException e1) {
        LOGGER.error("Failure", e1);
        fail();
    } catch (SourceUnavailableException e1) {
        LOGGER.error("Failure", e1);
        fail();
    }
    assertEquals(createResponse.getCreatedMetacards().size(), metacards.size());
    for (Metacard curCard : createResponse.getCreatedMetacards()) {
        if (curCard.getExpirationDate().equals(card1Exp.getTime())) {
            mcId1 = curCard.getId();
        } else {
            mcId2 = curCard.getId();
        }
        assertNotNull(curCard.getId());
    }
    FilterFactory filterFactory = new FilterFactoryImpl();
    Period beginsPeriod = new DefaultPeriod(new DefaultInstant(new DefaultPosition(beginsStart.getTime())), new DefaultInstant(new DefaultPosition(beginsEnd.getTime())));
    QueryImpl query = new QueryImpl(filterFactory.begins(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beginsPeriod)));
    QueryRequest queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Expecting return 0 results.", 0, response.getHits());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
    beginsPeriod = new DefaultPeriod(new DefaultInstant(new DefaultPosition(card1Exp.getTime())), new DefaultInstant(new DefaultPosition(beginsEnd.getTime())));
    query = new QueryImpl(filterFactory.begins(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beginsPeriod)));
    queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Begins filter should return 1 result", 1, response.getHits());
        assertEquals("Begins filter should return metacard[" + mcId1 + "]", mcId1, response.getResults().get(0).getMetacard().getId());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
    beginsPeriod = new DefaultPeriod(new DefaultInstant(new DefaultPosition(card2Exp.getTime())), new DefaultInstant(new DefaultPosition(beginsEnd.getTime())));
    query = new QueryImpl(filterFactory.begins(filterFactory.property(Metacard.EXPIRATION), filterFactory.literal(beginsPeriod)));
    queryReq = new QueryRequestImpl(query, false);
    try {
        QueryResponse response = framework.query(queryReq);
        assertEquals("Begins filter should return 1 result", 1, response.getHits());
        assertEquals("Begins filter should return metacard[" + mcId2 + "]", mcId2, response.getResults().get(0).getMetacard().getId());
    } catch (UnsupportedQueryException | SourceUnavailableException | FederationException e) {
        LOGGER.error("Failure", e);
        fail();
    }
}

Example 20 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class AbstractCswStore method create.

@Override
public CreateResponse create(CreateRequest createRequest) throws IngestException {
    Map<String, Serializable> properties = new HashMap<>();
    validateOperation();
    Subject subject = (Subject) createRequest.getPropertyValue(SecurityConstants.SECURITY_SUBJECT);
    Csw csw = factory.getClientForSubject(subject);
    CswTransactionRequest transactionRequest = getTransactionRequest();
    List<Metacard> metacards = createRequest.getMetacards();
    List<String> metacardIds = metacards.stream().map(Metacard::getId).collect(Collectors.toList());
    List<Metacard> createdMetacards = new ArrayList<>();
    List<Filter> createdMetacardFilters;
    HashSet<ProcessingDetails> errors = new HashSet<>();
    String insertTypeName = schemaTransformerManager.getTransformerIdForSchema(cswSourceConfiguration.getOutputSchema());
    if (insertTypeName == null) {
        throw new IngestException("Could not find transformer for output schema " + cswSourceConfiguration.getOutputSchema());
    }
    transactionRequest.getInsertActions().add(new InsertAction(insertTypeName, null, metacards));
    try {
        TransactionResponseType response = csw.transaction(transactionRequest);
        Set<String> processedIds = new HashSet<>();
        //dive down into the response to get the created ID's. We need these so we can query
        //the source again to get the created metacards and put them in the result
        createdMetacardFilters = response.getInsertResult().stream().map(InsertResultType::getBriefRecord).flatMap(Collection::stream).map(BriefRecordType::getIdentifier).flatMap(Collection::stream).map(JAXBElement::getValue).map(SimpleLiteral::getContent).flatMap(Collection::stream).map(id -> {
            processedIds.add(id);
            return filterBuilder.attribute(Core.ID).is().equalTo().text(id);
        }).collect(Collectors.toList());
        metacardIds.removeAll(processedIds);
        errors.addAll(metacardIds.stream().map(id -> new ProcessingDetailsImpl(id, null, "Failed to create metacard")).collect(Collectors.toList()));
    } catch (CswException e) {
        throw new IngestException("Csw Transaction Failed : ", e);
    }
    try {
        createdMetacards = transactionQuery(createdMetacardFilters, subject);
    } catch (UnsupportedQueryException e) {
        errors.add(new ProcessingDetailsImpl(this.getId(), e, "Failed to retrieve newly created metacards"));
    }
    return new CreateResponseImpl(createRequest, properties, createdMetacards, errors);
}