Examples with IngestException ddf.catalog.source.IngestException used on opensource projects

Example 6 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class CatalogComponentFrameworkTest method testUpdateWithIngestException.

@Test
public /**
     * Operation: UPDATE
     * Body contains:  Metacard
     */
void testUpdateWithIngestException() throws Exception {
    resetMocks();
    // Setup expectations to verify
    final MockEndpoint mockVerifierEndpoint = getMockEndpoint("mock:result");
    mockVerifierEndpoint.expectedMessageCount(1);
    final List<Metacard> metacards = new ArrayList<Metacard>();
    metacards.add(metacard1);
    // setup mock catalog framework
    final Update update = new UpdateImpl(metacard1, metacard2);
    List<Update> updates = new ArrayList<Update>();
    updates.add(update);
    final String[] metacardIds = new String[metacards.size()];
    for (int i = 0; i < metacards.size(); i++) {
        metacardIds[i] = metacards.get(i).getId();
    }
    UpdateRequest updateRequest = new UpdateRequestImpl(metacardIds, metacards);
    UpdateResponse updateResponse = new UpdateResponseImpl(updateRequest, new HashMap(), updates);
    when(catalogFramework.update(any(UpdateRequest.class))).thenThrow(new IngestException());
    // Exercise the route with a UPDATE operation
    template.sendBodyAndHeader("direct:sampleInput", metacards, "Operation", "UPDATE");
    // Verify that the number of metacards in the exchange after the records
    // is identical to the input
    assertListSize(mockVerifierEndpoint.getExchanges(), 1);
    final Exchange exchange = mockVerifierEndpoint.getExchanges().get(0);
    final List<Update> cardsUpdated = (List<Update>) exchange.getIn().getBody();
    assertListSize(cardsUpdated, 0);
    mockVerifierEndpoint.assertIsSatisfied();
}

Example 7 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class IngestCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (batchSize * multithreaded > MAX_QUEUE_SIZE) {
        throw new IngestException(String.format("batchsize * multithreaded cannot be larger than %d.", MAX_QUEUE_SIZE));
    }
    final File inputFile = getInputFile();
    if (inputFile == null) {
        return null;
    }
    int totalFiles = totalFileCount(inputFile);
    fileCount.set(totalFiles);
    final ArrayBlockingQueue<Metacard> metacardQueue = new ArrayBlockingQueue<>(batchSize * multithreaded);
    ExecutorService queueExecutor = Executors.newSingleThreadExecutor();
    final long start = System.currentTimeMillis();
    printProgressAndFlush(start, fileCount.get(), 0);
    // Registering for the main thread and on behalf of the buildQueue thread;
    // the buildQueue thread will unregister itself when the files have all
    // been added to the blocking queue and the final registration will
    // be held for the await.
    phaser.register();
    phaser.register();
    queueExecutor.submit(() -> buildQueue(inputFile, metacardQueue, start));
    final ScheduledExecutorService batchScheduler = Executors.newSingleThreadScheduledExecutor();
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, rejectedExecutionHandler);
    final CatalogFacade catalog = getCatalog();
    submitToCatalog(batchScheduler, executorService, metacardQueue, catalog, start);
    // await on catalog processing threads to complete emptying queue
    phaser.awaitAdvance(phaser.arrive());
    try {
        queueExecutor.shutdown();
        executorService.shutdown();
        batchScheduler.shutdown();
    } catch (SecurityException e) {
        LOGGER.info("Executor service shutdown was not permitted: {}", e);
    }
    printProgressAndFlush(start, fileCount.get(), ingestCount.get() + ignoreCount.get());
    long end = System.currentTimeMillis();
    console.println();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.println();
    console.printf(" %d file(s) ingested in %s %n", ingestCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    INGEST_LOGGER.info("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    if (fileCount.get() != ingestCount.get()) {
        console.println();
        if ((fileCount.get() - ingestCount.get() - ignoreCount.get()) >= 1) {
            String failedAmount = Integer.toString(fileCount.get() - ingestCount.get() - ignoreCount.get());
            printErrorMessage(failedAmount + " file(s) failed to be ingested.  See the ingest log for more details.");
            INGEST_LOGGER.warn("{} files(s) failed to be ingested.", failedAmount);
        }
        if (ignoreList != null) {
            String ignoredAmount = Integer.toString(ignoreCount.get());
            printColor(Ansi.Color.YELLOW, ignoredAmount + " file(s) ignored.  See the ingest log for more details.");
            INGEST_LOGGER.warn("{} files(s) were ignored.", ignoredAmount);
        }
    }
    console.println();
    SecurityLogger.audit("Ingested {} files from {}", ingestCount.get(), filePath);
    return null;
}

Example 8 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class IngestCommand method readMetacard.

private Metacard readMetacard(File file) throws IngestException {
    Metacard result = null;
    FileInputStream fis = null;
    ObjectInputStream ois = null;
    try {
        if (SERIALIZED_OBJECT_ID.matches(transformerId)) {
            ois = new ObjectInputStream(new FileInputStream(file));
            result = (Metacard) ois.readObject();
            ois.close();
        } else {
            fis = new FileInputStream(file);
            result = generateMetacard(fis);
            if (StringUtils.isBlank(result.getTitle())) {
                LOGGER.debug("Metacard title was blank. Setting title to filename.");
                result.setAttribute(new AttributeImpl(Metacard.TITLE, file.getName()));
            }
            fis.close();
        }
    } catch (IOException | IllegalArgumentException | ClassNotFoundException e) {
        throw new IngestException(e);
    } finally {
        if (fis != null) {
            try {
                fis.close();
            } catch (IOException e1) {
                console.println(e1);
            }
        }
        if (ois != null) {
            try {
                ois.close();
            } catch (IOException e2) {
                console.println(e2);
            }
        }
    }
    return result;
}

Example 9 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class IngestCommand method processBatch.

private boolean processBatch(CatalogFacade catalog, ArrayList<Metacard> metacards) throws SourceUnavailableException {
    CreateResponse createResponse = null;
    try {
        createResponse = createMetacards(catalog, metacards);
    } catch (IngestException e) {
        printErrorMessage("Error executing command: " + e.getMessage());
        if (INGEST_LOGGER.isWarnEnabled()) {
            INGEST_LOGGER.warn("Error ingesting metacard batch {}", buildIngestLog(metacards), e);
        }
    } catch (SourceUnavailableException e) {
        if (INGEST_LOGGER.isWarnEnabled()) {
            INGEST_LOGGER.warn("Error on process batch, local Provider not available. {}" + " metacards failed to ingest. {}", metacards.size(), buildIngestLog(metacards), e);
        }
    } finally {
        IntStream range = IntStream.range(0, metacards.size());
        range.forEach(i -> phaser.arriveAndDeregister());
        range.close();
    }
    if (createResponse != null) {
        ingestCount.getAndAdd(metacards.size());
    }
    return createResponse != null;
}

Example 10 with IngestException

use of ddf.catalog.source.IngestException in project ddf by codice.

the class FederationStrategyTest method testQueryTimeout.

/**
     * Tests that the framework properly times out using the default federation strategy.
     */
@Test
public void testQueryTimeout() throws Exception {
    long queryDelay = 100;
    UuidGenerator uuidGenerator = mock(UuidGenerator.class);
    when(uuidGenerator.generateUuid()).thenReturn(UUID.randomUUID().toString());
    MockDelayProvider provider = new MockDelayProvider("Provider", "Provider", "v1.0", "DDF", new HashSet<ContentType>(), true, new Date());
    provider.setQueryDelayMillis(queryDelay);
    // Mock register the provider in the container
    SourcePollerRunner runner = new SourcePollerRunner();
    SourcePoller poller = new SourcePoller(runner);
    runner.bind(provider);
    // Must have more than one thread or sleeps will block the monitor
    SortedFederationStrategy fedStrategy = new SortedFederationStrategy(executor, new ArrayList<>(), new ArrayList<>());
    FrameworkProperties props = new FrameworkProperties();
    props.setCatalogProviders(Collections.singletonList(provider));
    props.setFederationStrategy(fedStrategy);
    props.setSourcePoller(poller);
    props.setQueryResponsePostProcessor(mock(QueryResponsePostProcessor.class));
    props.setFilterBuilder(new GeotoolsFilterBuilder());
    props.setDefaultAttributeValueRegistry(new DefaultAttributeValueRegistryImpl());
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(props.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(props, metacardFactory);
    Historian historian = new Historian();
    historian.setHistoryEnabled(false);
    SourceOperations sourceOperations = new SourceOperations(props);
    QueryOperations queryOperations = new QueryOperations(props, sourceOperations, opsSecurity, opsMetacard);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(props, sourceOperations);
    CreateOperations createOperations = new CreateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    UpdateOperations updateOperations = new UpdateOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard, opsCatStore, opsStorage);
    DeleteOperations deleteOperations = new DeleteOperations(props, queryOperations, sourceOperations, opsSecurity, opsMetacard);
    opsStorage.setHistorian(historian);
    updateOperations.setHistorian(historian);
    deleteOperations.setHistorian(historian);
    deleteOperations.setOpsCatStoreSupport(opsCatStore);
    CatalogFrameworkImpl framework = new CatalogFrameworkImpl(createOperations, updateOperations, deleteOperations, queryOperations, null, null, null);
    sourceOperations.bind(provider);
    List<Metacard> metacards = new ArrayList<Metacard>();
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    metacards.add(newCard);
    CreateResponse createResponse = null;
    try {
        try {
            createResponse = framework.create(new CreateRequestImpl(metacards, null));
        } catch (SourceUnavailableException e) {
            long timeout = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
            //this is a hack because the unit test is flaky and should be removed once a better test is possible
            while (System.currentTimeMillis() < timeout) {
                Thread.sleep(1000);
                try {
                    createResponse = framework.create(new CreateRequestImpl(metacards, null));
                    break;
                } catch (SourceUnavailableException e1) {
                //ignore
                }
            }
        }
        if (createResponse == null) {
            fail();
        }
    } catch (IngestException e) {
        fail();
    }
    assertEquals(createResponse.getCreatedMetacards().size(), provider.size());
    for (Metacard curCard : createResponse.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
    }
    QueryImpl query = new QueryImpl(filterFactory.equals(filterFactory.property(Metacard.ID), filterFactory.literal(createResponse.getCreatedMetacards().get(0).getId())));
    query.setTimeoutMillis(SHORT_TIMEOUT);
    query.setSortBy(new FilterFactoryImpl().sort(Result.RELEVANCE, SortOrder.ASCENDING));
    QueryRequest fedQueryRequest = new QueryRequestImpl(query);
    try {
        QueryResponse response = framework.query(fedQueryRequest);
        assertEquals("Timeout should happen before results return", 0, response.getHits());
    } catch (UnsupportedQueryException e) {
        fail();
    } catch (FederationException e) {
        LOGGER.error("Unexpected federation exception during test", e);
        fail();
    }
}