Examples with ShutdownMessage org.icij.datashare.com.ShutdownMessage used on opensource projects

Example 1 with ShutdownMessage

use of org.icij.datashare.com.ShutdownMessage in project datashare by ICIJ.

the class IndexTask method call.

@Override
public Long call() throws Exception {
    logger.info("Processing up to {} file(s) in parallel", parallelism);
    totalToProcess = drainer.drain(POISON).get();
    drainer.shutdown();
    // drain is finished
    drainer.awaitTermination(10, SECONDS);
    logger.info("drained {} documents. Waiting for consumer to shutdown", totalToProcess);
    publisher.publish(Channel.NLP, new Message(INIT_MONITORING).add(VALUE, valueOf(totalToProcess)));
    consumer.shutdown();
    // documents could be currently processed
    try {
        while (!consumer.awaitTermination(30, MINUTES)) {
            logger.info("Consumer has not terminated yet.");
        }
    } catch (InterruptedException iex) {
        logger.info("Got InterruptedException while waiting for the consumer shutdown.");
    }
    publisher.publish(Channel.NLP, new ShutdownMessage());
    if (consumer.getReporter() != null)
        consumer.getReporter().close();
    queue.close();
    logger.info("exiting");
    return totalToProcess;
}

Example 2 with ShutdownMessage

use of org.icij.datashare.com.ShutdownMessage in project datashare by ICIJ.

the class ResumeNlpTask method call.

@Override
public Long call() throws IOException {
    Indexer.Searcher searcher = indexer.search(projectName, Document.class).withSource("rootDocument").without(nlpPipelines.toArray(new Pipeline.Type[] {}));
    logger.info("resuming NLP name finding for index {} and {} : {} documents found", projectName, nlpPipelines, searcher.totalHits());
    List<? extends Entity> docsToProcess = searcher.scroll().collect(toList());
    long totalHits = searcher.totalHits();
    this.publisher.publish(Channel.NLP, new Message(Message.Type.INIT_MONITORING).add(Message.Field.VALUE, valueOf(totalHits)));
    do {
        docsToProcess.forEach(doc -> this.publisher.publish(Channel.NLP, new Message(Message.Type.EXTRACT_NLP).add(Message.Field.INDEX_NAME, projectName).add(Message.Field.DOC_ID, doc.getId()).add(Message.Field.R_ID, ((Document) doc).getRootDocument())));
        docsToProcess = searcher.scroll().collect(toList());
    } while (docsToProcess.size() != 0);
    logger.info("sent {} message for {} files without {} pipeline tags", Message.Type.EXTRACT_NLP, totalHits, nlpPipelines);
    searcher.clearScroll();
    this.publisher.publish(Channel.NLP, new ShutdownMessage());
    return totalHits;
}