Examples with BatchSearch org.icij.datashare.batch.BatchSearch used on opensource projects

Example 1 with BatchSearch

use of org.icij.datashare.batch.BatchSearch in project datashare by ICIJ.

the class BatchSearchResource method copySearch.

/**
 * Create a new batch search based on a previous one given its id, and enqueue it for running
 *
 * it returns 404 if the source BatchSearch object is not found in the repository.
 *
 * @param sourceBatchId: the id of BatchSearch to copy
 * @param context : the context of request (containing body)
 * @return 200 or 404
 *
 * Example:
 * $(curl localhost:8080/api/batch/search/copy/b7bee2d8-5ede-4c56-8b69-987629742146 -H 'Content-Type: application/json' -d "{\"name\": \"my new batch\", \"description\":\"desc\"}"
 */
@Post("/search/copy/:sourcebatchid")
public String copySearch(String sourceBatchId, Context context) throws Exception {
    BatchSearch sourceBatchSearch = batchSearchRepository.get((User) context.currentUser(), sourceBatchId);
    if (sourceBatchSearch == null) {
        throw new NotFoundException();
    }
    BatchSearch copy = new BatchSearch(sourceBatchSearch, context.extract(HashMap.class));
    boolean isSaved = batchSearchRepository.save(copy);
    if (isSaved)
        batchSearchQueue.put(copy.uuid);
    return copy.uuid;
}

Example 2 with BatchSearch

use of org.icij.datashare.batch.BatchSearch in project datashare by ICIJ.

the class BatchSearchLoop method run.

public void run() {
    logger.info("Datashare running in batch mode. Waiting batch from ds:batchsearch.queue ({})", batchSearchQueue.getClass());
    String currentBatchId = null;
    waitForMainLoopCalled.countDown();
    loopThread = Thread.currentThread();
    while (!POISON.equals(currentBatchId) && !exitAsked) {
        try {
            currentBatchId = batchSearchQueue.poll(60, TimeUnit.SECONDS);
            if (currentBatchId != null && !POISON.equals(currentBatchId)) {
                BatchSearch batchSearch = repository.get(currentBatchId);
                if (batchSearch.state == BatchSearchRecord.State.QUEUED) {
                    repository.setState(batchSearch.uuid, BatchSearchRecord.State.RUNNING);
                    currentBatchSearchRunner.set(factory.createBatchSearchRunner(batchSearch, repository::saveResults));
                    currentBatchSearchRunner.get().call();
                    currentBatchSearchRunner.set(null);
                    repository.setState(batchSearch.uuid, BatchSearchRecord.State.SUCCESS);
                } else {
                    logger.warn("batch search {} not ran because in state {}", batchSearch.uuid, batchSearch.state);
                }
            }
        } catch (JooqBatchSearchRepository.BatchNotFoundException notFound) {
            logger.warn("batch was not executed : {}", notFound.toString());
        } catch (BatchSearchRunner.CancelException cancelEx) {
            logger.info("cancelling batch search {}", currentBatchId);
            batchSearchQueue.offer(currentBatchId);
            repository.reset(currentBatchId);
        } catch (SearchException sex) {
            logger.error("exception while running batch " + currentBatchId, sex);
            repository.setState(currentBatchId, sex);
        } catch (InterruptedException e) {
            logger.warn("main loop interrupted");
        }
    }
    logger.info("exiting main loop");
}

Example 3 with BatchSearch

use of org.icij.datashare.batch.BatchSearch in project datashare by ICIJ.

the class BatchSearchLoopTestInt method test_main_loop_exit_with_sigterm_and_queued_batches.

@Test
public void test_main_loop_exit_with_sigterm_and_queued_batches() throws InterruptedException {
    SleepingBatchSearchRunner batchSearchRunner = new SleepingBatchSearchRunner(100);
    when(factory.createBatchSearchRunner(any(), any())).thenReturn(batchSearchRunner);
    BatchSearch bs1 = new BatchSearch(Project.project("prj"), "name1", "desc", CollectionUtils.asSet("query1"), local());
    BatchSearch bs2 = new BatchSearch(Project.project("prj"), "name2", "desc", CollectionUtils.asSet("query2"), local());
    BatchSearchLoop app = new BatchSearchLoop(repository, batchSearchQueue, factory);
    batchSearchQueue.add(bs1.uuid);
    batchSearchQueue.add(bs2.uuid);
    when(repository.get(bs1.uuid)).thenReturn(bs1);
    when(repository.get(bs2.uuid)).thenReturn(bs2);
    executor.submit(app::run);
    waitQueueToHaveSize(1);
    Signal term = new Signal("TERM");
    Signal.raise(term);
    executor.shutdown();
    assertThat(executor.awaitTermination(2, TimeUnit.SECONDS)).isTrue();
    assertThat(batchSearchQueue).excludes("poison");
    assertThat(batchSearchQueue).containsOnly(bs1.uuid, bs2.uuid);
}

Example 4 with BatchSearch

use of org.icij.datashare.batch.BatchSearch in project datashare by ICIJ.

the class BatchSearchRunnerTest method test_run_batch_search_with_throttle.

@Test
public void test_run_batch_search_with_throttle() throws Exception {
    mockSearch.willReturn(1, createDoc("doc").build());
    BatchSearch batchSearch = new BatchSearch("uuid1", project("test-datashare"), "name1", "desc1", asSet("query1", "query2"), new Date(), BatchSearch.State.QUEUED, local());
    Date beforeBatch = timeRule.now;
    new BatchSearchRunner(indexer, new PropertiesProvider(new HashMap<String, String>() {

        {
            put(BATCH_THROTTLE, "1000");
        }
    }), batchSearch, resultConsumer).call();
    assertThat(timeRule.now().getTime() - beforeBatch.getTime()).isEqualTo(1000);
}

Example 5 with BatchSearch

use of org.icij.datashare.batch.BatchSearch in project datashare by ICIJ.

the class BatchSearchRunnerTest method test_run_batch_search.

@Test
public void test_run_batch_search() throws Exception {
    Document[] documents = { createDoc("doc1").build(), createDoc("doc2").build() };
    mockSearch.willReturn(1, documents);
    BatchSearch search = new BatchSearch("uuid1", project("test-datashare"), "name1", "desc1", asSet("query1", "query2"), new Date(), BatchSearch.State.QUEUED, User.local());
    assertThat(new BatchSearchRunner(indexer, new PropertiesProvider(), search, resultConsumer).call()).isEqualTo(2);
    verify(resultConsumer).apply("uuid1", "query1", asList(documents));
}