Examples with BatchDownload org.icij.datashare.batch.BatchDownload used on opensource projects

Example 16 with BatchDownload

use of org.icij.datashare.batch.BatchDownload in project datashare by ICIJ.

the class TaskManagerRedisTest method test_start_task.

@Test
public void test_start_task() {
    BatchDownload batchDownload = new BatchDownload(project("prj"), User.local(), "foo", Paths.get("dir"), false);
    BatchDownloadRunner downloadTask = new BatchDownloadRunner(mock(Indexer.class), propertiesProvider, batchDownload, t -> null);
    assertThat(taskManager.startTask(downloadTask, new HashMap<String, Object>() {

        {
            put("batchDownload", batchDownload);
        }
    })).isNotNull();
    assertThat(taskManager.get()).hasSize(1);
    assertThat(batchDownloadQueue).hasSize(1);
    assertThat(redis.hlen("test:task:manager")).isEqualTo(1);
}

Example 17 with BatchDownload

use of org.icij.datashare.batch.BatchDownload in project datashare by ICIJ.

the class BatchDownloadLoopTest method test_loop.

@Test
public void test_loop() throws Exception {
    BatchDownloadCleaner batchDownloadCleaner = mock(BatchDownloadCleaner.class);
    BatchDownloadLoop app = new BatchDownloadLoop(new PropertiesProvider(), batchDownloadQueue, factory, manager) {

        @Override
        public BatchDownloadCleaner createDownloadCleaner(Path downloadDir, int ttlHour) {
            return batchDownloadCleaner;
        }
    };
    batchDownloadQueue.add(new BatchDownload(project("prj"), User.local(), "query"));
    app.enqueuePoison();
    app.run();
    verify(batchRunner).call();
    verify(manager).save(argCaptor.capture());
    verify(batchDownloadCleaner, times(2)).run();
    assertThat(argCaptor.getValue().getState()).isEqualTo(TaskView.State.DONE);
}

Example 18 with BatchDownload

use of org.icij.datashare.batch.BatchDownload in project datashare by ICIJ.

the class BatchDownloadLoopTest method test_elasticsearch_exception__should_not_be_serialized.

@Test
public void test_elasticsearch_exception__should_not_be_serialized() throws Exception {
    when(batchRunner.call()).thenThrow(new ElasticsearchStatusException("error", RestStatus.BAD_REQUEST, new RuntimeException()));
    BatchDownloadCleaner batchDownloadCleaner = mock(BatchDownloadCleaner.class);
    BatchDownloadLoop app = new BatchDownloadLoop(new PropertiesProvider(), batchDownloadQueue, factory, manager) {

        @Override
        public BatchDownloadCleaner createDownloadCleaner(Path downloadDir, int ttlHour) {
            return batchDownloadCleaner;
        }
    };
    batchDownloadQueue.add(new BatchDownload(project("prj"), User.local(), "query"));
    app.enqueuePoison();
    app.run();
    verify(manager).save(argCaptor.capture());
    assertThat(argCaptor.getValue().getState()).isEqualTo(TaskView.State.ERROR);
    assertThat(argCaptor.getValue().error.getClass()).isNotEqualTo(ElasticsearchStatusException.class);
}

Example 19 with BatchDownload

use of org.icij.datashare.batch.BatchDownload in project datashare by ICIJ.

the class BatchDownloadRunnerTest method test_elasticsearch_status_exception__should_be_sent.

@Test(expected = ElasticsearchStatusException.class)
public void test_elasticsearch_status_exception__should_be_sent() throws Exception {
    mockSearch.willThrow(new ElasticsearchStatusException("error", RestStatus.BAD_REQUEST, new RuntimeException()));
    new BatchDownloadRunner(indexer, new PropertiesProvider(), new BatchDownload(project("test-datashare"), User.local(), "query"), updater).call();
}

Example 20 with BatchDownload

use of org.icij.datashare.batch.BatchDownload in project datashare by ICIJ.

the class BatchDownloadRunnerTest method test_max_zip_size.

@Test
public void test_max_zip_size() throws Exception {
    Document[] documents = IntStream.range(0, 3).mapToObj(i -> createDoc("doc" + i).with(createFile(i)).with("hello world " + i).build()).toArray(Document[]::new);
    mockSearch.willReturn(2, documents);
    File zip = new BatchDownloadRunner(indexer, new PropertiesProvider(new HashMap<String, String>() {

        {
            put(BATCH_DOWNLOAD_MAX_SIZE, valueOf("hello world 1".getBytes(StandardCharsets.UTF_8).length * 3));
            put(SCROLL_SIZE, "3");
        }
    }), new BatchDownload(project("test-datashare"), User.local(), "query"), updater).call();
    assertThat(new ZipFile(zip).size()).isEqualTo(4);
}