use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.get.GetRequest in project datashare by ICIJ.
the class ElasticsearchIndexerTest method test_bulk_add_should_add_ner_pipeline_once_and_for_empty_list.
@Test
public void test_bulk_add_should_add_ner_pipeline_once_and_for_empty_list() throws IOException {
Document doc = new org.icij.datashare.text.Document("id", project("prj"), Paths.get("doc.txt"), "content", Language.FRENCH, Charset.defaultCharset(), "application/pdf", new HashMap<>(), INDEXED, new HashSet<Pipeline.Type>() {
{
add(OPENNLP);
}
}, 432L);
indexer.add(TEST_INDEX, doc);
assertThat(indexer.bulkAdd(TEST_INDEX, OPENNLP, emptyList(), doc)).isTrue();
GetResponse resp = es.client.get(new GetRequest(TEST_INDEX, doc.getId()), RequestOptions.DEFAULT);
assertThat(resp.getSourceAsMap().get("status")).isEqualTo("DONE");
assertThat((ArrayList<String>) resp.getSourceAsMap().get("nerTags")).containsExactly("OPENNLP");
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.get.GetRequest in project datashare by ICIJ.
the class ElasticsearchIndexerTest method test_bulk_add_with_root_document.
@Test
public void test_bulk_add_with_root_document() throws IOException {
Document root = createDoc("root").build();
assertThat(indexer.bulkAdd(TEST_INDEX, asList(createDoc("doc1").withRootId(root.getId()).build(), createDoc("doc2").withRootId(root.getId()).build()))).isTrue();
assertThat(((Document) indexer.get(TEST_INDEX, "doc1")).getRootDocument()).isEqualTo(root.getId());
assertThat(((Document) indexer.get(TEST_INDEX, "doc2")).getRootDocument()).isEqualTo(root.getId());
assertThat(es.client.get(new GetRequest(TEST_INDEX, "doc1"), RequestOptions.DEFAULT).getFields().get("_routing").getValues()).isEqualTo(asList(root.getId()));
assertThat(es.client.get(new GetRequest(TEST_INDEX, "doc1"), RequestOptions.DEFAULT).getFields().get("_routing").getValues()).isEqualTo(asList(root.getId()));
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.get.GetRequest in project datashare by ICIJ.
the class ElasticsearchSpewerTest method test_simple_write.
@Test
public void test_simple_write() throws Exception {
final TikaDocument document = new DocumentFactory().withIdentifier(new PathIdentifier()).create(get("test-file.txt"));
final ParsingReader reader = new ParsingReader(new ByteArrayInputStream("test".getBytes()));
document.setReader(reader);
spewer.write(document);
GetResponse documentFields = es.client.get(new GetRequest(TEST_INDEX, document.getId()), RequestOptions.DEFAULT);
assertThat(documentFields.isExists()).isTrue();
assertThat(documentFields.getId()).isEqualTo(document.getId());
assertEquals(new HashMap<String, String>() {
{
put("name", "Document");
}
}, documentFields.getSourceAsMap().get("join"));
ArgumentCaptor<Message> argument = ArgumentCaptor.forClass(Message.class);
verify(publisher).publish(eq(Channel.NLP), argument.capture());
assertThat(argument.getValue().content).includes(entry(Field.DOC_ID, document.getId()));
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.get.GetRequest in project datashare by ICIJ.
the class ElasticsearchSpewerTest method test_embedded_document.
@Test
public void test_embedded_document() throws Exception {
Path path = get(Objects.requireNonNull(getClass().getResource("/docs/embedded_doc.eml")).getPath());
final TikaDocument document = new Extractor().extract(path);
spewer.write(document);
GetResponse documentFields = es.client.get(new GetRequest(TEST_INDEX, document.getId()), RequestOptions.DEFAULT);
assertTrue(documentFields.isExists());
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.multiMatchQuery("simple.tiff", "content"));
searchRequest.source(searchSourceBuilder);
SearchResponse response = es.client.search(searchRequest, RequestOptions.DEFAULT);
assertThat(response.getHits().getTotalHits().value).isGreaterThan(0);
// assertThat(response.getHits().getAt(0).getId()).endsWith("embedded.pdf");
verify(publisher, times(2)).publish(eq(Channel.NLP), any(Message.class));
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.get.GetRequest in project datashare by ICIJ.
the class ElasticsearchSpewerTest method test_truncated_content_if_document_is_smaller_than_limit.
@Test
public void test_truncated_content_if_document_is_smaller_than_limit() throws Exception {
ElasticsearchSpewer limitedContentSpewer = new ElasticsearchSpewer(es.client, text -> Language.ENGLISH, new FieldNames(), publisher, new PropertiesProvider(new HashMap<String, String>() {
{
put("maxContentLength", "20");
}
})).withRefresh(IMMEDIATE).withIndex("test-datashare");
final TikaDocument document = new DocumentFactory().withIdentifier(new PathIdentifier()).create(get("ok-file.txt"));
final ParsingReader reader = new ParsingReader(new ByteArrayInputStream("this content is ok".getBytes()));
document.setReader(reader);
limitedContentSpewer.write(document);
GetResponse documentFields = es.client.get(new GetRequest(TEST_INDEX, document.getId()), RequestOptions.DEFAULT);
assertThat(documentFields.getSourceAsMap()).includes(entry("content", "this content is ok"));
}
Aggregations