Examples with MessageId org.apache.metron.common.writer.MessageId used on opensource projects

Example 1 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteManySuccessfullyWithSetDocumentId.

@Test
public void shouldWriteManySuccessfullyWithSetDocumentId() {
    when(writerConfiguration.isSetDocumentId("bro")).thenReturn(true);
    when(writerConfiguration.getFieldNameConverter("bro")).thenReturn("NOOP");
    // create a few message ids and the messages associated with the ids
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    // documents should have field converted
    MessageIdBasedDocument document1 = createDocument(messages.get(0));
    MessageIdBasedDocument document2 = createDocument(messages.get(1));
    MessageIdBasedDocument document3 = createDocument(messages.get(2));
    // documents should have guid as documentID
    document1.setDocumentID(document1.getGuid());
    document2.setDocumentID(document1.getGuid());
    document3.setDocumentID(document1.getGuid());
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(document1);
    results.addSuccess(document2);
    results.addSuccess(document3);
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = spy(ElasticsearchWriter.class);
    doReturn(new SimpleDateFormat()).when(esWriter).getIndexFormat(globals());
    doReturn("bro_index").when(esWriter).getIndexName(eq("bro"), any(), eq(writerConfiguration));
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // documents should have metron guid as documentID
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    verify(docWriter, times(1)).addDocument(document1, "bro_index");
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
    assertTrue(response.getSuccesses().contains(new MessageId("message3")));
}

Example 2 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldHandlePartialFailures.

@Test
public void shouldHandlePartialFailures() {
    // create a few message ids and the messages associated with the ids
    int count = 2;
    List<BulkMessage<JSONObject>> messages = createMessages(count);
    Exception cause = new Exception();
    // create a document writer that will fail one and succeed the other
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    results.addSuccess(createDocument(messages.get(1)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should contain some successes and some failures
    assertEquals(1, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    assertTrue(response.getErrors().get(cause).contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
}

Example 3 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldHandleManyWriteFailures.

@Test
public void shouldHandleManyWriteFailures() {
    // create a few message ids and the messages associated with the ids
    int count = 3;
    List<BulkMessage<JSONObject>> messages = createMessages(count);
    Exception cause = new Exception();
    // create a document writer which will fail all writes
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    results.addFailure(createDocument(messages.get(1)), cause, "error");
    results.addFailure(createDocument(messages.get(2)), cause, "error");
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // the writer response should only contain failures
    assertEquals(0, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    Collection<MessageId> errors = response.getErrors().get(cause);
    assertTrue(errors.contains(new MessageId("message1")));
    assertTrue(errors.contains(new MessageId("message2")));
    assertTrue(errors.contains(new MessageId("message3")));
}

Example 4 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteManySuccessfully.

@Test
public void shouldWriteManySuccessfully() {
    // create a few message ids and the messages associated with the ids
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    results.addSuccess(createDocument(messages.get(1)));
    results.addSuccess(createDocument(messages.get(2)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
    assertTrue(response.getSuccesses().contains(new MessageId("message2")));
    assertTrue(response.getSuccesses().contains(new MessageId("message3")));
}

Example 5 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method createDocument.

private MessageIdBasedDocument createDocument(BulkMessage<JSONObject> bulkWriterMessage) {
    MessageId messageId = bulkWriterMessage.getId();
    JSONObject message = bulkWriterMessage.getMessage();
    Long timestamp = (Long) bulkWriterMessage.getMessage().get(Constants.Fields.TIMESTAMP.getName());
    String guid = (String) message.get(Constants.GUID);
    String sensorType = (String) message.get(Constants.SENSOR_TYPE);
    return new MessageIdBasedDocument(message, guid, sensorType, timestamp, messageId);
}