Examples with MessageId org.apache.metron.common.writer.MessageId used on opensource projects

Example 16 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldHandleWriteFailure.

@Test
public void shouldHandleWriteFailure() {
    // create a message id and a message associated with that id
    List<BulkMessage<JSONObject>> messages = createMessages(3);
    Exception cause = new Exception();
    // create a document writer which will fail all writes
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addFailure(createDocument(messages.get(0)), cause, "error");
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // the writer response should only contain failures
    assertEquals(0, response.getSuccesses().size());
    assertEquals(1, response.getErrors().size());
    Collection<MessageId> errors = response.getErrors().get(cause);
    assertTrue(errors.contains(new MessageId("message1")));
}

Example 17 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfullyWhenMissingGUID.

@Test
public void shouldWriteSuccessfullyWhenMissingGUID() {
    // create a message id and a message associated with that tuple
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    // remove the GUID from the message
    assertNotNull(messages.get(0).getMessage().remove(Constants.GUID));
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}

Example 18 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class ElasticsearchWriterTest method shouldWriteSuccessfully.

@Test
public void shouldWriteSuccessfully() {
    // create a message id and a message associated with that id
    List<BulkMessage<JSONObject>> messages = createMessages(1);
    // create a document writer which will successfully write all
    BulkDocumentWriterResults<MessageIdBasedDocument> results = new BulkDocumentWriterResults<>();
    results.addSuccess(createDocument(messages.get(0)));
    BulkDocumentWriter<MessageIdBasedDocument> docWriter = mock(BulkDocumentWriter.class);
    when(docWriter.write()).thenReturn(results);
    // attempt to write
    ElasticsearchWriter esWriter = new ElasticsearchWriter();
    esWriter.setDocumentWriter(docWriter);
    esWriter.init(stormConf, writerConfiguration);
    BulkWriterResponse response = esWriter.write("bro", writerConfiguration, messages);
    // response should only contain successes
    assertFalse(response.hasErrors());
    assertTrue(response.getSuccesses().contains(new MessageId("message1")));
}

Example 19 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class WriterBoltTest method testBatchHappyPath.

@Test
public void testBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 5; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Ensure the batch returns the good Tuples
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(tuples.get(4));
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 20 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class WriterBoltTest method testBatchErrorWriteFailure.

@Test
public void testBatchErrorWriteFailure() throws Exception {
    ParserConfigurations configurations = getConfigurations(6);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    JSONObject goodMessage = new JSONObject();
    goodMessage.put(Constants.GUID, "goodMessageId");
    goodMessage.put("value", "goodMessage");
    JSONObject errorMessage = new JSONObject();
    goodMessage.put(Constants.GUID, "errorMessageId");
    errorMessage.put("value", "errorMessage");
    when(goodTuple.getValueByField(eq("message"))).thenReturn(goodMessage);
    when(errorTuple.getValueByField(eq("message"))).thenReturn(errorMessage);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Add both the good and error Tuples. This simulates a seemingly good Tuple that fails on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    writerResponse.addSuccess(new MessageId("goodMessageId"));
    writerResponse.addError(new IllegalStateException(), new MessageId("errorMessageId"));
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.FATAL);
    bolt.execute(goodTuple);
    UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}