Examples with MessageId org.apache.metron.common.writer.MessageId used on opensource projects

Example 21 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class WriterBoltTest method testBatchErrorPath.

@Test
public void testBatchErrorPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    WriterBolt bolt = spy(new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType));
    List<Tuple> tuples = new ArrayList<>();
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 4; ++i) {
        Tuple t = mock(Tuple.class);
        String messageId = String.format(MESSAGE_ID_FORMAT, i + 1);
        messageIds.add(new MessageId(messageId));
        JSONObject message = new JSONObject();
        message.put("value", String.format(MESSAGE_FORMAT, i + 1));
        when(t.getValueByField(eq("message"))).thenReturn(message);
        tuples.add(t);
    }
    Tuple errorTuple = mock(Tuple.class);
    Tuple goodTuple = mock(Tuple.class);
    when(goodTuple.getValueByField(eq("message"))).thenReturn(new JSONObject());
    when(errorTuple.getValueByField(eq("message"))).thenThrow(new IllegalStateException());
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any());
    }
    // Add the good tuples.  Do not add the error tuple, because this is testing an exception on access, not a failure on write.
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(messageIds);
    writerResponse.addSuccess(new MessageId("goodMessage"));
    when(batchWriter.write(any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(errorTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(0)).ack(t);
    }
    bolt.execute(goodTuple);
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(outputCollector, times(1)).ack(goodTuple);
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 22 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class AckTuplesPolicyTest method shouldProperlyHandleSuccessAndErrors.

@Test
public void shouldProperlyHandleSuccessAndErrors() throws Exception {
    String messageId1 = "messageId1";
    String messageId2 = "messageId2";
    String messageId3 = "messageId3";
    JSONObject message1 = new JSONObject();
    JSONObject message2 = new JSONObject();
    JSONObject message3 = new JSONObject();
    message1.put("value", "message1");
    message2.put("value", "message2");
    message3.put("value", "message3");
    Tuple tuple3 = mock(Tuple.class);
    Throwable e = new Exception("test exception");
    MetronError expectedError1 = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message1));
    MetronError expectedError2 = new MetronError().withSensorType(Collections.singleton(sensorType)).withErrorType(Constants.ErrorType.INDEXING_ERROR).withThrowable(e).withRawMessages(Collections.singletonList(message2));
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllErrors(e, Arrays.asList(new MessageId(messageId1), new MessageId(messageId2)));
    response.addSuccess(new MessageId(messageId3));
    when(messageGetStrategy.get(tuple1)).thenReturn(message1);
    when(messageGetStrategy.get(tuple2)).thenReturn(message2);
    ackTuplesPolicy.addTupleMessageIds(tuple1, Collections.singleton(messageId1));
    ackTuplesPolicy.addTupleMessageIds(tuple2, Collections.singleton(messageId2));
    ackTuplesPolicy.addTupleMessageIds(tuple3, Collections.singleton(messageId3));
    ackTuplesPolicy.onFlush(sensorType, response);
    assertEquals(0, ackTuplesPolicy.getTupleMessageMap().size());
    assertEquals(0, ackTuplesPolicy.getTupleErrorMap().size());
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError1.getJSONObject()))));
    verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), new Values(argThat(new MetronErrorJSONMatcher(expectedError2.getJSONObject()))));
    verify(collector, times(1)).ack(tuple1);
    verify(collector, times(1)).ack(tuple2);
    verify(collector, times(1)).ack(tuple3);
    verify(collector, times(1)).reportError(e);
    verifyNoMoreInteractions(collector);
}

Example 23 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class BulkMessageWriterBoltTest method parseMessages.

@BeforeEach
public void parseMessages() throws ParseException {
    JSONParser parser = new JSONParser();
    fullMessageList = new ArrayList<>();
    sampleMessage = (JSONObject) parser.parse(sampleMessageString);
    sampleMessage.put(Constants.GUID, "message1");
    sampleMessage.put("field", "value1");
    fullMessageList.add(((JSONObject) sampleMessage.clone()));
    sampleMessage.put(Constants.GUID, "message2");
    sampleMessage.put("field", "value2");
    fullMessageList.add(((JSONObject) sampleMessage.clone()));
    sampleMessage.put(Constants.GUID, "message3");
    sampleMessage.put("field", "value3");
    fullMessageList.add(((JSONObject) sampleMessage.clone()));
    sampleMessage.put(Constants.GUID, "message4");
    sampleMessage.put("field", "value4");
    fullMessageList.add(((JSONObject) sampleMessage.clone()));
    sampleMessage.put(Constants.GUID, "message5");
    sampleMessage.put("field", "value5");
    fullMessageList.add(((JSONObject) sampleMessage.clone()));
    MockitoAnnotations.initMocks(this);
    messageIdList = new ArrayList<>();
    tupleList = new ArrayList<>();
    messageList = new ArrayList<>();
    bulkMessageWriterBolt = spy(new BulkMessageWriterBolt<IndexingConfigurations>("zookeeperUrl", "INDEXING").withBulkMessageWriter(bulkMessageWriter).withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()).withMessageGetterField("message"));
    for (int i = 0; i < 5; i++) {
        String messageId = String.format("message%s", i + 1);
        messageIdList.add(new MessageId(messageId));
        JSONObject message = fullMessageList.get(i);
        Tuple tuple = mock(Tuple.class);
        when(tuple.getValueByField("message")).thenReturn(message);
        tupleList.add(tuple);
        messageList.add(new BulkMessage<>(messageId, message));
    }
}

Example 24 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class KafkaWriterTest method testWriterShouldReturnResponse.

@Test
public void testWriterShouldReturnResponse() throws Exception {
    KafkaWriter writer = spy(new KafkaWriter());
    writer.setKafkaProducer(kafkaProducer);
    List<BulkMessage<JSONObject>> messages = new ArrayList<>();
    JSONObject successMessage = new JSONObject();
    successMessage.put("value", "success");
    JSONObject errorMessage = new JSONObject();
    errorMessage.put("value", "error");
    JSONObject droppedMessage = new JSONObject();
    droppedMessage.put("value", "dropped");
    messages.add(new BulkMessage<>("successId", successMessage));
    messages.add(new BulkMessage<>("errorId", errorMessage));
    messages.add(new BulkMessage<>("droppedId", droppedMessage));
    doReturn(Optional.of("successTopic")).when(writer).getKafkaTopic(successMessage);
    doReturn(Optional.of("errorTopic")).when(writer).getKafkaTopic(errorMessage);
    doReturn(Optional.empty()).when(writer).getKafkaTopic(droppedMessage);
    Future successFuture = mock(Future.class);
    Future errorFuture = mock(Future.class);
    ExecutionException throwable = new ExecutionException(new Exception("kafka error"));
    when(kafkaProducer.send(new ProducerRecord<String, String>("errorTopic", "{\"value\":\"error\"}"))).thenReturn(errorFuture);
    when(kafkaProducer.send(new ProducerRecord<String, String>("successTopic", "{\"value\":\"success\"}"))).thenReturn(successFuture);
    when(errorFuture.get()).thenThrow(throwable);
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(new MessageId("successId"));
    response.addError(throwable, new MessageId("errorId"));
    assertEquals(response, writer.write(SENSOR_TYPE, createConfiguration(new HashMap<>()), messages));
    verify(kafkaProducer, times(1)).flush();
    verify(kafkaProducer, times(1)).send(new ProducerRecord<String, String>("successTopic", "{\"value\":\"success\"}"));
    verify(kafkaProducer, times(1)).send(new ProducerRecord<String, String>("errorTopic", "{\"value\":\"error\"}"));
    verifyNoMoreInteractions(kafkaProducer);
}

Example 25 with MessageId

use of org.apache.metron.common.writer.MessageId in project metron by apache.

the class HdfsWriter method write.

@Override
public BulkWriterResponse write(String sensorType, WriterConfiguration configurations, List<BulkMessage<JSONObject>> messages) throws Exception {
    BulkWriterResponse response = new BulkWriterResponse();
    Set<MessageId> ids = messages.stream().map(BulkMessage::getId).collect(Collectors.toSet());
    // Messages can all result in different HDFS paths, because of Stellar Expressions, so we'll need to iterate through
    for (BulkMessage<JSONObject> bulkWriterMessage : messages) {
        JSONObject message = bulkWriterMessage.getMessage();
        String path = getHdfsPathExtension(sensorType, (String) configurations.getSensorConfig(sensorType).getOrDefault(IndexingConfigurations.OUTPUT_PATH_FUNCTION_CONF, ""), message);
        try {
            LOG.trace("Writing message {} to path: {}", () -> message.toJSONString(), () -> path);
            SourceHandler handler = getSourceHandler(sensorType, path, configurations);
            handler.handle(message, sensorType, configurations, syncPolicyCreator);
        } catch (Exception e) {
            LOG.error("HdfsWriter encountered error writing. Source type: {}. # messages: {}. Output path: {}.", sensorType, messages.size(), path, e);
            response.addAllErrors(e, ids);
        }
    }
    response.addAllSuccesses(ids);
    return response;
}