Examples with RawMessage org.apache.metron.common.message.metadata.RawMessage used on opensource projects

Example 11 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccessWithMultipleMessages.

@Test
public void shouldExecuteOnSuccessWithMultipleMessages() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    List<BulkMessage<JSONObject>> messages = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        String messageId = String.format("messageId%s", i + 1);
        JSONObject message = new JSONObject();
        message.put(Constants.GUID, messageId);
        message.put("field", String.format("value%s", i + 1));
        messages.add(new BulkMessage<>(messageId, message));
    }
    mockParserRunner.setMessages(messages.stream().map(BulkMessage::getMessage).collect(Collectors.toList()));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        // Verify the correct message is written and ack is handled
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        InOrder inOrder = inOrder(bulkWriterResponseHandler, writerHandler);
        inOrder.verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Arrays.asList("messageId1", "messageId2", "messageId3", "messageId4", "messageId5"));
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(0), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(1), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(2), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(3), parserConfigurations);
        inOrder.verify(writerHandler, times(1)).write("yaf", messages.get(4), parserConfigurations);
    }
    verifyNoMoreInteractions(writerHandler, bulkWriterResponseHandler, outputCollector);
}

Example 12 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserRunnerImplTest method shouldExecuteWithMasterThrowable.

@Test
public void shouldExecuteWithMasterThrowable() {
    parserRunner = spy(parserRunner);
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    Throwable masterThrowable = mock(Throwable.class);
    MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(masterThrowable);
    when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
    verify(parserRunner, times(0)).processMessage(any(), any(), any(), any(), any());
    MetronError expectedError = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(masterThrowable).withSensorType(Collections.singleton("bro")).addRawMessage(rawMessage.getMessage());
    assertEquals(1, parserRunnerResults.getErrors().size());
    assertTrue(parserRunnerResults.getErrors().contains(expectedError));
}

Example 13 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserRunnerImplTest method shouldNotOverwriteOriginalStringAddedByParser.

/**
 * This is only testing the processMessage method
 */
@Test
public void shouldNotOverwriteOriginalStringAddedByParser() {
    JSONObject inputMessage = new JSONObject();
    inputMessage.put("guid", "guid");
    inputMessage.put("ip_src_addr", "192.168.1.1");
    inputMessage.put("ip_dst_addr", "192.168.1.2");
    inputMessage.put(Fields.ORIGINAL.getName(), "original_string_added_by_parser");
    RawMessage rawMessage = new RawMessage("raw_message_for_testing".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    JSONObject expectedOutput = new JSONObject();
    expectedOutput.put("guid", "guid");
    expectedOutput.put("source.type", "bro");
    expectedOutput.put("ip_src_addr", "192.168.1.1");
    expectedOutput.put("ip_dst_addr", "192.168.1.2");
    expectedOutput.put(Fields.ORIGINAL.getName(), "original_string_added_by_parser");
    when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
    when(broParser.validate(expectedOutput)).thenReturn(true);
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
    assertTrue(processResult.isPresent());
    assertFalse(processResult.get().isError());
    assertEquals(expectedOutput, processResult.get().getMessage());
}

Example 14 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserRunnerImplTest method shouldReturnMetronErrorOnInvalidMessage.

@Test
public void shouldReturnMetronErrorOnInvalidMessage() {
    Map<String, Object> metadata = new HashMap<>();
    metadata.put("metron.metadata.topic", "bro");
    metadata.put("metron.metadata.partition", 0);
    metadata.put("metron.metadata.offset", 123);
    JSONObject inputMessage = new JSONObject();
    inputMessage.put("guid", "guid");
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), metadata);
    JSONObject expectedOutput = new JSONObject();
    expectedOutput.put("guid", "guid");
    expectedOutput.put("source.type", "bro");
    expectedOutput.put(Fields.ORIGINAL.getName(), "raw_message");
    MetronError expectedMetronError = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("bro")).withMetadata(metadata).addRawMessage(inputMessage);
    when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
    // This is the important switch. Not to be confused with field validators.
    when(broParser.validate(expectedOutput)).thenReturn(false);
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
    assertTrue(processResult.isPresent());
    assertTrue(processResult.get().isError());
    assertEquals(expectedMetronError, processResult.get().getError());
}