Examples with RawMessage org.apache.metron.common.message.metadata.RawMessage used on opensource projects

Example 6 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    if (TupleUtils.isTick(tuple)) {
        handleTickTuple(tuple);
        return;
    }
    byte[] originalMessage = (byte[]) messageGetStrategy.get(tuple);
    String topic = tuple.getStringByField(FieldsConfiguration.TOPIC.getFieldName());
    String sensorType = topicToSensorMap.get(topic);
    try {
        ParserConfigurations parserConfigurations = getConfigurations();
        SensorParserConfig sensorParserConfig = parserConfigurations.getSensorParserConfig(sensorType);
        RawMessage rawMessage = RawMessageUtil.INSTANCE.getRawMessage(sensorParserConfig.getRawMessageStrategy(), tuple, originalMessage, sensorParserConfig.getReadMetadata(), sensorParserConfig.getRawMessageStrategyConfig());
        ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute(sensorType, rawMessage, parserConfigurations);
        parserRunnerResults.getErrors().forEach(error -> handleError(collector, error));
        WriterHandler writer = sensorToWriterMap.get(sensorType);
        int numWritten = 0;
        List<JSONObject> messages = parserRunnerResults.getMessages();
        List<String> messageIds = messages.stream().map(MessageUtils::getGuid).collect(Collectors.toList());
        ackTuplesPolicy.addTupleMessageIds(tuple, messageIds);
        for (int i = 0; i < messages.size(); i++) {
            String messageId = messageIds.get(i);
            JSONObject message = messages.get(i);
            try {
                writer.write(sensorType, new BulkMessage<>(messageId, message), getConfigurations());
                numWritten++;
            } catch (Exception ex) {
                handleError(sensorType, originalMessage, tuple, ex, collector);
            }
        }
        if (numWritten == 0) {
            collector.ack(tuple);
        }
    } catch (Throwable ex) {
        handleError(sensorType, originalMessage, tuple, ex, collector);
        collector.ack(tuple);
    }
}

Example 7 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserRunnerImplTest method shouldReturnMetronErrorOnFailedFieldValidator.

@Test
public void shouldReturnMetronErrorOnFailedFieldValidator() {
    Map<String, Object> metadata = new HashMap<>();
    metadata.put("metron.metadata.topic", "bro");
    metadata.put("metron.metadata.partition", 0);
    metadata.put("metron.metadata.offset", 123);
    JSONObject inputMessage = new JSONObject();
    inputMessage.put("guid", "guid");
    inputMessage.put("ip_src_addr", "test");
    inputMessage.put("ip_dst_addr", "test");
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), metadata);
    JSONObject expectedOutput = new JSONObject();
    expectedOutput.put("guid", "guid");
    expectedOutput.put("ip_src_addr", "test");
    expectedOutput.put("ip_dst_addr", "test");
    expectedOutput.put("source.type", "bro");
    expectedOutput.put(Fields.ORIGINAL.getName(), "raw_message");
    MetronError expectedMetronError = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("bro")).addRawMessage(inputMessage).withMetadata(metadata).withErrorFields(new HashSet<>(Arrays.asList("ip_src_addr", "ip_dst_addr")));
    when(stellarFilter.emit(expectedOutput, parserRunner.getStellarContext())).thenReturn(true);
    when(broParser.validate(expectedOutput)).thenReturn(true);
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    Optional<ParserRunnerImpl.ProcessResult> processResult = parserRunner.processMessage("bro", inputMessage, rawMessage, broParser, parserConfigurations);
    assertTrue(processResult.isPresent());
    assertTrue(processResult.get().isError());
    assertEquals(expectedMetronError, processResult.get().getError());
}

Example 8 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserRunnerImplTest method shouldExecute.

/**
 * This is only testing the execute method. It mocks out processMessage().
 */
@Test
public void shouldExecute() {
    parserRunner = spy(parserRunner);
    RawMessage rawMessage = new RawMessage("raw_message".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    JSONObject parsedMessage1 = new JSONObject();
    parsedMessage1.put("field", "parsedMessage1");
    JSONObject parsedMessage2 = new JSONObject();
    parsedMessage2.put("field", "parsedMessage2");
    Object rawMessage1 = new RawMessage("raw_message1".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    Object rawMessage2 = new RawMessage("raw_message2".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    Throwable throwable1 = mock(Throwable.class);
    Throwable throwable2 = mock(Throwable.class);
    MessageParserResult<JSONObject> messageParserResult = new DefaultMessageParserResult<>(Arrays.asList(parsedMessage1, parsedMessage2), new HashMap<Object, Throwable>() {

        {
            put(rawMessage1, throwable1);
            put(rawMessage2, throwable2);
        }
    });
    JSONObject processedMessage = new JSONObject();
    processedMessage.put("field", "processedMessage1");
    MetronError processedError = new MetronError().withMessage("processedError");
    ProcessResult processedMessageResult = mock(ProcessResult.class);
    ProcessResult processedErrorResult = mock(ProcessResult.class);
    when(broParser.parseOptionalResult(rawMessage.getMessage())).thenReturn(Optional.of(messageParserResult));
    when(processedMessageResult.getMessage()).thenReturn(processedMessage);
    when(processedErrorResult.isError()).thenReturn(true);
    when(processedErrorResult.getError()).thenReturn(processedError);
    doReturn(Optional.of(processedMessageResult)).when(parserRunner).processMessage("bro", parsedMessage1, rawMessage, broParser, parserConfigurations);
    doReturn(Optional.of(processedErrorResult)).when(parserRunner).processMessage("bro", parsedMessage2, rawMessage, broParser, parserConfigurations);
    MetronError expectedParseError1 = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(throwable1).withSensorType(Collections.singleton("bro")).addRawMessage(rawMessage1);
    MetronError expectedParseError2 = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(throwable2).withSensorType(Collections.singleton("bro")).addRawMessage(rawMessage2);
    parserRunner.setSensorToParserComponentMap(new HashMap<String, ParserComponent>() {

        {
            put("bro", new ParserComponent(broParser, stellarFilter));
        }
    });
    ParserRunnerResults<JSONObject> parserRunnerResults = parserRunner.execute("bro", rawMessage, parserConfigurations);
    assertEquals(1, parserRunnerResults.getMessages().size());
    assertTrue(parserRunnerResults.getMessages().contains(processedMessage));
    assertEquals(3, parserRunnerResults.getErrors().size());
    assertTrue(parserRunnerResults.getErrors().contains(processedError));
    assertTrue(parserRunnerResults.getErrors().contains(expectedParseError1));
    assertTrue(parserRunnerResults.getErrors().contains(expectedParseError2));
}

Example 9 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccess.

@Test
public void shouldExecuteOnSuccess() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Collections.singletonList("messageId"));
        verify(writerHandler, times(1)).write("yaf", new BulkMessage<>("messageId", message), parserConfigurations);
    }
}

Example 10 with RawMessage

use of org.apache.metron.common.message.metadata.RawMessage in project metron by apache.

the class ParserBoltTest method shouldExecuteOnError.

@Test
public void shouldExecuteOnError() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    mockParserRunner.setInvalid(true);
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    };
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    JSONObject message = new JSONObject();
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("yaf")).addRawMessage(message);
    parserBolt.execute(t1);
    assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).ack(t1);
}