Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 51 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldThrowExceptionOnFailedExecute.

@Test
public void shouldThrowExceptionOnFailedExecute() {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations));
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    };
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new IllegalStateException("parserRunner.execute failed")).withSensorType(Collections.singleton("yaf")).addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
    verify(outputCollector, times(1)).ack(t1);
}

Example 52 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccess.

@Test
public void shouldExecuteOnSuccess() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Collections.singletonList("messageId"));
        verify(writerHandler, times(1)).write("yaf", new BulkMessage<>("messageId", message), parserConfigurations);
    }
}

Example 53 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserBoltTest method shouldExecuteOnError.

@Test
public void shouldExecuteOnError() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    mockParserRunner.setInvalid(true);
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    };
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    JSONObject message = new JSONObject();
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("yaf")).addRawMessage(message);
    parserBolt.execute(t1);
    assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).ack(t1);
}

Example 54 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithWriterClassName.

@Test
public void shouldCreateWriterConfigWithWriterClassName() {
    SensorParserConfig yafConfig = new SensorParserConfig();
    yafConfig.setSensorTopic("yaf");
    yafConfig.setWriterClassName("org.apache.metron.writer.NoopWriter");
    when(configs.getSensorParserConfig("yaf")).thenReturn(yafConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("yaf", yafConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.empty());
    assertEquals(1, writerConfigs.size());
    assertTrue(writerConfigs.get("yaf").getBulkMessageWriter() instanceof NoopWriter);
}

Example 55 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilderTest method shouldCreateWriterConfigWithSuppliedOutputTopic.

@Test
public void shouldCreateWriterConfigWithSuppliedOutputTopic() {
    SensorParserConfig snortConfig = new SensorParserConfig();
    snortConfig.setSensorTopic("snort");
    when(configs.getSensorParserConfig("snort")).thenReturn(snortConfig);
    Map<String, SensorParserConfig> sensorTypeToParserConfig = new HashMap<String, SensorParserConfig>() {

        {
            put("snort", snortConfig);
        }
    };
    Map<String, WriterHandler> writerConfigs = ParserTopologyBuilder.createWriterConfigs("zookeeperUrl", Optional.of("brokerUrl"), sensorTypeToParserConfig, Optional.of("securityProtocol"), configs, Optional.of("supplied_topic"));
    assertEquals(1, writerConfigs.size());
    // Can't directly verify against mocks because this is all static. However, knowing that we have a KafkaWriter
    // and the appropriate topic lets us know we've created the underlying config.
    BulkMessageWriter writer = writerConfigs.get("snort").getBulkMessageWriter();
    assertTrue(writer instanceof KafkaWriter);
    assertEquals("supplied_topic", ((KafkaWriter) writer).getKafkaTopic(new JSONObject()).get());
}