Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 36 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldThrowExceptionOnFailedExecute.

@Test
public void shouldThrowExceptionOnFailedExecute() {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    doThrow(new IllegalStateException("parserRunner.execute failed")).when(parserRunner).execute(eq("yaf"), any(), eq(parserConfigurations));
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    };
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new IllegalStateException("parserRunner.execute failed")).withSensorType(Collections.singleton("yaf")).addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
    verify(outputCollector, times(1)).ack(t1);
}

Example 37 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldExecuteOnSuccess.

@Test
public void shouldExecuteOnSuccess() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    {
        parserBolt.execute(t1);
        assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
        verify(bulkWriterResponseHandler).addTupleMessageIds(t1, Collections.singletonList("messageId"));
        verify(writerHandler, times(1)).write("yaf", new BulkMessage<>("messageId", message), parserConfigurations);
    }
}

Example 38 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldExecuteOnError.

@Test
public void shouldExecuteOnError() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    mockParserRunner.setInvalid(true);
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    };
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    JSONObject message = new JSONObject();
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    RawMessage expectedRawMessage = new RawMessage("originalMessage".getBytes(StandardCharsets.UTF_8), new HashMap<>());
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(Collections.singleton("yaf")).addRawMessage(message);
    parserBolt.execute(t1);
    assertEquals(expectedRawMessage, mockParserRunner.getRawMessage());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).ack(t1);
}

Example 39 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldThrowExceptionOnFailedWrite.

@Test
public void shouldThrowExceptionOnFailedWrite() throws Exception {
    when(messageGetStrategy.get(t1)).thenReturn("originalMessage".getBytes(StandardCharsets.UTF_8));
    when(t1.getStringByField(FieldsConfiguration.TOPIC.getFieldName())).thenReturn("yafTopic");
    MockParserRunner mockParserRunner = new MockParserRunner(new HashSet<String>() {

        {
            add("yaf");
        }
    });
    ParserConfigurations parserConfigurations = new ParserConfigurations();
    parserConfigurations.updateSensorParserConfig("yaf", new SensorParserConfig());
    doThrow(new IllegalStateException("write failed")).when(writerHandler).write(any(), any(), any());
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", mockParserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    parserBolt.setMessageGetStrategy(messageGetStrategy);
    parserBolt.setOutputCollector(outputCollector);
    parserBolt.setTopicToSensorMap(new HashMap<String, String>() {

        {
            put("yafTopic", "yaf");
        }
    });
    parserBolt.setAckTuplesPolicy(bulkWriterResponseHandler);
    JSONObject message = new JSONObject();
    message.put(Constants.GUID, "messageId");
    message.put("field", "value");
    mockParserRunner.setMessages(Collections.singletonList(message));
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new IllegalStateException("write failed")).withSensorType(Collections.singleton("yaf")).addRawMessage("originalMessage".getBytes(StandardCharsets.UTF_8));
    parserBolt.execute(t1);
    verify(bulkWriterResponseHandler, times(1)).addTupleMessageIds(t1, Collections.singletonList("messageId"));
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
    verify(outputCollector, times(1)).reportError(any(IllegalStateException.class));
    verify(outputCollector, times(1)).ack(t1);
}

Example 40 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method shouldPrepare.

@Test
public void shouldPrepare() {
    Map stormConf = mock(Map.class);
    SensorParserConfig yafConfig = mock(SensorParserConfig.class);
    when(yafConfig.getSensorTopic()).thenReturn("yafTopic");
    when(yafConfig.getParserConfig()).thenReturn(new HashMap<String, Object>() {

        {
            put(IndexingConfigurations.BATCH_SIZE_CONF, 10);
        }
    });
    ParserConfigurations parserConfigurations = mock(ParserConfigurations.class);
    ParserBolt parserBolt = spy(new ParserBolt("zookeeperUrl", parserRunner, new HashMap<String, WriterHandler>() {

        {
            put("yaf", writerHandler);
        }
    }) {

        @Override
        protected SensorParserConfig getSensorParserConfig(String sensorType) {
            if ("yaf".equals(sensorType)) {
                return yafConfig;
            }
            return null;
        }

        @Override
        public ParserConfigurations getConfigurations() {
            return parserConfigurations;
        }
    });
    doReturn(stellarContext).when(parserBolt).initializeStellar();
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(stormConf, topologyContext, outputCollector);
    verify(parserRunner, times(1)).init(any(Supplier.class), eq(stellarContext));
    verify(yafConfig, times(1)).init();
    Map<String, String> topicToSensorMap = parserBolt.getTopicToSensorMap();
    assertEquals(1, topicToSensorMap.size());
    assertEquals("yaf", topicToSensorMap.get("yafTopic"));
    verify(writerHandler).init(eq(stormConf), eq(topologyContext), eq(outputCollector), eq(parserConfigurations), any(AckTuplesPolicy.class), eq(14));
}