Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 21 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testFilterFailure.

/**
 * Tests to ensure that a message filtered out results in no writes, but an ack.
 * @throws Exception
 */
@Test
public void testFilterFailure() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected SensorParserConfig getSensorParserConfig() {
            try {
                return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {

        {
            put("field2", "blah");
        }
    }))));
    parserBolt.execute(t1);
    verify(batchWriter, times(0)).write(any(), any(), any(), any());
    verify(outputCollector, times(1)).ack(t1);
}

Example 22 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testEmpty.

@Test
public void testEmpty() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(writer, times(1)).init();
    byte[] sampleBinary = "some binary message".getBytes();
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    when(parser.parseOptional(sampleBinary)).thenReturn(null);
    parserBolt.execute(tuple);
    verify(parser, times(0)).validate(any());
    verify(writer, times(0)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), any());
    verify(outputCollector, times(1)).ack(tuple);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_ERROR).withThrowable(new NullPointerException()).withSensorType(sensorType).addRawMessage(sampleBinary);
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 23 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testImplicitBatchOfOne.

@Test
public void testImplicitBatchOfOne() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    BulkWriterResponse response = new BulkWriterResponse();
    response.addSuccess(t1);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(Collections.singleton(t1)), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(t1);
    verify(outputCollector, times(1)).ack(t1);
}

Example 24 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method test.

@Test
public void test() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(writer, times(1)).init();
    byte[] sampleBinary = "some binary message".getBytes();
    JSONParser jsonParser = new JSONParser();
    final JSONObject sampleMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
    final JSONObject sampleMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
    List<JSONObject> messages = new ArrayList<JSONObject>() {

        {
            add(sampleMessage1);
            add(sampleMessage2);
        }
    };
    final JSONObject finalMessage1 = (JSONObject) jsonParser.parse("{ \"field1\":\"value1\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
    final JSONObject finalMessage2 = (JSONObject) jsonParser.parse("{ \"field2\":\"value2\", \"source.type\":\"" + sensorType + "\", \"guid\": \"this-is-unique-identifier-for-tuple\" }");
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messages));
    when(parser.validate(eq(messages.get(0)))).thenReturn(true);
    when(parser.validate(eq(messages.get(1)))).thenReturn(false);
    parserBolt.execute(tuple);
    verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage1));
    verify(outputCollector, times(1)).ack(tuple);
    when(parser.validate(eq(messages.get(0)))).thenReturn(true);
    when(parser.validate(eq(messages.get(1)))).thenReturn(true);
    when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false);
    when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true);
    parserBolt.withMessageFilter(filter);
    parserBolt.execute(tuple);
    verify(writer, times(1)).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
    verify(outputCollector, times(2)).ack(tuple);
    doThrow(new Exception()).when(writer).write(eq(sensorType), any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
    parserBolt.execute(tuple);
    verify(outputCollector, times(1)).reportError(any(Throwable.class));
}

Example 25 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testBatchHappyPath.

@Test
public void testBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(5);
    String sensorType = "test";
    List<Tuple> tuples = new ArrayList<>();
    for (int i = 0; i < 5; ++i) {
        Tuple t = mock(Tuple.class);
        when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
        tuples.add(t);
    }
    WriterBolt bolt = new WriterBolt(new WriterHandler(batchWriter), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(batchWriter, times(1)).init(any(), any(), any());
    for (int i = 0; i < 4; ++i) {
        Tuple t = tuples.get(i);
        bolt.execute(t);
        verify(outputCollector, times(0)).ack(t);
        verify(batchWriter, times(0)).write(eq(sensorType), any(), any(), any());
    }
    // Ensure the batch returns the good Tuples
    BulkWriterResponse writerResponse = new BulkWriterResponse();
    writerResponse.addAllSuccesses(tuples);
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(writerResponse);
    bolt.execute(tuples.get(4));
    for (Tuple t : tuples) {
        verify(outputCollector, times(1)).ack(t);
    }
    verify(batchWriter, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}