Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 11 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testInvalid.

@Test
public void testInvalid() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(writer)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    buildGlobalConfig(parserBolt);
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    byte[] sampleBinary = "some binary message".getBytes();
    when(tuple.getBinary(0)).thenReturn(sampleBinary);
    JSONObject parsedMessage = new JSONObject();
    parsedMessage.put("field", "invalidValue");
    parsedMessage.put("guid", "this-is-unique-identifier-for-tuple");
    List<JSONObject> messageList = new ArrayList<>();
    messageList.add(parsedMessage);
    when(parser.parseOptional(sampleBinary)).thenReturn(Optional.of(messageList));
    when(parser.validate(parsedMessage)).thenReturn(true);
    parserBolt.execute(tuple);
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.PARSER_INVALID).withSensorType(sensorType).withErrorFields(new HashSet<String>() {

        {
            add("field");
        }
    }).addRawMessage(new JSONObject() {

        {
            put("field", "invalidValue");
            put("source.type", "yaf");
            put("guid", "this-is-unique-identifier-for-tuple");
        }
    });
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 12 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testNonBatchErrorPath.

@Test
public void testNonBatchErrorPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(1);
    String sensorType = "test";
    Tuple t = mock(Tuple.class);
    when(t.getValueByField(eq("message"))).thenThrow(new IllegalStateException());
    WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(writer, times(1)).init();
    bolt.execute(t);
    verify(outputCollector, times(1)).ack(t);
    verify(writer, times(0)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 13 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testNonBatchErrorPathErrorInWrite.

@Test
public void testNonBatchErrorPathErrorInWrite() throws Exception {
    ParserConfigurations configurations = getConfigurations(1);
    String sensorType = "test";
    Tuple t = mock(Tuple.class);
    when(t.toString()).thenReturn("tuple");
    when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    doThrow(new Exception("write error")).when(writer).write(any(), any(), any(), any());
    verify(writer, times(1)).init();
    bolt.execute(t);
    verify(outputCollector, times(1)).ack(t);
    verify(writer, times(1)).write(eq(sensorType), any(), any(), any());
    verify(outputCollector, times(1)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
    MetronError error = new MetronError().withErrorType(Constants.ErrorType.DEFAULT_ERROR).withThrowable(new IllegalStateException("Unhandled bulk errors in response: {java.lang.Exception: write error=[tuple]}")).withSensorType(sensorType).addRawMessage(new JSONObject());
    verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
}

Example 14 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithOnlyParsers.

@Test
public void getAllIndicesWithOnlyParsers() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    Assert.assertEquals(2, indices.size());
    Assert.assertTrue(indices.contains("bro"));
    Assert.assertTrue(indices.contains("snort"));
}

Example 15 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithOnlyIndexing.

@Test
public void getAllIndicesWithOnlyIndexing() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(Collections.emptyList());
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    // rename bro, include snort by default configs, and disable yaf
    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "yaf"));
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(false);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    Assert.assertEquals(2, indices.size());
    Assert.assertTrue(indices.contains("renamed_bro"));
    Assert.assertTrue(indices.contains("snort"));
}