Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 46 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class WriterBoltTest method testNonBatchHappyPath.

@Test
public void testNonBatchHappyPath() throws Exception {
    ParserConfigurations configurations = getConfigurations(1);
    String sensorType = "test";
    Tuple t = mock(Tuple.class);
    when(t.getValueByField(eq("message"))).thenReturn(new JSONObject());
    WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType);
    bolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(writer, times(1)).init();
    bolt.execute(t);
    verify(outputCollector, times(1)).ack(t);
    verify(writer, times(1)).write(eq(sensorType), any(), any());
    verify(outputCollector, times(0)).reportError(any());
    verify(outputCollector, times(0)).fail(any());
}

Example 47 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserWriterConfigurationTest method testDefaultBatchSize.

@Test
public void testDefaultBatchSize() {
    ParserWriterConfiguration config = new ParserWriterConfiguration(new ParserConfigurations());
    assertEquals(1, config.getBatchSize("foo"));
}

Example 48 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class StellarParserRunner method doParse.

private List<JSONObject> doParse(List<String> messages) {
    // initialize
    HashSet<String> sensorTypes = new HashSet<>();
    sensorTypes.add(sensorType);
    ParserRunnerImpl runner = new ParserRunnerImpl(sensorTypes);
    runner.init(() -> parserConfigurations, context);
    // parse each message
    List<ParserRunnerResults<JSONObject>> results = messages.stream().map(str -> str.getBytes(StandardCharsets.UTF_8)).map(bytes -> DEFAULT.get(emptyMap(), bytes, false, emptyMap())).map(msg -> runner.execute(sensorType, msg, parserConfigurations)).collect(Collectors.toList());
    // aggregate both successes and errors into a list that can be returned
    List<JSONObject> successes = results.stream().flatMap(result -> result.getMessages().stream()).collect(Collectors.toList());
    successCount += successes.size();
    List<JSONObject> errors = results.stream().flatMap(result -> result.getErrors().stream()).map(err -> err.getJSONObject()).collect(Collectors.toList());
    errorCount += errors.size();
    // return a list of both successes and errors
    successes.addAll(errors);
    return successes;
}

Example 49 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithParsersAndIndexConfigs.

@Test
public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
    when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    assertEquals(3, indices.size());
    assertTrue(indices.contains("renamed_bro"));
    assertTrue(indices.contains("snort"));
    assertTrue(indices.contains("yaf"));
}

Example 50 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithOnlyParsers.

@Test
public void getAllIndicesWithOnlyParsers() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    assertEquals(2, indices.size());
    assertTrue(indices.contains("bro"));
    assertTrue(indices.contains("snort"));
}