Examples with ParserConfigurations org.apache.metron.common.configuration.ParserConfigurations used on opensource projects

Example 16 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithParsersAndIndexConfigs.

@Test
public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
    when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    Assert.assertEquals(3, indices.size());
    Assert.assertTrue(indices.contains("renamed_bro"));
    Assert.assertTrue(indices.contains("snort"));
    Assert.assertTrue(indices.contains("yaf"));
}

Example 17 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ConfiguredParserBoltTest method test.

@Test
public void test() throws Exception {
    ParserConfigurations sampleConfigurations = new ParserConfigurations();
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.FATAL);
    try {
        StandAloneConfiguredParserBolt configuredBolt = new StandAloneConfiguredParserBolt(null);
        configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
        Assert.fail("A valid zookeeper url must be supplied");
    } catch (RuntimeException e) {
    }
    UnitTestHelper.setLog4jLevel(ConfiguredBolt.class, Level.ERROR);
    configsUpdated = new HashSet<>();
    sampleConfigurations.updateGlobalConfig(ConfigurationsUtils.readGlobalConfigFromFile(TestConstants.SAMPLE_CONFIG_PATH));
    Map<String, byte[]> sensorParserConfigs = ConfigurationsUtils.readSensorParserConfigsFromFile(TestConstants.PARSER_CONFIGS_PATH);
    for (String sensorType : sensorParserConfigs.keySet()) {
        sampleConfigurations.updateSensorParserConfig(sensorType, sensorParserConfigs.get(sensorType));
    }
    StandAloneConfiguredParserBolt configuredBolt = new StandAloneConfiguredParserBolt(zookeeperUrl);
    configuredBolt.prepare(new HashMap(), topologyContext, outputCollector);
    waitForConfigUpdate(parserConfigurationTypes);
    Assert.assertEquals(sampleConfigurations, configuredBolt.getConfigurations());
    configsUpdated = new HashSet<>();
    Map<String, Object> sampleGlobalConfig = sampleConfigurations.getGlobalConfig();
    sampleGlobalConfig.put("newGlobalField", "newGlobalValue");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    Assert.assertEquals("Add global config field", sampleConfigurations.getGlobalConfig(), configuredBolt.getConfigurations().getGlobalConfig());
    configsUpdated = new HashSet<>();
    sampleGlobalConfig.remove("newGlobalField");
    ConfigurationsUtils.writeGlobalConfigToZookeeper(sampleGlobalConfig, zookeeperUrl);
    waitForConfigUpdate(ConfigurationType.GLOBAL.getTypeName());
    Assert.assertEquals("Remove global config field", sampleConfigurations, configuredBolt.getConfigurations());
    configsUpdated = new HashSet<>();
    String sensorType = "testSensorConfig";
    SensorParserConfig testSensorConfig = new SensorParserConfig();
    testSensorConfig.setParserClassName("className");
    testSensorConfig.setSensorTopic("sensorTopic");
    testSensorConfig.setParserConfig(new HashMap<String, Object>() {

        {
            put("configName", "configObject");
        }
    });
    sampleConfigurations.updateSensorParserConfig(sensorType, testSensorConfig);
    ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorType, testSensorConfig, zookeeperUrl);
    waitForConfigUpdate(sensorType);
    Assert.assertEquals("Add new sensor config", sampleConfigurations, configuredBolt.getConfigurations());
    configuredBolt.cleanup();
}

Example 18 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserWriterConfigurationTest method testDefaultBatchSize.

@Test
public void testDefaultBatchSize() {
    ParserWriterConfiguration config = new ParserWriterConfiguration(new ParserConfigurations());
    Assert.assertEquals(1, config.getBatchSize("foo"));
}

Example 19 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testFilterSuccess.

/**
 * Tests to ensure that a message that is unfiltered results in one write and an ack.
 * @throws Exception
 */
@Test
public void testFilterSuccess() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected SensorParserConfig getSensorParserConfig() {
            try {
                return SensorParserConfig.fromBytes(Bytes.toBytes(sensorParserConfig));
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    BulkWriterResponse successResponse = mock(BulkWriterResponse.class);
    when(successResponse.getSuccesses()).thenReturn(ImmutableList.of(t1));
    when(batchWriter.write(any(), any(), any(), any())).thenReturn(successResponse);
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject(new HashMap<String, Object>() {

        {
            put("field1", "blah");
        }
    }))));
    parserBolt.execute(t1);
    verify(batchWriter, times(1)).write(any(), any(), any(), any());
    verify(outputCollector, times(1)).ack(t1);
}

Example 20 with ParserConfigurations

use of org.apache.metron.common.configuration.ParserConfigurations in project metron by apache.

the class ParserBoltTest method testBatchOfFive.

@Test
public void testBatchOfFive() throws Exception {
    String sensorType = "yaf";
    ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser, new WriterHandler(batchWriter)) {

        @Override
        protected ConfigurationsUpdater<ParserConfigurations> createUpdater() {
            return ParserBoltTest.createUpdater();
        }
    };
    parserBolt.setCuratorFramework(client);
    parserBolt.setZKCache(cache);
    parserBolt.prepare(new HashMap(), topologyContext, outputCollector);
    verify(parser, times(1)).init();
    verify(batchWriter, times(1)).init(any(), any(), any());
    when(parser.validate(any())).thenReturn(true);
    when(parser.parseOptional(any())).thenReturn(Optional.of(ImmutableList.of(new JSONObject())));
    when(filter.emitTuple(any(), any(Context.class))).thenReturn(true);
    Set<Tuple> tuples = Stream.of(t1, t2, t3, t4, t5).collect(Collectors.toSet());
    BulkWriterResponse response = new BulkWriterResponse();
    response.addAllSuccesses(tuples);
    when(batchWriter.write(eq(sensorType), any(WriterConfiguration.class), eq(tuples), any())).thenReturn(response);
    parserBolt.withMessageFilter(filter);
    writeNonBatch(outputCollector, parserBolt, t1);
    writeNonBatch(outputCollector, parserBolt, t2);
    writeNonBatch(outputCollector, parserBolt, t3);
    writeNonBatch(outputCollector, parserBolt, t4);
    parserBolt.execute(t5);
    verify(outputCollector, times(1)).ack(t1);
    verify(outputCollector, times(1)).ack(t2);
    verify(outputCollector, times(1)).ack(t3);
    verify(outputCollector, times(1)).ack(t4);
    verify(outputCollector, times(1)).ack(t5);
}