Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 1 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilder method build.

/**
 * Builds a Storm topology that parses telemetry data received from an external sensor.
 *
 * @param zookeeperUrl             Zookeeper URL
 * @param brokerUrl                Kafka Broker URL
 * @param sensorType               Type of sensor
 * @param spoutParallelismSupplier         Supplier for the parallelism hint for the spout
 * @param spoutNumTasksSupplier            Supplier for the number of tasks for the spout
 * @param parserParallelismSupplier        Supplier for the parallelism hint for the parser bolt
 * @param parserNumTasksSupplier           Supplier for the number of tasks for the parser bolt
 * @param errorWriterParallelismSupplier   Supplier for the parallelism hint for the bolt that handles errors
 * @param errorWriterNumTasksSupplier      Supplier for the number of tasks for the bolt that handles errors
 * @param kafkaSpoutConfigSupplier         Supplier for the configuration options for the kafka spout
 * @param securityProtocolSupplier         Supplier for the security protocol
 * @param outputTopic                      The output kafka topic
 * @param stormConfigSupplier              Supplier for the storm config
 * @return A Storm topology that parses telemetry data received from an external sensor
 * @throws Exception
 */
public static ParserTopology build(String zookeeperUrl, Optional<String> brokerUrl, String sensorType, ValueSupplier<Integer> spoutParallelismSupplier, ValueSupplier<Integer> spoutNumTasksSupplier, ValueSupplier<Integer> parserParallelismSupplier, ValueSupplier<Integer> parserNumTasksSupplier, ValueSupplier<Integer> errorWriterParallelismSupplier, ValueSupplier<Integer> errorWriterNumTasksSupplier, ValueSupplier<Map> kafkaSpoutConfigSupplier, ValueSupplier<String> securityProtocolSupplier, Optional<String> outputTopic, ValueSupplier<Config> stormConfigSupplier) throws Exception {
    // fetch configuration from zookeeper
    ParserConfigurations configs = new ParserConfigurations();
    SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs);
    int spoutParallelism = spoutParallelismSupplier.get(parserConfig, Integer.class);
    int spoutNumTasks = spoutNumTasksSupplier.get(parserConfig, Integer.class);
    int parserParallelism = parserParallelismSupplier.get(parserConfig, Integer.class);
    int parserNumTasks = parserNumTasksSupplier.get(parserConfig, Integer.class);
    int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfig, Integer.class);
    int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfig, Integer.class);
    Map<String, Object> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfig, Map.class);
    Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfig, String.class));
    // create the spout
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpout kafkaSpout = createKafkaSpout(zookeeperUrl, sensorType, securityProtocol, Optional.ofNullable(kafkaSpoutConfig), parserConfig);
    builder.setSpout("kafkaSpout", kafkaSpout, spoutParallelism).setNumTasks(spoutNumTasks);
    // create the parser bolt
    ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic);
    builder.setBolt("parserBolt", parserBolt, parserParallelism).setNumTasks(parserNumTasks).localOrShuffleGrouping("kafkaSpout");
    // create the error bolt, if needed
    if (errorWriterNumTasks > 0) {
        WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig);
        builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism).setNumTasks(errorWriterNumTasks).localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
    }
    return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
}

Example 2 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilder method getSensorParserConfig.

/**
 * Fetch the parser configuration from Zookeeper.
 *
 * @param zookeeperUrl Zookeeper URL
 * @param sensorType   Type of sensor
 * @param configs
 * @return
 * @throws Exception
 */
private static SensorParserConfig getSensorParserConfig(String zookeeperUrl, String sensorType, ParserConfigurations configs) throws Exception {
    try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) {
        client.start();
        ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client);
        SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType);
        if (parserConfig == null) {
            throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." + "  Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.");
        }
        return parserConfig;
    }
}

Example 3 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class SensorParserGroupServiceImplTest method shouldSaveNewGroup.

@Test
public void shouldSaveNewGroup() throws Exception {
    when(cache.get(ParserConfigurations.class)).thenReturn(new ParserConfigurations());
    when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig());
    SensorParserGroup sensorParserGroup = new SensorParserGroup();
    sensorParserGroup.setName("group1");
    sensorParserGroup.setDescription("description 1");
    sensorParserGroup.setSensors(Collections.singleton("bro"));
    Map<String, Object> expectedGlobalConfig = new HashMap<>();
    Collection<SensorParserGroup> expectedGroup = Collections.singleton(sensorParserGroup);
    expectedGlobalConfig.put(PARSER_GROUPS_CONF, expectedGroup);
    assertEquals(sensorParserGroup, sensorParserGroupService.save(sensorParserGroup));
    verify(globalConfigService, times(1)).save(expectedGlobalConfig);
    verifyNoMoreInteractions(globalConfigService);
}

Example 4 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class StormAdminServiceImplTest method startParserTopologyByGroupShouldProperlyReturnSuccessTopologyResponse.

@Test
public void startParserTopologyByGroupShouldProperlyReturnSuccessTopologyResponse() throws Exception {
    SensorParserGroup group = new SensorParserGroup();
    group.setName("group");
    group.setSensors(new HashSet<String>() {

        {
            add("bro");
            add("snort");
        }
    });
    when(sensorParserGroupService.findOne("group")).thenReturn(group);
    when(stormCLIClientWrapper.startParserTopology("bro,snort")).thenReturn(0);
    when(globalConfigService.get()).thenReturn(new HashMap<String, Object>());
    when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig());
    when(sensorParserConfigService.findOne("snort")).thenReturn(new SensorParserConfig());
    TopologyResponse expected = new TopologyResponse();
    expected.setSuccessMessage(TopologyStatusCode.STARTED.toString());
    TopologyResponse actual = stormAdminService.startParserTopology("group");
    assertEquals(expected, actual);
    assertEquals(expected.hashCode(), actual.hashCode());
}

Example 5 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserRunnerImplTest method shouldInit.

@Test
public void shouldInit() {
    Context stellarContext = mock(Context.class);
    SensorParserConfig broParserConfig = parserConfigurations.getSensorParserConfig("bro");
    SensorParserConfig snortParserConfig = parserConfigurations.getSensorParserConfig("snort");
    // Effectively using a spy to avoid calling static methods. However, we're using mock with settings to use the constuctor.
    // This forces the use of the doReturn Mockito declaration.
    // This is all because the configurations use parsers from metron-parsers, which are not available in metron-parsers-common
    parserRunner = mock(ParserRunnerImpl.class, withSettings().useConstructor(new HashSet<>(Arrays.asList("bro", "snort"))).defaultAnswer(CALLS_REAL_METHODS));
    doReturn(broParser).when(parserRunner).createParserInstance(broParserConfig);
    doReturn(snortParser).when(parserRunner).createParserInstance(snortParserConfig);
    doReturn(stellarFilter).when(parserRunner).getMessageFilter(eq(broParserConfig), any());
    parserRunner.init(() -> parserConfigurations, stellarContext);
    {
        // Verify Stellar context
        assertEquals(stellarContext, parserRunner.getStellarContext());
    }
    Map<String, ParserComponent> sensorToParserComponentMap = parserRunner.getSensorToParserComponentMap();
    {
        // Verify Bro parser initialization
        assertEquals(2, sensorToParserComponentMap.size());
        ParserComponent broComponent = sensorToParserComponentMap.get("bro");
        assertEquals(broParser, broComponent.getMessageParser());
        assertEquals(stellarFilter, broComponent.getFilter());
        verify(broParser, times(1)).init();
        verify(broParser, times(1)).configure(broParserConfig.getParserConfig());
        verifyNoMoreInteractions(broParser);
        verifyNoMoreInteractions(stellarFilter);
    }
    {
        // Verify Snort parser initialization
        ParserComponent snortComponent = sensorToParserComponentMap.get("snort");
        assertEquals(snortParser, snortComponent.getMessageParser());
        assertNull(snortComponent.getFilter());
        verify(snortParser, times(1)).init();
        verify(snortParser, times(1)).configure(snortParserConfig.getParserConfig());
        verifyNoMoreInteractions(snortParser);
    }
}