Examples with SensorParserConfig org.apache.metron.common.configuration.SensorParserConfig used on opensource projects

Example 16 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilder method build.

/**
 * Builds a Storm topology that parses telemetry data received from an external sensor.
 *
 * @param zookeeperUrl             Zookeeper URL
 * @param brokerUrl                Kafka Broker URL
 * @param sensorType               Type of sensor
 * @param spoutParallelismSupplier         Supplier for the parallelism hint for the spout
 * @param spoutNumTasksSupplier            Supplier for the number of tasks for the spout
 * @param parserParallelismSupplier        Supplier for the parallelism hint for the parser bolt
 * @param parserNumTasksSupplier           Supplier for the number of tasks for the parser bolt
 * @param errorWriterParallelismSupplier   Supplier for the parallelism hint for the bolt that handles errors
 * @param errorWriterNumTasksSupplier      Supplier for the number of tasks for the bolt that handles errors
 * @param kafkaSpoutConfigSupplier         Supplier for the configuration options for the kafka spout
 * @param securityProtocolSupplier         Supplier for the security protocol
 * @param outputTopic                      The output kafka topic
 * @param stormConfigSupplier              Supplier for the storm config
 * @return A Storm topology that parses telemetry data received from an external sensor
 * @throws Exception
 */
public static ParserTopology build(String zookeeperUrl, Optional<String> brokerUrl, String sensorType, ValueSupplier<Integer> spoutParallelismSupplier, ValueSupplier<Integer> spoutNumTasksSupplier, ValueSupplier<Integer> parserParallelismSupplier, ValueSupplier<Integer> parserNumTasksSupplier, ValueSupplier<Integer> errorWriterParallelismSupplier, ValueSupplier<Integer> errorWriterNumTasksSupplier, ValueSupplier<Map> kafkaSpoutConfigSupplier, ValueSupplier<String> securityProtocolSupplier, Optional<String> outputTopic, ValueSupplier<Config> stormConfigSupplier) throws Exception {
    // fetch configuration from zookeeper
    ParserConfigurations configs = new ParserConfigurations();
    SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs);
    int spoutParallelism = spoutParallelismSupplier.get(parserConfig, Integer.class);
    int spoutNumTasks = spoutNumTasksSupplier.get(parserConfig, Integer.class);
    int parserParallelism = parserParallelismSupplier.get(parserConfig, Integer.class);
    int parserNumTasks = parserNumTasksSupplier.get(parserConfig, Integer.class);
    int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfig, Integer.class);
    int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfig, Integer.class);
    Map<String, Object> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfig, Map.class);
    Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfig, String.class));
    // create the spout
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpout kafkaSpout = createKafkaSpout(zookeeperUrl, sensorType, securityProtocol, Optional.ofNullable(kafkaSpoutConfig), parserConfig);
    builder.setSpout("kafkaSpout", kafkaSpout, spoutParallelism).setNumTasks(spoutNumTasks);
    // create the parser bolt
    ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic);
    builder.setBolt("parserBolt", parserBolt, parserParallelism).setNumTasks(parserNumTasks).localOrShuffleGrouping("kafkaSpout");
    // create the error bolt, if needed
    if (errorWriterNumTasks > 0) {
        WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig);
        builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism).setNumTasks(errorWriterNumTasks).localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
    }
    return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
}

Example 17 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyBuilder method getSensorParserConfig.

/**
 * Fetch the parser configuration from Zookeeper.
 *
 * @param zookeeperUrl Zookeeper URL
 * @param sensorType   Type of sensor
 * @param configs
 * @return
 * @throws Exception
 */
private static SensorParserConfig getSensorParserConfig(String zookeeperUrl, String sensorType, ParserConfigurations configs) throws Exception {
    try (CuratorFramework client = ConfigurationsUtils.getClient(zookeeperUrl)) {
        client.start();
        ConfigurationsUtils.updateParserConfigsFromZookeeper(configs, client);
        SensorParserConfig parserConfig = configs.getSensorParserConfig(sensorType);
        if (parserConfig == null) {
            throw new IllegalStateException("Cannot find the parser configuration in zookeeper for " + sensorType + "." + "  Please check that it exists in zookeeper by using the 'zk_load_configs.sh -m DUMP' command.");
        }
        return parserConfig;
    }
}

Example 18 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testSecurityProtocol_fromSpout.

@Test
public void testSecurityProtocol_fromSpout() throws Exception {
    // Ultimately the order of precedence is CLI > spout config > parser config
    File extraConfig = File.createTempFile("spoutConfig", "json");
    extraConfig.deleteOnExit();
    writeMap(extraConfig, new HashMap<String, Object>() {

        {
            put("security.protocol", "PLAINTEXTSASL");
        }
    });
    {
        // Ensure that the CLI spout config takes precedence
        testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {

            {
                put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
                put(ParserTopologyCLI.ParserOptions.SECURITY_PROTOCOL, "PLAINTEXT");
            }
        }, input -> input.getSecurityProtocol().equals("PLAINTEXT"), () -> {
            SensorParserConfig config = getBaseConfig();
            config.setSecurityProtocol("PLAINTEXTSASL_FROM_ZK");
            return config;
        }, input -> input.getSecurityProtocol().equals("PLAINTEXTSASL_FROM_ZK"));
    }
    {
        // Ensure that the spout config takes precedence
        testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {

            {
                put(ParserTopologyCLI.ParserOptions.SPOUT_CONFIG, extraConfig.getAbsolutePath());
            }
        }, input -> input.getSecurityProtocol().equals("PLAINTEXTSASL"), () -> {
            SensorParserConfig config = getBaseConfig();
            config.setSecurityProtocol("PLAINTEXTSASL_FROM_ZK");
            return config;
        }, input -> input.getSecurityProtocol().equals("PLAINTEXTSASL_FROM_ZK"));
    }
}

Example 19 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testParserNumTasks.

@Test
public void testParserNumTasks() throws Exception {
    testConfigOption(ParserTopologyCLI.ParserOptions.PARSER_NUM_TASKS, "10", input -> input.getParserNumTasks().equals(10), () -> {
        SensorParserConfig config = getBaseConfig();
        config.setParserNumTasks(20);
        return config;
    }, input -> input.getParserNumTasks().equals(20));
}

Example 20 with SensorParserConfig

use of org.apache.metron.common.configuration.SensorParserConfig in project metron by apache.

the class ParserTopologyCLITest method testSpoutNumTasks.

@Test
public void testSpoutNumTasks() throws Exception {
    testConfigOption(ParserTopologyCLI.ParserOptions.SPOUT_NUM_TASKS, "10", input -> input.getSpoutNumTasks().equals(10), () -> {
        SensorParserConfig config = getBaseConfig();
        config.setSpoutNumTasks(20);
        return config;
    }, input -> input.getSpoutNumTasks().equals(20));
}