Examples with ValueSupplier org.apache.metron.parsers.topology.config.ValueSupplier used on opensource projects

Example 1 with ValueSupplier

use of org.apache.metron.parsers.topology.config.ValueSupplier in project metron by apache.

the class ParserTopologyCLI method createParserTopology.

public ParserTopologyBuilder.ParserTopology createParserTopology(final CommandLine cmd) throws Exception {
    String zookeeperUrl = ParserOptions.ZK_QUORUM.get(cmd);
    Optional<String> brokerUrl = ParserOptions.BROKER_URL.has(cmd) ? Optional.of(ParserOptions.BROKER_URL.get(cmd)) : Optional.empty();
    String sensorType = ParserOptions.SENSOR_TYPE.get(cmd);
    /*
    It bears mentioning why we're creating this ValueSupplier indirection here.
    As a separation of responsibilities, the CLI class defines the order of precedence
    for the various topological and structural properties for creating a parser.  This is
    desirable because there are now (i.e. integration tests)
    and may be in the future (i.e. a REST service to start parsers without using the CLI)
    other mechanisms to construct parser topologies.  It's sensible to split those concerns..

    Unfortunately, determining the structural parameters for a parser requires interacting with
    external services (e.g. zookeeper) that are set up well within the ParserTopology class.
    Rather than pulling the infrastructure to interact with those services out and moving it into the
    CLI class and breaking that separation of concerns, we've created a supplier
    indirection where are providing the logic as to how to create precedence in the CLI class
    without owning the responsibility of constructing the infrastructure where the values are
    necessarily supplied.

     */
    ValueSupplier<Integer> spoutParallelism = (parserConfig, clazz) -> {
        if (ParserOptions.SPOUT_PARALLELISM.has(cmd)) {
            return Integer.parseInt(ParserOptions.SPOUT_PARALLELISM.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getSpoutParallelism()).orElse(1);
    };
    ValueSupplier<Integer> spoutNumTasks = (parserConfig, clazz) -> {
        if (ParserOptions.SPOUT_NUM_TASKS.has(cmd)) {
            return Integer.parseInt(ParserOptions.SPOUT_NUM_TASKS.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getSpoutNumTasks()).orElse(1);
    };
    ValueSupplier<Integer> parserParallelism = (parserConfig, clazz) -> {
        if (ParserOptions.PARSER_PARALLELISM.has(cmd)) {
            return Integer.parseInt(ParserOptions.PARSER_PARALLELISM.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getParserParallelism()).orElse(1);
    };
    ValueSupplier<Integer> parserNumTasks = (parserConfig, clazz) -> {
        if (ParserOptions.PARSER_NUM_TASKS.has(cmd)) {
            return Integer.parseInt(ParserOptions.PARSER_NUM_TASKS.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getParserNumTasks()).orElse(1);
    };
    ValueSupplier<Integer> errorParallelism = (parserConfig, clazz) -> {
        if (ParserOptions.ERROR_WRITER_PARALLELISM.has(cmd)) {
            return Integer.parseInt(ParserOptions.ERROR_WRITER_PARALLELISM.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getErrorWriterParallelism()).orElse(1);
    };
    ValueSupplier<Integer> errorNumTasks = (parserConfig, clazz) -> {
        if (ParserOptions.ERROR_WRITER_NUM_TASKS.has(cmd)) {
            return Integer.parseInt(ParserOptions.ERROR_WRITER_NUM_TASKS.get(cmd, "1"));
        }
        return Optional.ofNullable(parserConfig.getErrorWriterNumTasks()).orElse(1);
    };
    ValueSupplier<Map> spoutConfig = (parserConfig, clazz) -> {
        if (ParserOptions.SPOUT_CONFIG.has(cmd)) {
            return readJSONMapFromFile(new File(ParserOptions.SPOUT_CONFIG.get(cmd)));
        }
        return Optional.ofNullable(parserConfig.getSpoutConfig()).orElse(new HashMap<>());
    };
    ValueSupplier<String> securityProtocol = (parserConfig, clazz) -> {
        Optional<String> sp = Optional.empty();
        if (ParserOptions.SECURITY_PROTOCOL.has(cmd)) {
            sp = Optional.of(ParserOptions.SECURITY_PROTOCOL.get(cmd));
        }
        if (!sp.isPresent()) {
            sp = getSecurityProtocol(sp, spoutConfig.get(parserConfig, Map.class));
        }
        return sp.orElse(Optional.ofNullable(parserConfig.getSecurityProtocol()).orElse(null));
    };
    ValueSupplier<Config> stormConf = (parserConfig, clazz) -> {
        Map<String, Object> c = parserConfig.getStormConfig();
        Config finalConfig = new Config();
        if (c != null && !c.isEmpty()) {
            finalConfig.putAll(c);
        }
        if (parserConfig.getNumAckers() != null) {
            Config.setNumAckers(finalConfig, parserConfig.getNumAckers());
        }
        if (parserConfig.getNumWorkers() != null) {
            Config.setNumWorkers(finalConfig, parserConfig.getNumWorkers());
        }
        return ParserOptions.getConfig(cmd, finalConfig).orElse(finalConfig);
    };
    Optional<String> outputTopic = ParserOptions.OUTPUT_TOPIC.has(cmd) ? Optional.of(ParserOptions.OUTPUT_TOPIC.get(cmd)) : Optional.empty();
    return getParserTopology(zookeeperUrl, brokerUrl, sensorType, spoutParallelism, spoutNumTasks, parserParallelism, parserNumTasks, errorParallelism, errorNumTasks, spoutConfig, securityProtocol, stormConf, outputTopic);
}