Examples with KafkaSpout org.apache.storm.kafka.spout.KafkaSpout used on opensource projects

Example 6 with KafkaSpout

use of org.apache.storm.kafka.spout.KafkaSpout in project metron by apache.

the class ParserTopologyBuilder method build.

/**
 * Builds a Storm topology that parses telemetry data received from an external sensor.
 *
 * @param zookeeperUrl             Zookeeper URL
 * @param brokerUrl                Kafka Broker URL
 * @param sensorType               Type of sensor
 * @param spoutParallelismSupplier         Supplier for the parallelism hint for the spout
 * @param spoutNumTasksSupplier            Supplier for the number of tasks for the spout
 * @param parserParallelismSupplier        Supplier for the parallelism hint for the parser bolt
 * @param parserNumTasksSupplier           Supplier for the number of tasks for the parser bolt
 * @param errorWriterParallelismSupplier   Supplier for the parallelism hint for the bolt that handles errors
 * @param errorWriterNumTasksSupplier      Supplier for the number of tasks for the bolt that handles errors
 * @param kafkaSpoutConfigSupplier         Supplier for the configuration options for the kafka spout
 * @param securityProtocolSupplier         Supplier for the security protocol
 * @param outputTopic                      The output kafka topic
 * @param stormConfigSupplier              Supplier for the storm config
 * @return A Storm topology that parses telemetry data received from an external sensor
 * @throws Exception
 */
public static ParserTopology build(String zookeeperUrl, Optional<String> brokerUrl, String sensorType, ValueSupplier<Integer> spoutParallelismSupplier, ValueSupplier<Integer> spoutNumTasksSupplier, ValueSupplier<Integer> parserParallelismSupplier, ValueSupplier<Integer> parserNumTasksSupplier, ValueSupplier<Integer> errorWriterParallelismSupplier, ValueSupplier<Integer> errorWriterNumTasksSupplier, ValueSupplier<Map> kafkaSpoutConfigSupplier, ValueSupplier<String> securityProtocolSupplier, Optional<String> outputTopic, ValueSupplier<Config> stormConfigSupplier) throws Exception {
    // fetch configuration from zookeeper
    ParserConfigurations configs = new ParserConfigurations();
    SensorParserConfig parserConfig = getSensorParserConfig(zookeeperUrl, sensorType, configs);
    int spoutParallelism = spoutParallelismSupplier.get(parserConfig, Integer.class);
    int spoutNumTasks = spoutNumTasksSupplier.get(parserConfig, Integer.class);
    int parserParallelism = parserParallelismSupplier.get(parserConfig, Integer.class);
    int parserNumTasks = parserNumTasksSupplier.get(parserConfig, Integer.class);
    int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfig, Integer.class);
    int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfig, Integer.class);
    Map<String, Object> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfig, Map.class);
    Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfig, String.class));
    // create the spout
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpout kafkaSpout = createKafkaSpout(zookeeperUrl, sensorType, securityProtocol, Optional.ofNullable(kafkaSpoutConfig), parserConfig);
    builder.setSpout("kafkaSpout", kafkaSpout, spoutParallelism).setNumTasks(spoutNumTasks);
    // create the parser bolt
    ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig, outputTopic);
    builder.setBolt("parserBolt", parserBolt, parserParallelism).setNumTasks(parserNumTasks).localOrShuffleGrouping("kafkaSpout");
    // create the error bolt, if needed
    if (errorWriterNumTasks > 0) {
        WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorType, securityProtocol, configs, parserConfig);
        builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism).setNumTasks(errorWriterNumTasks).localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
    }
    return new ParserTopology(builder, stormConfigSupplier.get(parserConfig, Config.class));
}

Example 7 with KafkaSpout

use of org.apache.storm.kafka.spout.KafkaSpout in project metron by apache.

the class ParserTopologyBuilder method build.

/**
 * Builds a Storm topology that parses telemetry data received from an external sensor.
 *
 * @param zookeeperUrl             Zookeeper URL
 * @param brokerUrl                Kafka Broker URL
 * @param sensorTypes               Type of sensor
 * @param spoutParallelismSupplier         Supplier for the parallelism hint for the spout
 * @param spoutNumTasksSupplier            Supplier for the number of tasks for the spout
 * @param parserParallelismSupplier        Supplier for the parallelism hint for the parser bolt
 * @param parserNumTasksSupplier           Supplier for the number of tasks for the parser bolt
 * @param errorWriterParallelismSupplier   Supplier for the parallelism hint for the bolt that handles errors
 * @param errorWriterNumTasksSupplier      Supplier for the number of tasks for the bolt that handles errors
 * @param kafkaSpoutConfigSupplier         Supplier for the configuration options for the kafka spout
 * @param securityProtocolSupplier         Supplier for the security protocol
 * @param outputTopicSupplier              Supplier for the output kafka topic
 * @param stormConfigSupplier              Supplier for the storm config
 * @return A Storm topology that parses telemetry data received from an external sensor
 * @throws Exception
 */
public static ParserTopology build(String zookeeperUrl, Optional<String> brokerUrl, List<String> sensorTypes, ValueSupplier<List> spoutParallelismSupplier, ValueSupplier<List> spoutNumTasksSupplier, ValueSupplier<Integer> parserParallelismSupplier, ValueSupplier<Integer> parserNumTasksSupplier, ValueSupplier<Integer> errorWriterParallelismSupplier, ValueSupplier<Integer> errorWriterNumTasksSupplier, ValueSupplier<List> kafkaSpoutConfigSupplier, ValueSupplier<String> securityProtocolSupplier, ValueSupplier<String> outputTopicSupplier, ValueSupplier<String> errorTopicSupplier, ValueSupplier<Config> stormConfigSupplier) throws Exception {
    // fetch configuration from zookeeper
    ParserConfigurations configs = new ParserConfigurations();
    Map<String, SensorParserConfig> sensorToParserConfigs = getSensorParserConfig(zookeeperUrl, sensorTypes, configs);
    Collection<SensorParserConfig> parserConfigs = sensorToParserConfigs.values();
    @SuppressWarnings("unchecked") List<Integer> spoutParallelism = (List<Integer>) spoutParallelismSupplier.get(parserConfigs, List.class);
    @SuppressWarnings("unchecked") List<Integer> spoutNumTasks = (List<Integer>) spoutNumTasksSupplier.get(parserConfigs, List.class);
    int parserParallelism = parserParallelismSupplier.get(parserConfigs, Integer.class);
    int parserNumTasks = parserNumTasksSupplier.get(parserConfigs, Integer.class);
    int errorWriterParallelism = errorWriterParallelismSupplier.get(parserConfigs, Integer.class);
    int errorWriterNumTasks = errorWriterNumTasksSupplier.get(parserConfigs, Integer.class);
    String outputTopic = outputTopicSupplier.get(parserConfigs, String.class);
    List<Map<String, Object>> kafkaSpoutConfig = kafkaSpoutConfigSupplier.get(parserConfigs, List.class);
    Optional<String> securityProtocol = Optional.ofNullable(securityProtocolSupplier.get(parserConfigs, String.class));
    // create the spout
    TopologyBuilder builder = new TopologyBuilder();
    int i = 0;
    List<String> spoutIds = new ArrayList<>();
    for (Entry<String, SensorParserConfig> entry : sensorToParserConfigs.entrySet()) {
        KafkaSpout kafkaSpout = createKafkaSpout(zookeeperUrl, entry.getKey(), securityProtocol, Optional.ofNullable(kafkaSpoutConfig.get(i)), entry.getValue());
        String spoutId = sensorToParserConfigs.size() > 1 ? "kafkaSpout-" + entry.getKey() : "kafkaSpout";
        builder.setSpout(spoutId, kafkaSpout, spoutParallelism.get(i)).setNumTasks(spoutNumTasks.get(i));
        spoutIds.add(spoutId);
        ++i;
    }
    // create the parser bolt
    ParserBolt parserBolt = createParserBolt(zookeeperUrl, brokerUrl, sensorToParserConfigs, securityProtocol, configs, Optional.ofNullable(outputTopic));
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", parserBolt, parserParallelism).setNumTasks(parserNumTasks);
    for (String spoutId : spoutIds) {
        boltDeclarer.localOrShuffleGrouping(spoutId);
    }
    // create the error bolt, if needed
    if (errorWriterNumTasks > 0) {
        String errorTopic = errorTopicSupplier.get(parserConfigs, String.class);
        WriterBolt errorBolt = createErrorBolt(zookeeperUrl, brokerUrl, sensorTypes.get(0), securityProtocol, configs, parserConfigs.iterator().next(), errorTopic);
        builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism).setNumTasks(errorWriterNumTasks).localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
    }
    return new ParserTopology(builder, stormConfigSupplier.get(parserConfigs, Config.class));
}

Example 8 with KafkaSpout

use of org.apache.storm.kafka.spout.KafkaSpout in project storm by apache.

the class KafkaClientHdfsTopo method getTopology.

static StormTopology getTopology(Map<String, Object> config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
    // 1 -  Setup Kafka Spout   --------
    String bootstrapHosts = getStr(config, KAFKA_BOOTSTRAP_HOSTS);
    String topicName = getStr(config, KAFKA_TOPIC);
    KafkaSpoutConfig<String, String> spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts, topicName).setFirstPollOffsetStrategy(FirstPollOffsetStrategy.EARLIEST).build();
    KafkaSpout<String, String> spout = new KafkaSpout<>(spoutConfig);
    // 2 -  Setup HFS Bolt   --------
    String hdfsUrls = getStr(config, HDFS_URI);
    RecordFormat format = new LineWriter("value");
    SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
    // Instantiate the HdfsBolt
    HdfsBolt bolt = new HdfsBolt().withFsUrl(hdfsUrls).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}

Example 9 with KafkaSpout

use of org.apache.storm.kafka.spout.KafkaSpout in project storm by apache.

the class KafkaClientSpoutNullBoltTopo method getTopology.

/**
 * Create and configure the topology.
 */
public static StormTopology getTopology(Map<String, Object> config) {
    final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    // 1 -  Setup Kafka Spout   --------
    String bootstrapServers = Optional.ofNullable(Helper.getStr(config, BOOTSTRAP_SERVERS)).orElse("127.0.0.1:9092");
    String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC)).orElse("storm-perf-null-bolt-topic");
    ProcessingGuarantee processingGuarantee = ProcessingGuarantee.valueOf(Optional.ofNullable(Helper.getStr(config, PROCESSING_GUARANTEE)).orElse(ProcessingGuarantee.AT_LEAST_ONCE.name()));
    int offsetCommitPeriodMs = Helper.getInt(config, OFFSET_COMMIT_PERIOD_MS, 30_000);
    KafkaSpoutConfig<String, String> kafkaSpoutConfig = KafkaSpoutConfig.builder(bootstrapServers, kafkaTopic).setProcessingGuarantee(processingGuarantee).setOffsetCommitPeriodMs(offsetCommitPeriodMs).setFirstPollOffsetStrategy(FirstPollOffsetStrategy.EARLIEST).setTupleTrackingEnforced(true).build();
    KafkaSpout<String, String> spout = new KafkaSpout<>(kafkaSpoutConfig);
    // 2 -   DevNull Bolt   --------
    DevNullBolt bolt = new DevNullBolt();
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}