Examples with KafkaBolt org.apache.storm.kafka.bolt.KafkaBolt used on opensource projects

Example 21 with KafkaBolt

use of org.apache.storm.kafka.bolt.KafkaBolt in project open-kilda by telstra.

the class RerouteTopology method metrics.

private void metrics(TopologyBuilder topologyBuilder) {
    String openTsdbTopic = topologyConfig.getKafkaTopics().getOtsdbTopic();
    KafkaBolt kafkaBolt = createKafkaBolt(openTsdbTopic);
    declareBolt(topologyBuilder, kafkaBolt, METRICS_BOLT_ID).shuffleGrouping(RerouteBolt.BOLT_ID, RerouteBolt.STREAM_TO_METRICS_BOLT);
}

Example 22 with KafkaBolt

use of org.apache.storm.kafka.bolt.KafkaBolt in project open-kilda by telstra.

the class PortStateTopology method createTopology.

@Override
public StormTopology createTopology() {
    logger.info("Creating PortStateTopology - {}", topologyName);
    TopologyBuilder builder = new TopologyBuilder();
    /*
         *  Topology:
         *
         *  TOPO_DISCO_SPOUT ---> TopoDiscoParseBolt ---> ParsePortInfoBolt ---> OtsdbKafkaBolt(kilda.otsdb topic)
         *                                                         ^
         *                                                         |
         *  WFM_STATS_SPOUT ---> WfmStatsParseBolt -----------------
         *
         *
         *  SwitchPortsSpout ---> SpeakerKafkaBolt(kilda.speaker topic)
         *
         */
    ZooKeeperSpout zooKeeperSpout = new ZooKeeperSpout(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig());
    declareSpout(builder, zooKeeperSpout, ZooKeeperSpout.SPOUT_ID);
    ZooKeeperBolt zooKeeperBolt = new ZooKeeperBolt(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig(), getBoltInstancesCount(TOPO_DISCO_PARSE_BOLT_NAME, WFM_STATS_PARSE_BOLT_NAME, REQUEST_SPEAKER_BOLT_NAME));
    declareBolt(builder, zooKeeperBolt, ZooKeeperBolt.BOLT_ID).allGrouping(TOPO_DISCO_PARSE_BOLT_NAME, ZkStreams.ZK.toString()).allGrouping(WFM_STATS_PARSE_BOLT_NAME, ZkStreams.ZK.toString()).allGrouping(REQUEST_SPEAKER_BOLT_NAME, ZkStreams.ZK.toString());
    // Setup spout and bolt for TOPO_DISCO_SPOUT line
    String topoDiscoTopic = topologyConfig.getKafkaTopoDiscoTopic();
    logger.debug("connecting to {} topic", topoDiscoTopic);
    declareKafkaSpout(builder, topoDiscoTopic, TOPO_DISCO_SPOUT);
    TopoDiscoParseBolt topoDiscoParseBolt = new TopoDiscoParseBolt(ZooKeeperSpout.SPOUT_ID);
    declareBolt(builder, topoDiscoParseBolt, TOPO_DISCO_PARSE_BOLT_NAME).shuffleGrouping(TOPO_DISCO_SPOUT).allGrouping(ZooKeeperSpout.SPOUT_ID);
    ParsePortInfoBolt parsePortInfoBolt = new ParsePortInfoBolt(topologyConfig.getMetricPrefix());
    declareBolt(builder, parsePortInfoBolt, PARSE_PORT_INFO_BOLT_NAME).shuffleGrouping(TOPO_DISCO_PARSE_BOLT_NAME, TopoDiscoParseBolt.TOPO_TO_PORT_INFO_STREAM).shuffleGrouping(WFM_STATS_PARSE_BOLT_NAME, WfmStatsParseBolt.WFM_TO_PARSE_PORT_INFO_STREAM);
    String openTsdbTopic = topologyConfig.getKafkaOtsdbTopic();
    KafkaBolt openTsdbBolt = createKafkaBolt(openTsdbTopic);
    declareBolt(builder, openTsdbBolt, OTSDB_KAFKA_BOLT_NAME).shuffleGrouping(PARSE_PORT_INFO_BOLT_NAME);
    // Setup spout and bolt for WFM_STATS_SPOUT line
    String wfmStatsTopic = topologyConfig.getKafkaStatsTopic();
    logger.debug("connecting to {} topic", wfmStatsTopic);
    declareKafkaSpout(builder, wfmStatsTopic, WFM_STATS_SPOUT);
    WfmStatsParseBolt wfmStatsParseBolt = new WfmStatsParseBolt(ZooKeeperSpout.SPOUT_ID);
    declareBolt(builder, wfmStatsParseBolt, WFM_STATS_PARSE_BOLT_NAME).shuffleGrouping(WFM_STATS_SPOUT).allGrouping(ZooKeeperSpout.SPOUT_ID);
    // Setup spout and bolt for sending SwitchPortsCommand every frequency seconds
    SwitchPortsSpout switchPortsSpout = new SwitchPortsSpout(JANITOR_REFRESH);
    declareSpout(builder, switchPortsSpout, SWITCH_PORTS_SPOUT_NAME);
    RequestSpeakerBolt bolt = new RequestSpeakerBolt(ZooKeeperSpout.SPOUT_ID);
    declareBolt(builder, bolt, REQUEST_SPEAKER_BOLT_NAME).shuffleGrouping(SWITCH_PORTS_SPOUT_NAME).allGrouping(ZooKeeperSpout.SPOUT_ID);
    String speakerTopic = topologyConfig.getKafkaSpeakerTopic();
    KafkaBolt speakerBolt = buildKafkaBolt(speakerTopic);
    declareBolt(builder, speakerBolt, SPEAKER_KAFKA_BOLT_NAME).shuffleGrouping(REQUEST_SPEAKER_BOLT_NAME);
    return builder.createTopology();
}

Example 23 with KafkaBolt

use of org.apache.storm.kafka.bolt.KafkaBolt in project open-kilda by telstra.

the class NbWorkerTopology method createTopology.

@Override
public StormTopology createTopology() {
    logger.info("Creating NbWorkerTopology - {}", topologyName);
    TopologyBuilder tb = new TopologyBuilder();
    declareSpout(tb, new CoordinatorSpout(), CoordinatorSpout.ID);
    declareBolt(tb, new CoordinatorBolt(), CoordinatorBolt.ID).allGrouping(CoordinatorSpout.ID).fieldsGrouping(FlowMeterModifyHubBolt.ID, CoordinatorBolt.INCOME_STREAM, FIELDS_KEY).fieldsGrouping(METER_MODIFY_WORKER_BOLT, CoordinatorBolt.INCOME_STREAM, FIELDS_KEY);
    declareKafkaSpout(tb, topologyConfig.getKafkaTopoNbTopic(), NB_SPOUT_ID);
    RouterBolt router = new RouterBolt(ZooKeeperSpout.SPOUT_ID);
    declareBolt(tb, router, ROUTER_BOLT_NAME).shuffleGrouping(NB_SPOUT_ID).allGrouping(ZooKeeperSpout.SPOUT_ID);
    ZooKeeperSpout zooKeeperSpout = new ZooKeeperSpout(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig());
    declareSpout(tb, zooKeeperSpout, ZooKeeperSpout.SPOUT_ID);
    ZooKeeperBolt zooKeeperBolt = new ZooKeeperBolt(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig(), getBoltInstancesCount(ROUTER_BOLT_NAME, FlowMeterModifyHubBolt.ID));
    declareBolt(tb, zooKeeperBolt, ZooKeeperBolt.BOLT_ID).allGrouping(ROUTER_BOLT_NAME, ZkStreams.ZK.toString()).allGrouping(FlowMeterModifyHubBolt.ID, ZkStreams.ZK.toString());
    PersistenceManager persistenceManager = new PersistenceManager(configurationProvider);
    PathComputerConfig pathComputerConfig = configurationProvider.getConfiguration(PathComputerConfig.class);
    HubBolt.Config meterModifyHubConfig = HubBolt.Config.builder().requestSenderComponent(ROUTER_BOLT_NAME).workerComponent(METER_MODIFY_WORKER_BOLT).lifeCycleEventComponent(ZooKeeperSpout.SPOUT_ID).timeoutMs((int) TimeUnit.SECONDS.toMillis(topologyConfig.getProcessTimeout())).build();
    declareBolt(tb, new FlowMeterModifyHubBolt(meterModifyHubConfig, persistenceManager), FlowMeterModifyHubBolt.ID).fieldsGrouping(ROUTER_BOLT_NAME, FlowMeterModifyHubBolt.INCOME_STREAM, FIELDS_KEY).directGrouping(METER_MODIFY_WORKER_BOLT, FlowMeterModifyHubBolt.INCOME_STREAM).directGrouping(CoordinatorBolt.ID).allGrouping(ZooKeeperSpout.SPOUT_ID);
    WorkerBolt.Config speakerMeterModifyWorkerConfig = WorkerBolt.Config.builder().hubComponent(FlowMeterModifyHubBolt.ID).streamToHub(FlowMeterModifyHubBolt.INCOME_STREAM).workerSpoutComponent(ROUTER_BOLT_NAME).defaultTimeout((int) TimeUnit.SECONDS.toMillis(topologyConfig.getOperationTimeout())).build();
    declareBolt(tb, new SpeakerWorkerBolt(speakerMeterModifyWorkerConfig), METER_MODIFY_WORKER_BOLT).fieldsGrouping(ROUTER_BOLT_NAME, SpeakerWorkerBolt.INCOME_STREAM, FIELDS_KEY).fieldsGrouping(FlowMeterModifyHubBolt.ID, StreamType.METER_MODIFY_WORKER.toString(), FIELDS_KEY).directGrouping(CoordinatorBolt.ID);
    SwitchOperationsBolt switchesBolt = new SwitchOperationsBolt(persistenceManager);
    declareBolt(tb, switchesBolt, SWITCHES_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.SWITCH.toString());
    LinkOperationsBolt linksBolt = new LinkOperationsBolt(persistenceManager);
    declareBolt(tb, linksBolt, LINKS_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.ISL.toString());
    FlowOperationsBolt flowsBolt = new FlowOperationsBolt(persistenceManager);
    declareBolt(tb, flowsBolt, FLOWS_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.FLOW.toString());
    FlowPatchBolt flowPatchBolt = new FlowPatchBolt(persistenceManager);
    declareBolt(tb, flowPatchBolt, FLOW_PATCH_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.FLOW_PATCH.toString());
    FeatureTogglesBolt featureTogglesBolt = new FeatureTogglesBolt(persistenceManager);
    declareBolt(tb, featureTogglesBolt, FEATURE_TOGGLES_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.FEATURE_TOGGLES.toString());
    KildaConfigurationBolt kildaConfigurationBolt = new KildaConfigurationBolt(persistenceManager);
    declareBolt(tb, kildaConfigurationBolt, KILDA_CONFIG_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.KILDA_CONFIG.toString());
    PathsBolt pathsBolt = new PathsBolt(persistenceManager, pathComputerConfig);
    declareBolt(tb, pathsBolt, PATHS_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.PATHS.toString());
    HistoryOperationsBolt historyBolt = new HistoryOperationsBolt(persistenceManager);
    declareBolt(tb, historyBolt, HISTORY_BOLT_NAME).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.HISTORY.toString());
    ResponseSplitterBolt splitterBolt = new ResponseSplitterBolt();
    declareBolt(tb, splitterBolt, SPLITTER_BOLT_NAME).shuffleGrouping(SWITCHES_BOLT_NAME).shuffleGrouping(LINKS_BOLT_NAME).shuffleGrouping(FLOWS_BOLT_NAME).shuffleGrouping(FEATURE_TOGGLES_BOLT_NAME).shuffleGrouping(KILDA_CONFIG_BOLT_NAME).shuffleGrouping(PATHS_BOLT_NAME).shuffleGrouping(HISTORY_BOLT_NAME).shuffleGrouping(FlowMeterModifyHubBolt.ID).shuffleGrouping(FLOW_PATCH_BOLT_NAME);
    MessageEncoder messageEncoder = new MessageEncoder();
    declareBolt(tb, messageEncoder, MESSAGE_ENCODER_BOLT_NAME).shuffleGrouping(LINKS_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(LINKS_BOLT_NAME, StreamType.REROUTE.toString()).shuffleGrouping(FLOWS_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(FLOWS_BOLT_NAME, StreamType.REROUTE.toString()).shuffleGrouping(SWITCHES_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(SWITCHES_BOLT_NAME, StreamType.REROUTE.toString()).shuffleGrouping(SWITCHES_BOLT_NAME, StreamType.TO_SWITCH_MANAGER.toString()).shuffleGrouping(ROUTER_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(FEATURE_TOGGLES_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(FEATURE_TOGGLES_BOLT_NAME, StreamType.TO_SWITCH_MANAGER.toString()).shuffleGrouping(KILDA_CONFIG_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(PATHS_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(HISTORY_BOLT_NAME, StreamType.ERROR.toString()).shuffleGrouping(FlowMeterModifyHubBolt.ID, StreamType.ERROR.toString()).shuffleGrouping(FLOW_PATCH_BOLT_NAME, StreamType.ERROR.toString());
    DiscoveryEncoderBolt discoveryEncoder = new DiscoveryEncoderBolt();
    declareBolt(tb, discoveryEncoder, DISCOVERY_ENCODER_BOLT_NAME).shuffleGrouping(LINKS_BOLT_NAME, StreamType.DISCO.toString()).shuffleGrouping(SWITCHES_BOLT_NAME, StreamType.DISCO.toString()).shuffleGrouping(FEATURE_TOGGLES_BOLT_NAME, FeatureTogglesBolt.STREAM_NOTIFICATION_ID);
    KafkaBolt kafkaNbBolt = buildKafkaBolt(topologyConfig.getKafkaNorthboundTopic());
    declareBolt(tb, kafkaNbBolt, NB_KAFKA_BOLT_NAME).fieldsGrouping(SPLITTER_BOLT_NAME, FIELDS_KEY).shuffleGrouping(MESSAGE_ENCODER_BOLT_NAME, StreamType.ERROR.toString());
    KafkaBolt kafkaFlowHsBolt = buildKafkaBolt(topologyConfig.getKafkaFlowHsTopic());
    declareBolt(tb, kafkaFlowHsBolt, FLOW_HS_KAFKA_BOLT_NAME).shuffleGrouping(MESSAGE_ENCODER_BOLT_NAME, StreamType.FLOWHS.toString()).shuffleGrouping(FLOW_PATCH_BOLT_NAME, StreamType.FLOWHS.toString());
    KafkaBolt kafkaDiscoBolt = buildKafkaBolt(topologyConfig.getKafkaDiscoTopic());
    declareBolt(tb, kafkaDiscoBolt, DISCO_KAFKA_BOLT_NAME).shuffleGrouping(DISCOVERY_ENCODER_BOLT_NAME);
    KafkaBolt kafkaPingBolt = buildKafkaBolt(topologyConfig.getKafkaPingTopic());
    declareBolt(tb, kafkaPingBolt, PING_KAFKA_BOLT_NAME).shuffleGrouping(FLOW_PATCH_BOLT_NAME, StreamType.PING.toString());
    declareBolt(tb, buildKafkaBolt(topologyConfig.getKafkaSpeakerTopic()), SPEAKER_KAFKA_BOLT).shuffleGrouping(METER_MODIFY_WORKER_BOLT, StreamType.TO_SPEAKER.toString());
    declareBolt(tb, buildKafkaBolt(topologyConfig.getKafkaSwitchManagerTopic()), SWITCH_MANAGER_KAFKA_BOLT).shuffleGrouping(MESSAGE_ENCODER_BOLT_NAME, StreamType.TO_SWITCH_MANAGER.toString());
    Server42EncoderBolt server42EncoderBolt = new Server42EncoderBolt();
    declareBolt(tb, server42EncoderBolt, SERVER42_ENCODER_BOLT_NAME).shuffleGrouping(SWITCHES_BOLT_NAME, StreamType.TO_SERVER42.toString()).shuffleGrouping(FEATURE_TOGGLES_BOLT_NAME, FeatureTogglesBolt.STREAM_NOTIFICATION_ID);
    declareBolt(tb, buildKafkaBolt(topologyConfig.getKafkaServer42StormNotifyTopic()), SERVER42_KAFKA_BOLT).shuffleGrouping(SERVER42_ENCODER_BOLT_NAME);
    declareBolt(tb, buildKafkaBolt(topologyConfig.getKafkaRerouteTopic()), REROUTE_KAFKA_BOLT).shuffleGrouping(MESSAGE_ENCODER_BOLT_NAME, StreamType.REROUTE.toString());
    metrics(tb);
    return tb.createTopology();
}

Example 24 with KafkaBolt

use of org.apache.storm.kafka.bolt.KafkaBolt in project open-kilda by telstra.

the class FlowHsTopology method pingOutput.

private void pingOutput(TopologyBuilder topologyBuilder) {
    KafkaBolt pingKafkaBolt = buildKafkaBolt(getConfig().getKafkaPingTopic());
    declareBolt(topologyBuilder, pingKafkaBolt, ComponentId.FLOW_PING_SENDER.name()).shuffleGrouping(ComponentId.FLOW_CREATE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.FLOW_UPDATE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.FLOW_DELETE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.FLOW_REROUTE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.FLOW_PATH_SWAP_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.YFLOW_CREATE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.YFLOW_UPDATE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.YFLOW_REROUTE_HUB.name(), Stream.HUB_TO_PING_SENDER.name()).shuffleGrouping(ComponentId.YFLOW_DELETE_HUB.name(), Stream.HUB_TO_PING_SENDER.name());
}

Example 25 with KafkaBolt

use of org.apache.storm.kafka.bolt.KafkaBolt in project open-kilda by telstra.

the class FlowMonitoringTopology method outputReroute.

private void outputReroute(TopologyBuilder topology) {
    RerouteEncoder bolt = new RerouteEncoder();
    declareBolt(topology, bolt, RerouteEncoder.BOLT_ID).shuffleGrouping(ComponentId.ACTION_BOLT.name());
    KafkaBolt output = buildKafkaBolt(getConfig().getKafkaTopoRerouteTopic());
    declareBolt(topology, output, ComponentId.REROUTE_BOLT.name()).shuffleGrouping(RerouteEncoder.BOLT_ID);
}