Search in sources :

Example 1 with DatapointParseBolt

use of org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt in project open-kilda by telstra.

the class OpenTSDBTopology method createTopology.

@Override
public StormTopology createTopology() {
    LOGGER.info("Creating OpenTSDB topology");
    TopologyBuilder tb = new TopologyBuilder();
    checkAndCreateTopic(topic);
    KafkaSpout kafkaSpout = createKafkaSpout(topic, spoutId);
    tb.setSpout(spoutId, kafkaSpout, config.getOpenTsdbNumSpouts());
    tb.setBolt(parseBoltId, new DatapointParseBolt(), config.getGetDatapointParseBoltExecutors()).setNumTasks(config.getGetDatapointParseBoltWorkers()).shuffleGrouping(spoutId);
    tb.setBolt(boltId, new OpenTSDBFilterBolt(), config.getOpenTsdbFilterBoltExecutors()).fieldsGrouping(parseBoltId, new Fields("hash"));
    OpenTsdbClient.Builder tsdbBuilder = OpenTsdbClient.newBuilder(config.getOpenTsDBHosts()).sync(config.getOpenTsdbTimeout()).returnDetails();
    OpenTsdbBolt openTsdbBolt = new OpenTsdbBolt(tsdbBuilder, Collections.singletonList(TupleOpenTsdbDatapointMapper.DEFAULT_MAPPER));
    openTsdbBolt.withBatchSize(config.getOpenTsdbBatchSize()).withFlushInterval(config.getOpenTsdbFlushInterval());
    // .failTupleForFailedMetrics();
    tb.setBolt("opentsdb", openTsdbBolt, config.getOpenTsdbBoltExecutors()).setNumTasks(config.getOpenTsdbBoltWorkers()).shuffleGrouping(boltId);
    return tb.createTopology();
}
Also used : DatapointParseBolt(org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt) Fields(org.apache.storm.tuple.Fields) OpenTsdbClient(org.apache.storm.opentsdb.client.OpenTsdbClient) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) OpenTsdbBolt(org.apache.storm.opentsdb.bolt.OpenTsdbBolt) KafkaSpout(org.apache.storm.kafka.spout.KafkaSpout) OpenTSDBFilterBolt(org.openkilda.wfm.topology.opentsdb.bolts.OpenTSDBFilterBolt)

Example 2 with DatapointParseBolt

use of org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt in project open-kilda by telstra.

the class OpenTsdbTopology method createTopology.

@Override
public StormTopology createTopology() {
    logger.info("Creating OpenTsdbTopology - {}", topologyName);
    TopologyBuilder tb = new TopologyBuilder();
    ZooKeeperSpout zooKeeperSpout = new ZooKeeperSpout(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig());
    declareSpout(tb, zooKeeperSpout, ZooKeeperSpout.SPOUT_ID);
    ZooKeeperBolt zooKeeperBolt = new ZooKeeperBolt(getConfig().getBlueGreenMode(), getZkTopoName(), getZookeeperConfig(), getBoltInstancesCount(OTSDB_PARSE_BOLT_ID));
    declareBolt(tb, zooKeeperBolt, ZooKeeperBolt.BOLT_ID).allGrouping(OTSDB_PARSE_BOLT_ID, ZkStreams.ZK.toString());
    attachInput(tb);
    OpenTsdbConfig openTsdbConfig = topologyConfig.getOpenTsdbConfig();
    declareBolt(tb, new DatapointParseBolt(), OTSDB_PARSE_BOLT_ID).shuffleGrouping(OTSDB_SPOUT_ID).allGrouping(ZooKeeperSpout.SPOUT_ID);
    declareBolt(tb, new OpenTSDBFilterBolt(), OTSDB_FILTER_BOLT_ID).fieldsGrouping(OTSDB_PARSE_BOLT_ID, new Fields("hash"));
    OpenTsdbClient.Builder tsdbBuilder = OpenTsdbClient.newBuilder(openTsdbConfig.getHosts()).returnDetails();
    if (openTsdbConfig.getClientChunkedRequestsEnabled()) {
        tsdbBuilder.enableChunkedEncoding();
    }
    OpenTsdbBolt openTsdbBolt = new OpenTsdbBolt(tsdbBuilder, Collections.singletonList(TupleOpenTsdbDatapointMapper.DEFAULT_MAPPER));
    openTsdbBolt.withBatchSize(openTsdbConfig.getBatchSize()).withFlushInterval(openTsdbConfig.getFlushInterval());
    declareBolt(tb, openTsdbBolt, OTSDB_BOLT_ID).shuffleGrouping(OTSDB_FILTER_BOLT_ID);
    return tb.createTopology();
}
Also used : DatapointParseBolt(org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt) Fields(org.apache.storm.tuple.Fields) OpenTsdbClient(org.apache.storm.opentsdb.client.OpenTsdbClient) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) ZooKeeperSpout(org.openkilda.wfm.share.zk.ZooKeeperSpout) ZooKeeperBolt(org.openkilda.wfm.share.zk.ZooKeeperBolt) OpenTsdbBolt(org.apache.storm.opentsdb.bolt.OpenTsdbBolt) OpenTsdbConfig(org.openkilda.wfm.topology.opentsdb.OpenTsdbTopologyConfig.OpenTsdbConfig) OpenTSDBFilterBolt(org.openkilda.wfm.topology.opentsdb.bolts.OpenTSDBFilterBolt)

Example 3 with DatapointParseBolt

use of org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt in project open-kilda by telstra.

the class OpenTsdbTopologyTest method activateDatapointParserBolt.

/**
 * Sets field `active` of DatapointParserBolt to true.
 * TODO Need to be replaced with normal activation by sending START signal or by testing services by unit test
 * At this moment we can't just send START signal to zookeeper spout because order of tuple processing is
 * unpredictable and bolt can handle START signal after handling of test tuple
 */
private void activateDatapointParserBolt(StormTopology stormTopology) throws IOException, ClassNotFoundException {
    // get bolt instance
    Bolt bolt = stormTopology.get_bolts().get(OTSDB_PARSE_BOLT_ID);
    byte[] serializedBolt = bolt.get_bolt_object().get_serialized_java();
    ObjectInput inputStream = new ObjectInputStream(new ByteArrayInputStream(serializedBolt));
    DatapointParseBolt datapointParseBolt = (DatapointParseBolt) inputStream.readObject();
    // activate bolt
    datapointParseBolt.active = true;
    // serialize bolt
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ObjectOutput outputStream = new ObjectOutputStream(byteArrayOutputStream);
    outputStream.writeObject(datapointParseBolt);
    byte[] updatedBolt = byteArrayOutputStream.toByteArray();
    outputStream.close();
    byteArrayOutputStream.close();
    // replace old bolt with new
    bolt.get_bolt_object().set_serialized_java(updatedBolt);
}
Also used : DatapointParseBolt(org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt) ObjectOutput(java.io.ObjectOutput) ByteArrayInputStream(java.io.ByteArrayInputStream) ZooKeeperBolt(org.openkilda.wfm.share.zk.ZooKeeperBolt) DatapointParseBolt(org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt) Bolt(org.apache.storm.generated.Bolt) ObjectInput(java.io.ObjectInput) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ObjectOutputStream(java.io.ObjectOutputStream) ObjectInputStream(java.io.ObjectInputStream)

Aggregations

DatapointParseBolt (org.openkilda.wfm.topology.opentsdb.bolts.DatapointParseBolt)3 OpenTsdbBolt (org.apache.storm.opentsdb.bolt.OpenTsdbBolt)2 OpenTsdbClient (org.apache.storm.opentsdb.client.OpenTsdbClient)2 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)2 Fields (org.apache.storm.tuple.Fields)2 ZooKeeperBolt (org.openkilda.wfm.share.zk.ZooKeeperBolt)2 OpenTSDBFilterBolt (org.openkilda.wfm.topology.opentsdb.bolts.OpenTSDBFilterBolt)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 ObjectInput (java.io.ObjectInput)1 ObjectInputStream (java.io.ObjectInputStream)1 ObjectOutput (java.io.ObjectOutput)1 ObjectOutputStream (java.io.ObjectOutputStream)1 Bolt (org.apache.storm.generated.Bolt)1 KafkaSpout (org.apache.storm.kafka.spout.KafkaSpout)1 ZooKeeperSpout (org.openkilda.wfm.share.zk.ZooKeeperSpout)1 OpenTsdbConfig (org.openkilda.wfm.topology.opentsdb.OpenTsdbTopologyConfig.OpenTsdbConfig)1