Examples with Bolt org.apache.storm.generated.Bolt used on opensource projects

Example 11 with Bolt

use of org.apache.storm.generated.Bolt in project storm by apache.

the class TestTridentTopology method testGenBoltId.

@Test
public void testGenBoltId() {
    Set<String> pre = null;
    for (int i = 0; i < 100; i++) {
        StormTopology topology = buildTopology();
        Map<String, Bolt> cur = topology.get_bolts();
        System.out.println(cur.keySet());
        if (pre != null) {
            Assert.assertTrue("bold id not consistent with group name", pre.equals(cur.keySet()));
        }
        pre = cur.keySet();
    }
}

Example 12 with Bolt

use of org.apache.storm.generated.Bolt in project storm by apache.

the class StormCommon method addSystemComponents.

@SuppressWarnings("unused")
public static void addSystemComponents(Map conf, StormTopology topology) {
    Map<String, StreamInfo> outputStreams = new HashMap<>();
    outputStreams.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("rate_secs")));
    outputStreams.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("interval")));
    outputStreams.put(Constants.CREDENTIALS_CHANGED_STREAM_ID, Thrift.outputFields(Arrays.asList("creds")));
    Map<String, Object> boltConf = new HashMap<>();
    boltConf.put(Config.TOPOLOGY_TASKS, 0);
    Bolt systemBoltSpec = Thrift.prepareSerializedBoltDetails(null, new SystemBolt(), outputStreams, 0, boltConf);
    topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, systemBoltSpec);
}

Example 13 with Bolt

use of org.apache.storm.generated.Bolt in project storm by apache.

the class StormCommon method addEventLogger.

public static void addEventLogger(Map conf, StormTopology topology) {
    Integer numExecutors = Utils.getInt(conf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    HashMap<String, Object> componentConf = new HashMap<>();
    componentConf.put(Config.TOPOLOGY_TASKS, numExecutors);
    componentConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Bolt eventLoggerBolt = Thrift.prepareSerializedBoltDetails(eventLoggerInputs(topology), new EventLoggerBolt(), null, numExecutors, componentConf);
    for (Object component : allComponents(topology).values()) {
        ComponentCommon common = getComponentCommon(component);
        common.put_to_streams(EVENTLOGGER_STREAM_ID, Thrift.outputFields(eventLoggerBoltFields()));
    }
    topology.put_to_bolts(EVENTLOGGER_COMPONENT_ID, eventLoggerBolt);
}

Example 14 with Bolt

use of org.apache.storm.generated.Bolt in project storm by apache.

the class StormCommon method metricsConsumerBoltSpecs.

@SuppressWarnings("unchecked")
public static Map<String, Bolt> metricsConsumerBoltSpecs(Map conf, StormTopology topology) {
    Map<String, Bolt> metricsConsumerBolts = new HashMap<>();
    Set<String> componentIdsEmitMetrics = new HashSet<>();
    componentIdsEmitMetrics.addAll(allComponents(topology).keySet());
    componentIdsEmitMetrics.add(Constants.SYSTEM_COMPONENT_ID);
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();
    for (String componentId : componentIdsEmitMetrics) {
        inputs.put(Utils.getGlobalStreamId(componentId, Constants.METRICS_STREAM_ID), Thrift.prepareShuffleGrouping());
    }
    List<Map<String, Object>> registerInfo = (List<Map<String, Object>>) conf.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER);
    if (registerInfo != null) {
        Map<String, Integer> classOccurrencesMap = new HashMap<String, Integer>();
        for (Map<String, Object> info : registerInfo) {
            String className = (String) info.get(TOPOLOGY_METRICS_CONSUMER_CLASS);
            Object argument = info.get(TOPOLOGY_METRICS_CONSUMER_ARGUMENT);
            Integer maxRetainMetricTuples = Utils.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES), 100);
            Integer phintNum = Utils.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT), 1);
            Map<String, Object> metricsConsumerConf = new HashMap<String, Object>();
            metricsConsumerConf.put(Config.TOPOLOGY_TASKS, phintNum);
            List<String> whitelist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_WHITELIST);
            List<String> blacklist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_BLACKLIST);
            FilterByMetricName filterPredicate = new FilterByMetricName(whitelist, blacklist);
            Boolean expandMapType = Utils.getBoolean(info.get(TOPOLOGY_METRICS_CONSUMER_EXPAND_MAP_TYPE), false);
            String metricNameSeparator = Utils.getString(info.get(TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR), ".");
            DataPointExpander expander = new DataPointExpander(expandMapType, metricNameSeparator);
            MetricsConsumerBolt boltInstance = new MetricsConsumerBolt(className, argument, maxRetainMetricTuples, filterPredicate, expander);
            Bolt metricsConsumerBolt = Thrift.prepareSerializedBoltDetails(inputs, boltInstance, null, phintNum, metricsConsumerConf);
            String id = className;
            if (classOccurrencesMap.containsKey(className)) {
                // e.g. [\"a\", \"b\", \"a\"]) => [\"a\", \"b\", \"a#2\"]"
                int occurrenceNum = classOccurrencesMap.get(className);
                occurrenceNum++;
                classOccurrencesMap.put(className, occurrenceNum);
                id = Constants.METRICS_COMPONENT_ID_PREFIX + className + "#" + occurrenceNum;
            } else {
                classOccurrencesMap.put(className, 1);
            }
            metricsConsumerBolts.put(id, metricsConsumerBolt);
        }
    }
    return metricsConsumerBolts;
}

Example 15 with Bolt

use of org.apache.storm.generated.Bolt in project storm by apache.

the class TestUtilsForResourceAwareScheduler method genExecsAndComps.

public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology) {
    Map<ExecutorDetails, String> retMap = new HashMap<ExecutorDetails, String>();
    int startTask = 0;
    int endTask = 0;
    for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) {
        SpoutSpec spout = entry.getValue();
        String spoutId = entry.getKey();
        int spoutParallelism = spout.get_common().get_parallelism_hint();
        for (int i = 0; i < spoutParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), spoutId);
            startTask++;
            endTask++;
        }
    }
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltId = entry.getKey();
        Bolt bolt = entry.getValue();
        int boltParallelism = bolt.get_common().get_parallelism_hint();
        for (int i = 0; i < boltParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), boltId);
            startTask++;
            endTask++;
        }
    }
    return retMap;
}