Search in sources :

Example 1 with MetricsConsumerBolt

use of org.apache.storm.metric.MetricsConsumerBolt in project storm by apache.

the class StormCommon method metricsConsumerBoltSpecs.

@SuppressWarnings("unchecked")
public static Map<String, Bolt> metricsConsumerBoltSpecs(Map<String, Object> conf, StormTopology topology) {
    Map<String, Bolt> metricsConsumerBolts = new HashMap<>();
    Set<String> componentIdsEmitMetrics = new HashSet<>();
    componentIdsEmitMetrics.addAll(allComponents(topology).keySet());
    componentIdsEmitMetrics.add(Constants.SYSTEM_COMPONENT_ID);
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();
    for (String componentId : componentIdsEmitMetrics) {
        inputs.put(Utils.getGlobalStreamId(componentId, Constants.METRICS_STREAM_ID), Thrift.prepareShuffleGrouping());
    }
    List<Map<String, Object>> registerInfo = (List<Map<String, Object>>) conf.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER);
    if (registerInfo != null) {
        Map<String, Integer> classOccurrencesMap = new HashMap<>();
        for (Map<String, Object> info : registerInfo) {
            String className = (String) info.get(TOPOLOGY_METRICS_CONSUMER_CLASS);
            Object argument = info.get(TOPOLOGY_METRICS_CONSUMER_ARGUMENT);
            Integer maxRetainMetricTuples = ObjectReader.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES), 100);
            Integer phintNum = ObjectReader.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT), 1);
            Map<String, Object> metricsConsumerConf = new HashMap<>();
            metricsConsumerConf.put(Config.TOPOLOGY_TASKS, phintNum);
            List<String> whitelist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_WHITELIST);
            List<String> blacklist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_BLACKLIST);
            FilterByMetricName filterPredicate = new FilterByMetricName(whitelist, blacklist);
            Boolean expandMapType = ObjectReader.getBoolean(info.get(TOPOLOGY_METRICS_CONSUMER_EXPAND_MAP_TYPE), false);
            String metricNameSeparator = ObjectReader.getString(info.get(TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR), ".");
            DataPointExpander expander = new DataPointExpander(expandMapType, metricNameSeparator);
            MetricsConsumerBolt boltInstance = new MetricsConsumerBolt(className, argument, maxRetainMetricTuples, filterPredicate, expander);
            Bolt metricsConsumerBolt = Thrift.prepareSerializedBoltDetails(inputs, boltInstance, null, phintNum, metricsConsumerConf);
            String id;
            if (classOccurrencesMap.containsKey(className)) {
                // e.g. [\"a\", \"b\", \"a\"]) => [\"a\", \"b\", \"a#2\"]"
                int occurrenceNum = classOccurrencesMap.get(className);
                occurrenceNum++;
                classOccurrencesMap.put(className, occurrenceNum);
                id = Constants.METRICS_COMPONENT_ID_PREFIX + className + "#" + occurrenceNum;
            } else {
                id = Constants.METRICS_COMPONENT_ID_PREFIX + className;
                classOccurrencesMap.put(className, 1);
            }
            metricsConsumerBolts.put(id, metricsConsumerBolt);
        }
    }
    return metricsConsumerBolts;
}
Also used : FilterByMetricName(org.apache.storm.metric.filter.FilterByMetricName) HashMap(java.util.HashMap) Bolt(org.apache.storm.generated.Bolt) MetricsConsumerBolt(org.apache.storm.metric.MetricsConsumerBolt) IBolt(org.apache.storm.task.IBolt) EventLoggerBolt(org.apache.storm.metric.EventLoggerBolt) SystemBolt(org.apache.storm.metric.SystemBolt) Grouping(org.apache.storm.generated.Grouping) MetricsConsumerBolt(org.apache.storm.metric.MetricsConsumerBolt) GlobalStreamId(org.apache.storm.generated.GlobalStreamId) ArrayList(java.util.ArrayList) List(java.util.List) DataPointExpander(org.apache.storm.metric.util.DataPointExpander) HashMap(java.util.HashMap) Map(java.util.Map) TreeMap(java.util.TreeMap) HashSet(java.util.HashSet)

Example 2 with MetricsConsumerBolt

use of org.apache.storm.metric.MetricsConsumerBolt in project storm by apache.

the class StormCommon method metricsConsumerBoltSpecs.

@SuppressWarnings("unchecked")
public static Map<String, Bolt> metricsConsumerBoltSpecs(Map conf, StormTopology topology) {
    Map<String, Bolt> metricsConsumerBolts = new HashMap<>();
    Set<String> componentIdsEmitMetrics = new HashSet<>();
    componentIdsEmitMetrics.addAll(allComponents(topology).keySet());
    componentIdsEmitMetrics.add(Constants.SYSTEM_COMPONENT_ID);
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();
    for (String componentId : componentIdsEmitMetrics) {
        inputs.put(Utils.getGlobalStreamId(componentId, Constants.METRICS_STREAM_ID), Thrift.prepareShuffleGrouping());
    }
    List<Map<String, Object>> registerInfo = (List<Map<String, Object>>) conf.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER);
    if (registerInfo != null) {
        Map<String, Integer> classOccurrencesMap = new HashMap<String, Integer>();
        for (Map<String, Object> info : registerInfo) {
            String className = (String) info.get(TOPOLOGY_METRICS_CONSUMER_CLASS);
            Object argument = info.get(TOPOLOGY_METRICS_CONSUMER_ARGUMENT);
            Integer maxRetainMetricTuples = Utils.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES), 100);
            Integer phintNum = Utils.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT), 1);
            Map<String, Object> metricsConsumerConf = new HashMap<String, Object>();
            metricsConsumerConf.put(Config.TOPOLOGY_TASKS, phintNum);
            List<String> whitelist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_WHITELIST);
            List<String> blacklist = (List<String>) info.get(TOPOLOGY_METRICS_CONSUMER_BLACKLIST);
            FilterByMetricName filterPredicate = new FilterByMetricName(whitelist, blacklist);
            Boolean expandMapType = Utils.getBoolean(info.get(TOPOLOGY_METRICS_CONSUMER_EXPAND_MAP_TYPE), false);
            String metricNameSeparator = Utils.getString(info.get(TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR), ".");
            DataPointExpander expander = new DataPointExpander(expandMapType, metricNameSeparator);
            MetricsConsumerBolt boltInstance = new MetricsConsumerBolt(className, argument, maxRetainMetricTuples, filterPredicate, expander);
            Bolt metricsConsumerBolt = Thrift.prepareSerializedBoltDetails(inputs, boltInstance, null, phintNum, metricsConsumerConf);
            String id = className;
            if (classOccurrencesMap.containsKey(className)) {
                // e.g. [\"a\", \"b\", \"a\"]) => [\"a\", \"b\", \"a#2\"]"
                int occurrenceNum = classOccurrencesMap.get(className);
                occurrenceNum++;
                classOccurrencesMap.put(className, occurrenceNum);
                id = Constants.METRICS_COMPONENT_ID_PREFIX + className + "#" + occurrenceNum;
            } else {
                classOccurrencesMap.put(className, 1);
            }
            metricsConsumerBolts.put(id, metricsConsumerBolt);
        }
    }
    return metricsConsumerBolts;
}
Also used : FilterByMetricName(org.apache.storm.metric.filter.FilterByMetricName) HashMap(java.util.HashMap) Bolt(org.apache.storm.generated.Bolt) MetricsConsumerBolt(org.apache.storm.metric.MetricsConsumerBolt) IBolt(org.apache.storm.task.IBolt) EventLoggerBolt(org.apache.storm.metric.EventLoggerBolt) SystemBolt(org.apache.storm.metric.SystemBolt) Grouping(org.apache.storm.generated.Grouping) MetricsConsumerBolt(org.apache.storm.metric.MetricsConsumerBolt) GlobalStreamId(org.apache.storm.generated.GlobalStreamId) ArrayList(java.util.ArrayList) List(java.util.List) DataPointExpander(org.apache.storm.metric.util.DataPointExpander) HashMap(java.util.HashMap) Map(java.util.Map) TreeMap(java.util.TreeMap) HashSet(java.util.HashSet)

Aggregations

ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 List (java.util.List)2 Map (java.util.Map)2 TreeMap (java.util.TreeMap)2 Bolt (org.apache.storm.generated.Bolt)2 GlobalStreamId (org.apache.storm.generated.GlobalStreamId)2 Grouping (org.apache.storm.generated.Grouping)2 EventLoggerBolt (org.apache.storm.metric.EventLoggerBolt)2 MetricsConsumerBolt (org.apache.storm.metric.MetricsConsumerBolt)2 SystemBolt (org.apache.storm.metric.SystemBolt)2 FilterByMetricName (org.apache.storm.metric.filter.FilterByMetricName)2 DataPointExpander (org.apache.storm.metric.util.DataPointExpander)2 IBolt (org.apache.storm.task.IBolt)2