Examples with IMetric org.apache.storm.metric.api.IMetric used on opensource projects

Example 1 with IMetric

use of org.apache.storm.metric.api.IMetric in project storm by apache.

the class EventHubSpout method open.

@Override
public void open(Map config, TopologyContext context, SpoutOutputCollector collector) {
    logger.info("begin: open()");
    String topologyName = (String) config.get(Config.TOPOLOGY_NAME);
    eventHubConfig.setTopologyName(topologyName);
    int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
    int taskIndex = context.getThisTaskIndex();
    if (totalTasks > eventHubConfig.getPartitionCount()) {
        throw new RuntimeException("total tasks of EventHubSpout is greater than partition count.");
    }
    logger.info(String.format("topologyName: %s, totalTasks: %d, taskIndex: %d", topologyName, totalTasks, taskIndex));
    try {
        preparePartitions(config, totalTasks, taskIndex, collector);
    } catch (Exception e) {
        collector.reportError(e);
        throw new RuntimeException(e);
    }
    //register metrics
    context.registerMetric("EventHubReceiver", new IMetric() {

        @Override
        public Object getValueAndReset() {
            Map concatMetricsDataMaps = new HashMap();
            for (IPartitionManager partitionManager : partitionCoordinator.getMyPartitionManagers()) {
                concatMetricsDataMaps.putAll(partitionManager.getMetricsData());
            }
            return concatMetricsDataMaps;
        }
    }, Integer.parseInt(config.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS).toString()));
    logger.info("end open()");
}

Example 2 with IMetric

use of org.apache.storm.metric.api.IMetric in project storm by apache.

the class KafkaSpout method open.

@Override
public void open(Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
    _collector = collector;
    String topologyInstanceId = context.getStormId();
    Map<String, Object> stateConf = new HashMap<>(conf);
    List<String> zkServers = _spoutConfig.zkServers;
    if (zkServers == null) {
        zkServers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
    }
    Integer zkPort = _spoutConfig.zkPort;
    if (zkPort == null) {
        zkPort = ((Number) conf.get(Config.STORM_ZOOKEEPER_PORT)).intValue();
    }
    stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, zkServers);
    stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_PORT, zkPort);
    stateConf.put(Config.TRANSACTIONAL_ZOOKEEPER_ROOT, _spoutConfig.zkRoot);
    _state = new ZkState(stateConf);
    _connections = new DynamicPartitionConnections(_spoutConfig, KafkaUtils.makeBrokerReader(conf, _spoutConfig));
    // using TransactionalState like this is a hack
    int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
    if (_spoutConfig.hosts instanceof StaticHosts) {
        _coordinator = new StaticCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, topologyInstanceId);
    } else {
        _coordinator = new ZkCoordinator(_connections, conf, _spoutConfig, _state, context.getThisTaskIndex(), totalTasks, topologyInstanceId);
    }
    context.registerMetric("kafkaOffset", new IMetric() {

        KafkaUtils.KafkaOffsetMetric _kafkaOffsetMetric = new KafkaUtils.KafkaOffsetMetric(_connections);

        @Override
        public Object getValueAndReset() {
            List<PartitionManager> pms = _coordinator.getMyManagedPartitions();
            Set<Partition> latestPartitions = new HashSet();
            for (PartitionManager pm : pms) {
                latestPartitions.add(pm.getPartition());
            }
            _kafkaOffsetMetric.refreshPartitions(latestPartitions);
            for (PartitionManager pm : pms) {
                _kafkaOffsetMetric.setOffsetData(pm.getPartition(), pm.getOffsetData());
            }
            return _kafkaOffsetMetric.getValueAndReset();
        }
    }, _spoutConfig.metricsTimeBucketSizeInSecs);
    context.registerMetric("kafkaPartition", new IMetric() {

        @Override
        public Object getValueAndReset() {
            List<PartitionManager> pms = _coordinator.getMyManagedPartitions();
            Map concatMetricsDataMaps = new HashMap();
            for (PartitionManager pm : pms) {
                concatMetricsDataMaps.putAll(pm.getMetricsDataMap());
            }
            return concatMetricsDataMaps;
        }
    }, _spoutConfig.metricsTimeBucketSizeInSecs);
}

Example 3 with IMetric

use of org.apache.storm.metric.api.IMetric in project storm by apache.

the class BuiltinMetricsUtil method registerIconnectionClientMetrics.

public static void registerIconnectionClientMetrics(final Map nodePortToSocket, Map stormConf, TopologyContext context) {
    IMetric metric = new IMetric() {

        @Override
        public Object getValueAndReset() {
            Map<Object, Object> ret = new HashMap<>();
            for (Object o : nodePortToSocket.entrySet()) {
                Map.Entry entry = (Map.Entry) o;
                Object nodePort = entry.getKey();
                Object connection = entry.getValue();
                if (connection instanceof IStatefulObject) {
                    ret.put(nodePort, ((IStatefulObject) connection).getState());
                }
            }
            return ret;
        }
    };
    registerMetric("__send-iconnection", metric, stormConf, context);
}

Example 4 with IMetric

use of org.apache.storm.metric.api.IMetric in project storm by apache.

the class Executor method metricsTick.

public void metricsTick(Task taskData, TupleImpl tuple) {
    try {
        Integer interval = tuple.getInteger(0);
        int taskId = taskData.getTaskId();
        Map<Integer, Map<String, IMetric>> taskToMetricToRegistry = intervalToTaskToMetricToRegistry.get(interval);
        Map<String, IMetric> nameToRegistry = null;
        if (taskToMetricToRegistry != null) {
            nameToRegistry = taskToMetricToRegistry.get(taskId);
        }
        if (nameToRegistry != null) {
            IMetricsConsumer.TaskInfo taskInfo = new IMetricsConsumer.TaskInfo(hostname, workerTopologyContext.getThisWorkerPort(), componentId, taskId, Time.currentTimeSecs(), interval);
            List<IMetricsConsumer.DataPoint> dataPoints = new ArrayList<>();
            for (Map.Entry<String, IMetric> entry : nameToRegistry.entrySet()) {
                IMetric metric = entry.getValue();
                Object value = metric.getValueAndReset();
                if (value != null) {
                    IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey(), value);
                    dataPoints.add(dataPoint);
                }
            }
            if (!dataPoints.isEmpty()) {
                sendUnanchored(taskData, Constants.METRICS_STREAM_ID, new Values(taskInfo, dataPoints), executorTransfer);
            }
        }
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}

Example 5 with IMetric

use of org.apache.storm.metric.api.IMetric in project storm by apache.

the class ShellSpout method handleMetrics.

private void handleMetrics(ShellMsg shellMsg) {
    //get metric name
    String name = shellMsg.getMetricName();
    if (name.isEmpty()) {
        throw new RuntimeException("Receive Metrics name is empty");
    }
    //get metric by name
    IMetric iMetric = _context.getRegisteredMetricByName(name);
    if (iMetric == null) {
        throw new RuntimeException("Could not find metric by name[" + name + "] ");
    }
    if (!(iMetric instanceof IShellMetric)) {
        throw new RuntimeException("Metric[" + name + "] is not IShellMetric, can not call by RPC");
    }
    IShellMetric iShellMetric = (IShellMetric) iMetric;
    //call updateMetricFromRPC with params
    Object paramsObj = shellMsg.getMetricParams();
    try {
        iShellMetric.updateMetricFromRPC(paramsObj);
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}