Search in sources :

Example 1 with TIMESTAMP_TUPLE_FIELD

use of org.apache.metron.profiler.storm.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD in project metron by apache.

the class ProfileBuilderBolt method log.

/**
 * Logs information about the {@link TupleWindow}.
 *
 * @param window The tuple window.
 */
private void log(TupleWindow window) {
    // summarize the newly received tuples
    LongSummaryStatistics received = window.get().stream().map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)).collect(Collectors.summarizingLong(Long::longValue));
    LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms", received.getCount(), received.getMin(), received.getMax(), received.getMax() - received.getMin());
    if (window.getExpired().size() > 0) {
        // summarize the expired tuples
        LongSummaryStatistics expired = window.getExpired().stream().map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)).collect(Collectors.summarizingLong(Long::longValue));
        LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms", expired.getCount(), expired.getMin(), expired.getMax(), expired.getMax() - expired.getMin(), received.getMin() - expired.getMin());
    }
}
Also used : LongSummaryStatistics(java.util.LongSummaryStatistics) OutputFieldsDeclarer(org.apache.storm.topology.OutputFieldsDeclarer) CuratorFrameworkFactory(org.apache.curator.framework.CuratorFrameworkFactory) ProfilerConfigurations(org.apache.metron.common.configuration.profiler.ProfilerConfigurations) BaseWindowedBolt(org.apache.storm.topology.base.BaseWindowedBolt) TopologyContext(org.apache.storm.task.TopologyContext) LoggerFactory(org.slf4j.LoggerFactory) TIMESTAMP_TUPLE_FIELD(org.apache.metron.profiler.storm.ProfileSplitterBolt.TIMESTAMP_TUPLE_FIELD) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) Reloadable(org.apache.metron.common.zookeeper.configurations.Reloadable) PROFILE_TUPLE_FIELD(org.apache.metron.profiler.storm.ProfileSplitterBolt.PROFILE_TUPLE_FIELD) TreeCacheEvent(org.apache.curator.framework.recipes.cache.TreeCacheEvent) Tuple(org.apache.storm.tuple.Tuple) TupleWindow(org.apache.storm.windowing.TupleWindow) OutputCollector(org.apache.storm.task.OutputCollector) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) Map(java.util.Map) ConfigurationType(org.apache.metron.common.configuration.ConfigurationType) ConfigurationsUtils(org.apache.metron.common.configuration.ConfigurationsUtils) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) DefaultMessageDistributor(org.apache.metron.profiler.DefaultMessageDistributor) LongSummaryStatistics(java.util.LongSummaryStatistics) Context(org.apache.metron.stellar.dsl.Context) Logger(org.slf4j.Logger) ConfigurationsUpdater(org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater) ZKCache(org.apache.metron.zookeeper.ZKCache) JSONParser(org.json.simple.parser.JSONParser) MethodHandles(java.lang.invoke.MethodHandles) ProfilerUpdater(org.apache.metron.common.zookeeper.configurations.ProfilerUpdater) Constants(org.apache.metron.common.Constants) MessageRoute(org.apache.metron.profiler.MessageRoute) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) String.format(java.lang.String.format) SimpleEventListener(org.apache.metron.zookeeper.SimpleEventListener) TimeUnit(java.util.concurrent.TimeUnit) ProfileMeasurement(org.apache.metron.profiler.ProfileMeasurement) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) JSONObject(org.json.simple.JSONObject) ENTITY_TUPLE_FIELD(org.apache.metron.profiler.storm.ProfileSplitterBolt.ENTITY_TUPLE_FIELD) MessageDistributor(org.apache.metron.profiler.MessageDistributor) RetryPolicy(org.apache.curator.RetryPolicy) ConversionUtils(org.apache.metron.stellar.common.utils.ConversionUtils) ProfileConfig(org.apache.metron.common.configuration.profiler.ProfileConfig) MESSAGE_TUPLE_FIELD(org.apache.metron.profiler.storm.ProfileSplitterBolt.MESSAGE_TUPLE_FIELD)

Aggregations

String.format (java.lang.String.format)1 MethodHandles (java.lang.invoke.MethodHandles)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 LongSummaryStatistics (java.util.LongSummaryStatistics)1 Map (java.util.Map)1 Executors (java.util.concurrent.Executors)1 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)1 TimeUnit (java.util.concurrent.TimeUnit)1 Collectors (java.util.stream.Collectors)1 CollectionUtils (org.apache.commons.collections4.CollectionUtils)1 RetryPolicy (org.apache.curator.RetryPolicy)1 CuratorFramework (org.apache.curator.framework.CuratorFramework)1 CuratorFrameworkFactory (org.apache.curator.framework.CuratorFrameworkFactory)1 TreeCacheEvent (org.apache.curator.framework.recipes.cache.TreeCacheEvent)1 ExponentialBackoffRetry (org.apache.curator.retry.ExponentialBackoffRetry)1 Constants (org.apache.metron.common.Constants)1 ConfigurationType (org.apache.metron.common.configuration.ConfigurationType)1 ConfigurationsUtils (org.apache.metron.common.configuration.ConfigurationsUtils)1 ProfileConfig (org.apache.metron.common.configuration.profiler.ProfileConfig)1