Examples with StreamWriterHeartbeat co.cask.cdap.data.stream.service.heartbeat.StreamWriterHeartbeat used on opensource projects

Example 1 with StreamWriterHeartbeat

use of co.cask.cdap.data.stream.service.heartbeat.StreamWriterHeartbeat in project cdap by caskdata.

the class DFSStreamHeartbeatsTest method getStreamSize.

private long getStreamSize(StreamId streamId, MockHeartbeatPublisher heartbeatPublisher) {
    StreamWriterHeartbeat heartbeat = heartbeatPublisher.getHeartbeat();
    if (heartbeat == null) {
        return 0L;
    }
    Long size = heartbeat.getStreamsSizes().get(streamId);
    return size == null ? 0L : size;
}

Example 2 with StreamWriterHeartbeat

use of co.cask.cdap.data.stream.service.heartbeat.StreamWriterHeartbeat in project cdap by caskdata.

the class DistributedStreamService method subscribeToHeartbeatsFeed.

/**
 * Subscribe to the streams heartbeat notification feed. One heartbeat contains data for all existing streams,
 * we filter that to only take into account the streams that this {@link DistributedStreamService} is a leader
 * of.
 *
 * @return a {@link Cancellable} to cancel the subscription
 * @throws NotificationFeedNotFoundException if the heartbeat feed does not exist
 */
private Cancellable subscribeToHeartbeatsFeed() throws NotificationFeedNotFoundException {
    LOG.debug("Subscribing to stream heartbeats notification feed");
    NotificationFeedId heartbeatsFeed = new NotificationFeedId(NamespaceId.SYSTEM.getNamespace(), Constants.Notification.Stream.STREAM_INTERNAL_FEED_CATEGORY, Constants.Notification.Stream.STREAM_HEARTBEAT_FEED_NAME);
    boolean isRetry = false;
    while (true) {
        try {
            return notificationService.subscribe(heartbeatsFeed, new NotificationHandler<StreamWriterHeartbeat>() {

                @Override
                public Type getNotificationType() {
                    return StreamWriterHeartbeat.class;
                }

                @Override
                public void received(StreamWriterHeartbeat heartbeat, NotificationContext notificationContext) {
                    LOG.trace("Received heartbeat {}", heartbeat);
                    for (Map.Entry<StreamId, Long> entry : heartbeat.getStreamsSizes().entrySet()) {
                        StreamSizeAggregator streamSizeAggregator = aggregators.get(entry.getKey());
                        if (streamSizeAggregator == null) {
                            LOG.trace("Aggregator for stream {} is null", entry.getKey());
                            continue;
                        }
                        streamSizeAggregator.bytesReceived(heartbeat.getInstanceId(), entry.getValue());
                    }
                }
            }, heartbeatsSubscriptionExecutor);
        } catch (NotificationFeedException e) {
            if (!isRetry) {
                LOG.warn("Unable to subscribe to HeartbeatsFeed. Will retry until successfully subscribed. " + "Retry failures will be logged at debug level.", e);
            } else {
                LOG.debug("Unable to subscribe to HeartbeatsFeed. Will retry until successfully subscribed. ", e);
            }
            isRetry = true;
            waitBeforeRetryHeartbeatsFeedOperation();
        }
    }
}

Example 3 with StreamWriterHeartbeat

use of co.cask.cdap.data.stream.service.heartbeat.StreamWriterHeartbeat in project cdap by caskdata.

the class DistributedStreamService method runOneIteration.

@Override
protected void runOneIteration() throws Exception {
    LOG.trace("Performing heartbeat publishing in Stream service instance {}", instanceId);
    ImmutableMap.Builder<StreamId, Long> sizes = ImmutableMap.builder();
    Map<StreamId, AtomicLong> streamSizes = streamWriterSizeCollector.getStreamSizes();
    for (Map.Entry<StreamId, AtomicLong> streamSize : streamSizes.entrySet()) {
        sizes.put(streamSize.getKey(), streamSize.getValue().get());
    }
    StreamWriterHeartbeat heartbeat = new StreamWriterHeartbeat(System.currentTimeMillis(), instanceId, sizes.build());
    LOG.trace("Publishing heartbeat {}", heartbeat);
    heartbeatPublisher.sendHeartbeat(heartbeat);
}