use of co.cask.cdap.proto.id.NotificationFeedId in project cdap by caskdata.
the class StreamSizeSchedulerTest method createMetricsPublisher.
@Override
protected StreamMetricsPublisher createMetricsPublisher(final StreamId streamId) {
final NotificationFeedId feed = new NotificationFeedId(streamId.getNamespace(), Constants.Notification.Stream.STREAM_FEED_CATEGORY, streamId.getStream() + "Size");
return new StreamMetricsPublisher() {
long totalSize;
@Override
public void increment(long size) throws Exception {
metricStore.add(new MetricValues(ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, streamId.getNamespace(), Constants.Metrics.Tag.STREAM, streamId.getStream()), "collect.bytes", TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()), size, MetricType.COUNTER));
totalSize += size;
notificationService.publish(feed, new StreamSizeNotification(System.currentTimeMillis(), totalSize));
}
};
}
use of co.cask.cdap.proto.id.NotificationFeedId in project cdap by caskdata.
the class NotificationTest method testPubSub.
/**
* Testing publishers/subscribers interaction.
*
* @param pubFeeds set of feeds to publish to
* @param publishersPerFeed number of publishers doing concurrent publishing for each feed
* @param messagesPerPublisher number of messages being published by each publisher
* @param subFeeds set of feeds to subscribe to
* @param subscribersPerFeed number of subscribers for each feed
* @param payloadType Class reprenseting the data type of the payload of the notification
* @param payloadFunction a function that transform {@link SimpleNotification} type to the payload type
* @param <T> type of the payload
*/
private <T> void testPubSub(Set<NotificationFeedId> pubFeeds, int publishersPerFeed, final int messagesPerPublisher, Set<NotificationFeedId> subFeeds, int subscribersPerFeed, final Class<T> payloadType, final Function<SimpleNotification, T> payloadFunction) throws Exception {
for (NotificationFeedId feedId : Sets.union(pubFeeds, subFeeds)) {
NotificationFeedInfo feedInfo = new NotificationFeedInfo(feedId.getNamespace(), feedId.getCategory(), feedId.getFeed(), "");
Assert.assertTrue(feedManager.createFeed(feedInfo));
}
try {
int totalMessages = subFeeds.size() * publishersPerFeed * messagesPerPublisher * subscribersPerFeed;
final CountDownLatch latch = new CountDownLatch(totalMessages);
final Queue<T> receivedMessages = new ConcurrentLinkedQueue<>();
List<Cancellable> cancellables = Lists.newArrayList();
try {
for (NotificationFeedId feedId : subFeeds) {
for (int i = 0; i < subscribersPerFeed; i++) {
Cancellable cancellable = notificationService.subscribe(feedId, new NotificationHandler<T>() {
@Override
public Type getNotificationType() {
return payloadType;
}
@Override
public void received(T notification, NotificationContext notificationContext) {
LOG.debug("Received notification payload: {}", notification);
receivedMessages.offer(notification);
latch.countDown();
}
});
cancellables.add(cancellable);
}
}
// Give the subscriber some time to prepare for published messages before starting the publisher
TimeUnit.MILLISECONDS.sleep(500);
// Starts publishers
final Map<NotificationFeedId, Queue<T>> publishedMessages = new ConcurrentHashMap<>();
ExecutorService executor = Executors.newFixedThreadPool(pubFeeds.size() * publishersPerFeed);
try {
for (final NotificationFeedId feedId : pubFeeds) {
final Queue<T> publishedQueue = new ConcurrentLinkedQueue<>();
publishedMessages.put(feedId, publishedQueue);
// Let all publishers start together
final CyclicBarrier publisherBarrier = new CyclicBarrier(publishersPerFeed);
for (int i = 0; i < publishersPerFeed; i++) {
final int publisherId = i;
executor.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
publisherBarrier.await();
for (int i = 0; i < messagesPerPublisher; i++) {
T notification = payloadFunction.apply(new SimpleNotification(publisherId, String.format("%s-%d", feedId, i)));
notificationService.publish(feedId, notification);
publishedQueue.add(notification);
TimeUnit.MILLISECONDS.sleep(10);
}
return null;
}
});
}
}
// Wait for subscriptions getting all messages
Assert.assertTrue(latch.await(5000, TimeUnit.SECONDS));
} finally {
executor.shutdown();
}
// Verify the result.
Multiset<T> received = HashMultiset.create(receivedMessages);
Assert.assertEquals(totalMessages, received.size());
// there should be (publisher per feed * subscriber per feed) of them
for (NotificationFeedId feedId : subFeeds) {
for (T notification : ImmutableMultiset.copyOf(publishedMessages.get(feedId)).elementSet()) {
Assert.assertEquals(publishersPerFeed * subscribersPerFeed, received.count(notification));
}
}
} finally {
for (Cancellable cancellable : cancellables) {
cancellable.cancel();
}
}
} finally {
for (NotificationFeedId feedId : Sets.union(pubFeeds, subFeeds)) {
feedManager.deleteFeed(feedId);
}
}
}
use of co.cask.cdap.proto.id.NotificationFeedId in project cdap by caskdata.
the class InMemoryNotificationFeedStore method createNotificationFeed.
@Nullable
@Override
public synchronized NotificationFeedInfo createNotificationFeed(NotificationFeedInfo feed) {
NotificationFeedInfo existingFeed = feeds.get(feed);
if (existingFeed != null) {
return existingFeed;
}
NotificationFeedId id = new NotificationFeedId(feed.getNamespace(), feed.getCategory(), feed.getFeed());
feeds.put(id, feed);
return null;
}
use of co.cask.cdap.proto.id.NotificationFeedId in project cdap by caskdata.
the class DistributedStreamService method subscribeToHeartbeatsFeed.
/**
* Subscribe to the streams heartbeat notification feed. One heartbeat contains data for all existing streams,
* we filter that to only take into account the streams that this {@link DistributedStreamService} is a leader
* of.
*
* @return a {@link Cancellable} to cancel the subscription
* @throws NotificationFeedNotFoundException if the heartbeat feed does not exist
*/
private Cancellable subscribeToHeartbeatsFeed() throws NotificationFeedNotFoundException {
LOG.debug("Subscribing to stream heartbeats notification feed");
NotificationFeedId heartbeatsFeed = new NotificationFeedId(NamespaceId.SYSTEM.getNamespace(), Constants.Notification.Stream.STREAM_INTERNAL_FEED_CATEGORY, Constants.Notification.Stream.STREAM_HEARTBEAT_FEED_NAME);
boolean isRetry = false;
while (true) {
try {
return notificationService.subscribe(heartbeatsFeed, new NotificationHandler<StreamWriterHeartbeat>() {
@Override
public Type getNotificationType() {
return StreamWriterHeartbeat.class;
}
@Override
public void received(StreamWriterHeartbeat heartbeat, NotificationContext notificationContext) {
LOG.trace("Received heartbeat {}", heartbeat);
for (Map.Entry<StreamId, Long> entry : heartbeat.getStreamsSizes().entrySet()) {
StreamSizeAggregator streamSizeAggregator = aggregators.get(entry.getKey());
if (streamSizeAggregator == null) {
LOG.trace("Aggregator for stream {} is null", entry.getKey());
continue;
}
streamSizeAggregator.bytesReceived(heartbeat.getInstanceId(), entry.getValue());
}
}
}, heartbeatsSubscriptionExecutor);
} catch (NotificationFeedException e) {
if (!isRetry) {
LOG.warn("Unable to subscribe to HeartbeatsFeed. Will retry until successfully subscribed. " + "Retry failures will be logged at debug level.", e);
} else {
LOG.debug("Unable to subscribe to HeartbeatsFeed. Will retry until successfully subscribed. ", e);
}
isRetry = true;
waitBeforeRetryHeartbeatsFeedOperation();
}
}
}
use of co.cask.cdap.proto.id.NotificationFeedId in project cdap by caskdata.
the class NotificationFeedHttpHandler method deleteFeed.
@DELETE
@Path("/feeds/categories/{feed-category}/names/{feed-name}")
public void deleteFeed(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("feed-category") String category, @PathParam("feed-name") String name) {
try {
NotificationFeedId feed;
try {
feed = new NotificationFeedId(namespaceId, category, name);
} catch (IllegalArgumentException e) {
responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
return;
}
feedManager.deleteFeed(feed);
responder.sendStatus(HttpResponseStatus.OK);
} catch (NotificationFeedNotFoundException e) {
responder.sendStatus(HttpResponseStatus.NOT_FOUND);
} catch (NotificationFeedException e) {
responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, String.format("Could not delete Notification Feed. %s", e.getMessage()));
} catch (Throwable t) {
LOG.debug("Error in deleting notification feed.", t);
responder.sendString(HttpResponseStatus.INTERNAL_SERVER_ERROR, t.getMessage());
}
}
Aggregations