use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.
the class ProvisionerNotifier method publish.
private void publish(Map<String, String> properties) {
final StoreRequest storeRequest = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(new Notification(Notification.Type.PROGRAM_STATUS, properties))).build();
Retries.supplyWithRetries(() -> {
try {
messagingService.publish(storeRequest);
} catch (TopicNotFoundException e) {
throw new RetryableException(e);
} catch (IOException | AccessException e) {
throw Throwables.propagate(e);
}
return null;
}, retryStrategy);
}
use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.
the class MessagingWorkflowStateWriter method setWorkflowToken.
@Override
public void setWorkflowToken(ProgramRunId workflowRunId, WorkflowToken token) {
MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_TOKEN, workflowRunId, GSON.toJsonTree(token));
StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
try {
Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
} catch (Exception e) {
// Don't log the workflow token, as it can be large and may contain sensitive data
throw new RuntimeException("Failed to publish workflow token for workflow run " + workflowRunId, e);
}
}
use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.
the class MessagingMetadataPublisher method publish.
@Override
public void publish(EntityId publisher, MetadataOperation operation) {
MetadataMessage message = new MetadataMessage(Type.METADATA_OPERATION, publisher, GSON.toJsonTree(operation));
StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
LOG.trace("Publishing message {} to topic {}", message, topic);
try {
Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
} catch (Exception e) {
throw new RuntimeException("Failed to publish metadata operation: " + operation, e);
}
}
use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.
the class MessagingUsageWriter method doRegisterAll.
private void doRegisterAll(Iterable<? extends EntityId> users, EntityId entityId) throws Exception {
// Only record usage from program
StoreRequest request = StoreRequestBuilder.of(topic).addPayloads(StreamSupport.stream(users.spliterator(), false).filter(ProgramId.class::isInstance).map(ProgramId.class::cast).map(id -> new MetadataMessage(MetadataMessage.Type.USAGE, id, GSON.toJsonTree(new DatasetUsage(entityId)))).map(GSON::toJson).map(s -> s.getBytes(StandardCharsets.UTF_8)).iterator()).build();
Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
}
use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.
the class ConcurrentMessageWriterTest method testMultiMaxSequence.
@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
TopicId topicId = new NamespaceId("ns1").topic("t1");
final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
// This test the case when multiple StoreRequests combined exceeding the 65536 payload.
// See testMaxSequence() for more details when it matters
// Generate 3 StoreRequests, each with 43690 messages
int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
int requestCount = 3;
List<StoreRequest> requests = new ArrayList<>();
for (int i = 0; i < requestCount; i++) {
List<String> payloads = new ArrayList<>(msgCount);
for (int j = 0; j < msgCount; j++) {
payloads.add(Integer.toString(j));
}
requests.add(new TestStoreRequest(topicId, payloads));
}
TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
// We use a custom metrics collector here to make all the persist calls reached the same latch,
// since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
// before flushing.
// This will make all requests batched together
final CountDownLatch latch = new CountDownLatch(requestCount);
final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {
@Override
public void increment(String metricName, long value) {
if ("persist.requested".equals(metricName)) {
latch.countDown();
Uninterruptibles.awaitUninterruptibly(latch);
}
}
@Override
public void gauge(String metricName, long value) {
LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
}
});
ExecutorService executor = Executors.newFixedThreadPool(3);
for (final StoreRequest request : requests) {
executor.submit(new Runnable() {
@Override
public void run() {
try {
writer.persist(request, metadata);
} catch (IOException e) {
LOG.error("Failed to persist", e);
}
}
});
}
executor.shutdown();
Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
// Validates all messages are being written
List<RawMessage> messages = testWriter.getMessages().get(topicId);
Assert.assertEquals(requestCount * msgCount, messages.size());
// We expect the payload is in repeated sequence of [0..msgCount-1]
int expectedPayload = 0;
// The sequenceId should be (i % SEQUENCE_ID_LIMIT)
for (int i = 0; i < messages.size(); i++) {
RawMessage message = messages.get(i);
MessageId messageId = new MessageId(message.getId());
Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
expectedPayload = (expectedPayload + 1) % msgCount;
}
}
Aggregations