Examples with StoreRequest io.cdap.cdap.messaging.StoreRequest used on opensource projects

Example 11 with StoreRequest

use of io.cdap.cdap.messaging.StoreRequest in project cdap by cdapio.

the class StoreHandler method publish.

@POST
@Path("/publish")
public BodyConsumer publish(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception {
    TopicId topicId = new NamespaceId(namespace).topic(topic);
    return new SpillableBodyConsumer(Files.createTempFile(tempDir, "tms.publish", ".tmp"), bufferSize) {

        @Override
        protected void processInput(InputStream inputStream, HttpResponder responder) throws Exception {
            StoreRequest storeRequest = createStoreRequest(topicId, request, inputStream);
            // Empty payload is only allowed for transactional publish
            if (!storeRequest.isTransactional() && !storeRequest.hasPayload()) {
                throw new BadRequestException("Empty payload is only allowed for publishing transactional message. Topic: " + topicId);
            }
            // Publish the message and response with the rollback information
            RollbackDetail rollbackInfo = messagingService.publish(storeRequest);
            if (rollbackInfo == null) {
                // Non-tx publish doesn't have rollback info.
                responder.sendStatus(HttpResponseStatus.OK);
            } else {
                ByteBuf response = encodeRollbackDetail(rollbackInfo);
                responder.sendContent(HttpResponseStatus.OK, response, new DefaultHttpHeaders().set(HttpHeaderNames.CONTENT_TYPE, "avro/binary"));
            }
        }
    };
}

Example 12 with StoreRequest

use of io.cdap.cdap.messaging.StoreRequest in project cdap by cdapio.

the class StoreHandler method store.

@POST
@Path("/store")
public BodyConsumer store(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception {
    TopicId topicId = new NamespaceId(namespace).topic(topic);
    return new SpillableBodyConsumer(Files.createTempFile(tempDir, "tms.store", ".tmp"), bufferSize) {

        @Override
        protected void processInput(InputStream inputStream, HttpResponder responder) throws Exception {
            StoreRequest storeRequest = createStoreRequest(topicId, request, inputStream);
            // It must be transactional with payload for store request
            if (!storeRequest.isTransactional() || !storeRequest.hasPayload()) {
                throw new BadRequestException("Store request must be transactional with payload. Topic: " + topicId);
            }
            messagingService.storePayload(storeRequest);
            responder.sendStatus(HttpResponseStatus.OK);
        }
    };
}

Example 13 with StoreRequest

use of io.cdap.cdap.messaging.StoreRequest in project cdap by cdapio.

the class ConcurrentMessageWriterTest method testMultiMaxSequence.

@Test
public void testMultiMaxSequence() throws IOException, InterruptedException {
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // This test the case when multiple StoreRequests combined exceeding the 65536 payload.
    // See testMaxSequence() for more details when it matters
    // Generate 3 StoreRequests, each with 43690 messages
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT / 3 * 2;
    int requestCount = 3;
    List<StoreRequest> requests = new ArrayList<>();
    for (int i = 0; i < requestCount; i++) {
        List<String> payloads = new ArrayList<>(msgCount);
        for (int j = 0; j < msgCount; j++) {
            payloads.add(Integer.toString(j));
        }
        requests.add(new TestStoreRequest(topicId, payloads));
    }
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    // We use a custom metrics collector here to make all the persist calls reached the same latch,
    // since we know that the ConcurrentMessageWriter will emit a metrics "persist.requested" after enqueued but
    // before flushing.
    // This will make all requests batched together
    final CountDownLatch latch = new CountDownLatch(requestCount);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter, new MetricsCollector() {

        @Override
        public void increment(String metricName, long value) {
            if ("persist.requested".equals(metricName)) {
                latch.countDown();
                Uninterruptibles.awaitUninterruptibly(latch);
            }
        }

        @Override
        public void gauge(String metricName, long value) {
            LOG.info("MetricsContext.gauge: {} = {}", metricName, value);
        }
    });
    ExecutorService executor = Executors.newFixedThreadPool(3);
    for (final StoreRequest request : requests) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    writer.persist(request, metadata);
                } catch (IOException e) {
                    LOG.error("Failed to persist", e);
                }
            }
        });
    }
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    // Validates all messages are being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(requestCount * msgCount, messages.size());
    // We expect the payload is in repeated sequence of [0..msgCount-1]
    int expectedPayload = 0;
    // The sequenceId should be (i % SEQUENCE_ID_LIMIT)
    for (int i = 0; i < messages.size(); i++) {
        RawMessage message = messages.get(i);
        MessageId messageId = new MessageId(message.getId());
        Assert.assertEquals(i / StoreRequestWriter.SEQUENCE_ID_LIMIT, messageId.getPublishTimestamp());
        Assert.assertEquals((short) (i % StoreRequestWriter.SEQUENCE_ID_LIMIT), messageId.getSequenceId());
        Assert.assertEquals(expectedPayload, Integer.parseInt(Bytes.toString(message.getPayload())));
        expectedPayload = (expectedPayload + 1) % msgCount;
    }
}

Example 14 with StoreRequest

use of io.cdap.cdap.messaging.StoreRequest in project cdap by cdapio.

the class DefaultAuditPublisher method publish.

@Override
public void publish(MetadataEntity metadataEntity, AuditType auditType, AuditPayload auditPayload) {
    String userId = Objects.firstNonNull(SecurityRequestContext.getUserId(), "");
    AuditMessage auditMessage = new AuditMessage(System.currentTimeMillis(), metadataEntity, userId, auditType, auditPayload);
    LOG.trace("Publishing audit message {}", auditMessage);
    StoreRequest storeRequest = StoreRequestBuilder.of(auditTopic).addPayload(GSON.toJson(auditMessage)).build();
    try {
        Retries.callWithRetries(() -> messagingService.publish(storeRequest), retryStrategy, Retries.ALWAYS_TRUE);
    } catch (TopicNotFoundException e) {
        LOG.error("Missing topic for audit publish: {}", auditTopic);
    } catch (Exception e) {
        LOG.error("Got exception publishing audit message {}. Exception:", auditMessage, e);
    }
}

Example 15 with StoreRequest

use of io.cdap.cdap.messaging.StoreRequest in project cdap by caskdata.

the class MessagingWorkflowStateWriter method addWorkflowNodeState.

@Override
public void addWorkflowNodeState(ProgramRunId workflowRunId, WorkflowNodeStateDetail state) {
    MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_STATE, workflowRunId, GSON.toJsonTree(state));
    StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build();
    try {
        Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE);
    } catch (Exception e) {
        throw new RuntimeException("Failed to publish workflow node state for workflow run " + workflowRunId + "of node " + state.getNodeId() + " with state " + state.getNodeStatus(), e);
    }
}