Search in sources :

Example 1 with MessageFetcher

use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class TetheringServerHandler method connectControlChannel.

/**
 * Sends control commands to the client.
 */
@GET
@Path("/tethering/controlchannels/{peer}")
public void connectControlChannel(FullHttpRequest request, HttpResponder responder, @PathParam("peer") String peer, @QueryParam("messageId") String messageId) throws IOException, NotImplementedException, PeerNotFoundException, ForbiddenException, BadRequestException {
    checkTetheringServerEnabled();
    store.updatePeerTimestamp(peer);
    TetheringStatus tetheringStatus = store.getPeer(peer).getTetheringStatus();
    if (tetheringStatus == TetheringStatus.PENDING) {
        throw new PeerNotFoundException(String.format("Peer %s not found", peer));
    } else if (tetheringStatus == TetheringStatus.REJECTED) {
        responder.sendStatus(HttpResponseStatus.FORBIDDEN);
        throw new ForbiddenException(String.format("Peer %s is not authorized", peer));
    }
    List<TetheringControlResponse> controlResponses = new ArrayList<>();
    MessageFetcher fetcher = messagingContext.getMessageFetcher();
    TopicId topic = new TopicId(NamespaceId.SYSTEM.getNamespace(), topicPrefix + peer);
    String lastMessageId = messageId;
    try (CloseableIterator<Message> iterator = fetcher.fetch(topic.getNamespace(), topic.getTopic(), 1, messageId)) {
        while (iterator.hasNext()) {
            Message message = iterator.next();
            TetheringControlMessage controlMessage = GSON.fromJson(message.getPayloadAsString(StandardCharsets.UTF_8), TetheringControlMessage.class);
            lastMessageId = message.getId();
            controlResponses.add(new TetheringControlResponse(lastMessageId, controlMessage));
        }
    } catch (TopicNotFoundException e) {
        LOG.warn("Received control connection from peer {} that's not tethered", peer);
    } catch (IllegalArgumentException e) {
        throw new BadRequestException(String.format("Invalid message id %s", messageId));
    }
    if (controlResponses.isEmpty()) {
        controlResponses.add(new TetheringControlResponse(lastMessageId, new TetheringControlMessage(TetheringControlMessage.Type.KEEPALIVE)));
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(controlResponses.toArray(new TetheringControlResponse[0]), TetheringControlResponse[].class));
}
Also used : ForbiddenException(io.cdap.cdap.common.ForbiddenException) MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) Message(io.cdap.cdap.api.messaging.Message) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) ArrayList(java.util.ArrayList) BadRequestException(io.cdap.cdap.common.BadRequestException) TopicId(io.cdap.cdap.proto.id.TopicId) Path(javax.ws.rs.Path) GET(javax.ws.rs.GET)

Example 2 with MessageFetcher

use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class DataPipelineTest method testAlertPublisher.

private void testAlertPublisher(Engine engine) throws Exception {
    String sourceName = "alertSource" + engine.name();
    String sinkName = "alertSink" + engine.name();
    String topic = "alertTopic" + engine.name();
    /*
     * source --> nullAlert --> sink
     *               |
     *               |--> TMS publisher
     */
    ETLBatchConfig config = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms alert", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms alert").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
    ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest-" + engine);
    ApplicationManager appManager = deployApplication(appId, appRequest);
    Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
    StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
    StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
    StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
    DataSetManager<Table> sourceTable = getDataset(sourceName);
    MockSource.writeInput(sourceTable, ImmutableList.of(record1, record2, alertRecord));
    WorkflowManager manager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    manager.start();
    manager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
    DataSetManager<Table> sinkTable = getDataset(sinkName);
    Set<StructuredRecord> actual = new HashSet<>(MockSink.readOutput(sinkTable));
    Set<StructuredRecord> expected = ImmutableSet.of(record1, record2);
    Assert.assertEquals(expected, actual);
    MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
    Set<Alert> actualMessages = new HashSet<>();
    try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
        while (iter.hasNext()) {
            Message message = iter.next();
            Alert alert = message.decodePayload(r -> GSON.fromJson(r, Alert.class));
            actualMessages.add(alert);
        }
    }
    Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<>()));
    Assert.assertEquals(expectedMessages, actualMessages);
    validateMetric(3, appId, "source.records.out");
    validateMetric(3, appId, "nullAlert.records.in");
    validateMetric(2, appId, "nullAlert.records.out");
    validateMetric(1, appId, "nullAlert.records.alert");
    validateMetric(2, appId, "sink.records.in");
    validateMetric(1, appId, "tms alert.records.in");
}
Also used : ApplicationManager(io.cdap.cdap.test.ApplicationManager) MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) Table(io.cdap.cdap.api.dataset.table.Table) KeyValueTable(io.cdap.cdap.api.dataset.lib.KeyValueTable) Message(io.cdap.cdap.api.messaging.Message) SpamMessage(io.cdap.cdap.datapipeline.mock.SpamMessage) HashMap(java.util.HashMap) Schema(io.cdap.cdap.api.data.schema.Schema) WorkflowManager(io.cdap.cdap.test.WorkflowManager) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) AppRequest(io.cdap.cdap.proto.artifact.AppRequest) ETLBatchConfig(io.cdap.cdap.etl.proto.v2.ETLBatchConfig) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) Alert(io.cdap.cdap.etl.api.Alert) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) HashSet(java.util.HashSet)

Example 3 with MessageFetcher

use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class DataStreamsTest method testAlertPublisher.

@Test
public void testAlertPublisher() throws Exception {
    String sinkName = "alertSink";
    String topic = "alertTopic";
    Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
    StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
    StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
    StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
    /*
     * source --> nullAlert --> sink
     *               |
     *               |--> TMS publisher
     */
    DataStreamsConfig config = DataStreamsConfig.builder().setBatchInterval("5s").addStage(new ETLStage("source", MockSource.getPlugin(schema, ImmutableList.of(record1, record2, alertRecord)))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms").setCheckpointDir(checkpointDir).build();
    AppRequest<DataStreamsConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
    ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest");
    ApplicationManager appManager = deployApplication(appId, appRequest);
    SparkManager sparkManager = appManager.getSparkManager(DataStreamsSparkLauncher.NAME);
    sparkManager.start();
    sparkManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
    final Set<StructuredRecord> expectedRecords = ImmutableSet.of(record1, record2);
    final Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<String, String>()));
    final DataSetManager<Table> sinkTable = getDataset(sinkName);
    Tasks.waitFor(true, () -> {
        // get alerts from TMS
        try {
            getMessagingAdmin(NamespaceId.DEFAULT.getNamespace()).getTopicProperties(topic);
        } catch (TopicNotFoundException e) {
            return false;
        }
        MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
        Set<Alert> actualMessages = new HashSet<>();
        try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
            while (iter.hasNext()) {
                Message message = iter.next();
                Alert alert = message.decodePayload(r -> GSON.fromJson(r, Alert.class));
                actualMessages.add(alert);
            }
        }
        // get records from sink
        sinkTable.flush();
        Set<StructuredRecord> outputRecords = new HashSet<>(MockSink.readOutput(sinkTable));
        return expectedRecords.equals(outputRecords) && expectedMessages.equals(actualMessages);
    }, 4, TimeUnit.MINUTES);
    sparkManager.stop();
    sparkManager.waitForStopped(10, TimeUnit.SECONDS);
    validateMetric(appId, "source.records.out", 3);
    validateMetric(appId, "nullAlert.records.in", 3);
    validateMetric(appId, "nullAlert.records.out", 2);
    validateMetric(appId, "nullAlert.records.alert", 1);
    validateMetric(appId, "sink.records.in", 2);
    validateMetric(appId, "tms.records.in", 1);
}
Also used : ApplicationManager(io.cdap.cdap.test.ApplicationManager) MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) SparkManager(io.cdap.cdap.test.SparkManager) Table(io.cdap.cdap.api.dataset.table.Table) Message(io.cdap.cdap.api.messaging.Message) HashMap(java.util.HashMap) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) Schema(io.cdap.cdap.api.data.schema.Schema) StructuredRecord(io.cdap.cdap.api.data.format.StructuredRecord) DataStreamsConfig(io.cdap.cdap.etl.proto.v2.DataStreamsConfig) AppRequest(io.cdap.cdap.proto.artifact.AppRequest) ETLStage(io.cdap.cdap.etl.proto.v2.ETLStage) Alert(io.cdap.cdap.etl.api.Alert) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 4 with MessageFetcher

use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class AbstractMessagingPollingService method fetchMessages.

/**
 * Fetching messages from TMS.
 */
protected List<Message> fetchMessages(@Nullable String messageId) throws TopicNotFoundException, IOException {
    List<Message> messages = new ArrayList<>();
    LOG.trace("Fetching from topic '{}' with messageId '{}'", topicId, messageId);
    MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
    try (CloseableIterator<Message> iterator = messageFetcher.fetch(getTopicId().getNamespace(), getTopicId().getTopic(), fetchSize, messageId)) {
        while (iterator.hasNext() && state() == State.RUNNING) {
            messages.add(iterator.next());
        }
    }
    return messages;
}
Also used : MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) Message(io.cdap.cdap.api.messaging.Message) ArrayList(java.util.ArrayList)

Example 5 with MessageFetcher

use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class MessagingAppTestRun method testSparkMessaging.

@Test
public void testSparkMessaging() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final SparkManager sparkManager = appManager.getSparkManager(MessagingSpark.class.getSimpleName()).start();
    final MessageFetcher fetcher = getMessagingContext().getMessageFetcher();
    final AtomicReference<String> messageId = new AtomicReference<>();
    // Wait for the Spark to create the topic
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE.getNamespace());
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // This is to verify failed transaction is not publishing anything.
    for (String expected : Arrays.asList("start", "block")) {
        Tasks.waitFor(expected, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                    if (!iterator.hasNext()) {
                        return null;
                    }
                    Message message = iterator.next();
                    messageId.set(message.getId());
                    return message.getPayloadAsString();
                }
            }
        }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    }
    // Publish a control message to unblock the Spark execution
    getMessagingContext().getMessagePublisher().publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, "go");
    // Expects a result message as "result-15", where 15 is the sum of 1,2,3,4,5
    Tasks.waitFor("result-15", new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                if (!iterator.hasNext()) {
                    return null;
                }
                Message message = iterator.next();
                messageId.set(message.getId());
                return message.getPayloadAsString();
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
}
Also used : ApplicationManager(io.cdap.cdap.test.ApplicationManager) MessageFetcher(io.cdap.cdap.api.messaging.MessageFetcher) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) SparkManager(io.cdap.cdap.test.SparkManager) Message(io.cdap.cdap.api.messaging.Message) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) AtomicReference(java.util.concurrent.atomic.AtomicReference) TimeoutException(java.util.concurrent.TimeoutException) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException) MessagingAdmin(io.cdap.cdap.api.messaging.MessagingAdmin) Test(org.junit.Test)

Aggregations

Message (io.cdap.cdap.api.messaging.Message)6 MessageFetcher (io.cdap.cdap.api.messaging.MessageFetcher)6 TopicNotFoundException (io.cdap.cdap.api.messaging.TopicNotFoundException)4 ApplicationManager (io.cdap.cdap.test.ApplicationManager)4 Test (org.junit.Test)3 StructuredRecord (io.cdap.cdap.api.data.format.StructuredRecord)2 Schema (io.cdap.cdap.api.data.schema.Schema)2 CloseableIterator (io.cdap.cdap.api.dataset.lib.CloseableIterator)2 Table (io.cdap.cdap.api.dataset.table.Table)2 MessagingAdmin (io.cdap.cdap.api.messaging.MessagingAdmin)2 Alert (io.cdap.cdap.etl.api.Alert)2 ETLStage (io.cdap.cdap.etl.proto.v2.ETLStage)2 AppRequest (io.cdap.cdap.proto.artifact.AppRequest)2 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)2 SparkManager (io.cdap.cdap.test.SparkManager)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 TimeoutException (java.util.concurrent.TimeoutException)2 KeyValueTable (io.cdap.cdap.api.dataset.lib.KeyValueTable)1