Examples with MessageFetcher co.cask.cdap.api.messaging.MessageFetcher used on opensource projects

Example 1 with MessageFetcher

use of co.cask.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class MessagingAppTestRun method testSparkMessaging.

@Test
public void testSparkMessaging() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final SparkManager sparkManager = appManager.getSparkManager(MessagingSpark.class.getSimpleName()).start();
    final MessageFetcher fetcher = getMessagingContext().getMessageFetcher();
    final AtomicReference<String> messageId = new AtomicReference<>();
    // Wait for the Spark to create the topic
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE.getNamespace());
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // This is to verify failed transaction is not publishing anything.
    for (String expected : Arrays.asList("start", "block")) {
        Tasks.waitFor(expected, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                    if (!iterator.hasNext()) {
                        return null;
                    }
                    Message message = iterator.next();
                    messageId.set(message.getId());
                    return message.getPayloadAsString();
                }
            }
        }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    }
    // Publish a control message to unblock the Spark execution
    getMessagingContext().getMessagePublisher().publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, "go");
    // Expects a result message as "result-15", where 15 is the sum of 1,2,3,4,5
    Tasks.waitFor("result-15", new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
                if (!iterator.hasNext()) {
                    return null;
                }
                Message message = iterator.next();
                messageId.set(message.getId());
                return message.getPayloadAsString();
            }
        }
    }, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
}

Example 2 with MessageFetcher

use of co.cask.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class DataPipelineTest method testAlertPublisher.

private void testAlertPublisher(Engine engine) throws Exception {
    String sourceName = "alertSource" + engine.name();
    String sinkName = "alertSink" + engine.name();
    String topic = "alertTopic" + engine.name();
    /*
     * source --> nullAlert --> sink
     *               |
     *               |--> TMS publisher
     */
    ETLBatchConfig config = ETLBatchConfig.builder("* * * * *").setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
    ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest-" + engine);
    ApplicationManager appManager = deployApplication(appId, appRequest);
    Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
    StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
    StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
    StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
    DataSetManager<Table> sourceTable = getDataset(sourceName);
    MockSource.writeInput(sourceTable, ImmutableList.of(record1, record2, alertRecord));
    WorkflowManager manager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    manager.start();
    manager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
    DataSetManager<Table> sinkTable = getDataset(sinkName);
    Set<StructuredRecord> actual = new HashSet<>(MockSink.readOutput(sinkTable));
    Set<StructuredRecord> expected = ImmutableSet.of(record1, record2);
    Assert.assertEquals(expected, actual);
    MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
    Set<Alert> actualMessages = new HashSet<>();
    try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
        while (iter.hasNext()) {
            Message message = iter.next();
            Alert alert = GSON.fromJson(message.getPayloadAsString(), Alert.class);
            actualMessages.add(alert);
        }
    }
    Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<String, String>()));
    Assert.assertEquals(expectedMessages, actualMessages);
    validateMetric(3, appId, "source.records.out");
    validateMetric(3, appId, "nullAlert.records.in");
    validateMetric(2, appId, "nullAlert.records.out");
    validateMetric(1, appId, "nullAlert.records.alert");
    validateMetric(2, appId, "sink.records.in");
    validateMetric(1, appId, "tms.records.in");
}

Example 3 with MessageFetcher

use of co.cask.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class MessagingAppTestRun method testWithWorker.

@Test
public void testWithWorker() throws Exception {
    ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
    final WorkerManager workerManager = appManager.getWorkerManager(MessagingApp.MessagingWorker.class.getSimpleName()).start();
    MessagingContext messagingContext = getMessagingContext();
    final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE);
    // Wait for the worker to create the topic
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            try {
                messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
                return true;
            } catch (TopicNotFoundException e) {
                return false;
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish a message
    String message = "message";
    MessagePublisher messagePublisher = messagingContext.getMessagePublisher();
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message);
    // The worker will publish back a message with payload as concat(message, message)
    final MessageFetcher messageFetcher = messagingContext.getMessageFetcher();
    Tasks.waitFor(message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Publish concat(message + message) to the app
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.TOPIC, message + message);
    // timeout.
    try {
        Tasks.waitFor(message + message + message + message, new Callable<String>() {

            @Override
            public String call() throws Exception {
                try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                    Message message = Iterators.getLast(iterator, null);
                    return message == null ? null : message.getPayloadAsString();
                }
            }
        }, 2, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Assert.fail("Expected timeout exception");
    } catch (TimeoutException e) {
    // expected
    }
    // Now publish a message to the control topic, to unblock the transaction block.
    messagePublisher.publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, message);
    // Should expect a new message as concat(message, message, message, message)
    Tasks.waitFor(message + message + message + message, new Callable<String>() {

        @Override
        public String call() throws Exception {
            try (CloseableIterator<Message> iterator = messageFetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, Integer.MAX_VALUE, 0L)) {
                Message message = Iterators.getLast(iterator, null);
                return message == null ? null : message.getPayloadAsString();
            }
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Wait for the worker to finish and verify that it completes successfully.
    workerManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.SECONDS);
}

Example 4 with MessageFetcher

use of co.cask.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class RuntimeMonitorTest method testRunTimeMonitor.

@Test
public void testRunTimeMonitor() throws Exception {
    Map<String, String> topics = new HashMap<>();
    topics.put(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC, "status");
    httpServer.createContext("/v3/runtime/monitor/topics", new HttpHandler() {

        public void handle(HttpExchange exchange) throws IOException {
            byte[] response = GSON.toJson(topics).getBytes();
            exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length);
            exchange.getResponseBody().write(response);
            exchange.close();
        }
    });
    ConnectionConfig connectionConfig = ConnectionConfig.builder().setHostname(address.getHostName()).setPort(1234).setSSLEnabled(false).build();
    ClientConfig.Builder clientConfigBuilder = ClientConfig.builder().setDefaultReadTimeout(20000).setConnectionConfig(connectionConfig);
    int limit = 2;
    MessagingContext messagingContext = new MultiThreadMessagingContext(messagingService);
    RuntimeMonitor runtimeMonitor = new RuntimeMonitor(new ProgramRunId("test", "app1", ProgramType.WORKFLOW, "p1", "run1"), cConf, messagingContext.getMessagePublisher(), clientConfigBuilder.build());
    Map<String, List<MonitorMessage>> messages = new LinkedHashMap<>();
    ArrayList<MonitorMessage> list = new ArrayList<>();
    list.add(new MonitorMessage("1", "message1"));
    list.add(new MonitorMessage("2", "message2"));
    list.add(new MonitorMessage("3", "message3"));
    list.add(new MonitorMessage("4", "message4"));
    list.add(new MonitorMessage("5", "message5"));
    list.add(new MonitorMessage("6", "message6"));
    list.add(new MonitorMessage("7", "message7"));
    list.add(new MonitorMessage("8", "message8"));
    list.add(new MonitorMessage("9", "message9"));
    list.add(new MonitorMessage("10", "message10"));
    messages.put("status", list);
    httpServer.createContext("/v3/runtime/metadata", new HttpHandler() {

        int count = 0;

        public void handle(HttpExchange exchange) throws IOException {
            Map<String, List<MonitorMessage>> toSend = new LinkedHashMap<>();
            ArrayList<MonitorMessage> list = new ArrayList<>();
            int start = count;
            int i = 0;
            for (MonitorMessage message : messages.get("status")) {
                if (start <= i && i < start + limit) {
                    list.add(message);
                    count++;
                }
                i++;
            }
            toSend.put("status", list);
            byte[] response = GSON.toJson(toSend).getBytes();
            exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.length);
            exchange.getResponseBody().write(response);
            exchange.close();
        }
    });
    HashSet<String> expected = new LinkedHashSet<>();
    expected.add("message1");
    expected.add("message2");
    expected.add("message3");
    expected.add("message4");
    expected.add("message5");
    expected.add("message6");
    expected.add("message7");
    expected.add("message8");
    expected.add("message9");
    expected.add("message10");
    HashSet<String> actual = new LinkedHashSet<>();
    final String[] messageId = { null };
    runtimeMonitor.startAndWait();
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            MessageFetcher messageFetcher = messagingContext.getMessageFetcher();
            try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.SYSTEM.getNamespace(), cConf.get(Constants.AppFabric.PROGRAM_STATUS_EVENT_TOPIC), 2, messageId[0])) {
                while (iter.hasNext()) {
                    Message message = iter.next();
                    messageId[0] = message.getId();
                    actual.add(message.getPayloadAsString());
                }
            }
            return expected.size() == actual.size() && expected.equals(actual);
        }
    }, 5, TimeUnit.MINUTES);
    runtimeMonitor.stopAndWait();
}

Example 5 with MessageFetcher

use of co.cask.cdap.api.messaging.MessageFetcher in project cdap by caskdata.

the class DataStreamsTest method testAlertPublisher.

@Test
public void testAlertPublisher() throws Exception {
    String sinkName = "alertSink";
    final String topic = "alertTopic";
    Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
    StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
    StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
    StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
    /*
     * source --> nullAlert --> sink
     *               |
     *               |--> TMS publisher
     */
    DataStreamsConfig config = DataStreamsConfig.builder().setBatchInterval("5s").addStage(new ETLStage("source", MockSource.getPlugin(schema, ImmutableList.of(record1, record2, alertRecord)))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms").build();
    AppRequest<DataStreamsConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
    ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest");
    ApplicationManager appManager = deployApplication(appId, appRequest);
    SparkManager sparkManager = appManager.getSparkManager(DataStreamsSparkLauncher.NAME);
    sparkManager.start();
    sparkManager.waitForStatus(true, 10, 1);
    final Set<StructuredRecord> expectedRecords = ImmutableSet.of(record1, record2);
    final Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<String, String>()));
    final DataSetManager<Table> sinkTable = getDataset(sinkName);
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            // get alerts from TMS
            try {
                getMessagingAdmin(NamespaceId.DEFAULT.getNamespace()).getTopicProperties(topic);
            } catch (TopicNotFoundException e) {
                return false;
            }
            MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
            Set<Alert> actualMessages = new HashSet<>();
            try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
                while (iter.hasNext()) {
                    Message message = iter.next();
                    Alert alert = GSON.fromJson(message.getPayloadAsString(), Alert.class);
                    actualMessages.add(alert);
                }
            }
            // get records from sink
            sinkTable.flush();
            Set<StructuredRecord> outputRecords = new HashSet<>();
            outputRecords.addAll(MockSink.readOutput(sinkTable));
            return expectedRecords.equals(outputRecords) && expectedMessages.equals(actualMessages);
        }
    }, 4, TimeUnit.MINUTES);
    sparkManager.stop();
    sparkManager.waitForStatus(false, 10, 1);
    validateMetric(appId, "source.records.out", 3);
    validateMetric(appId, "nullAlert.records.in", 3);
    validateMetric(appId, "nullAlert.records.out", 2);
    validateMetric(appId, "nullAlert.records.alert", 1);
    validateMetric(appId, "sink.records.in", 2);
    validateMetric(appId, "tms.records.in", 1);
}