use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.
the class TetheringServerHandler method connectControlChannel.
/**
* Sends control commands to the client.
*/
@GET
@Path("/tethering/controlchannels/{peer}")
public void connectControlChannel(FullHttpRequest request, HttpResponder responder, @PathParam("peer") String peer, @QueryParam("messageId") String messageId) throws IOException, NotImplementedException, PeerNotFoundException, ForbiddenException, BadRequestException {
checkTetheringServerEnabled();
store.updatePeerTimestamp(peer);
TetheringStatus tetheringStatus = store.getPeer(peer).getTetheringStatus();
if (tetheringStatus == TetheringStatus.PENDING) {
throw new PeerNotFoundException(String.format("Peer %s not found", peer));
} else if (tetheringStatus == TetheringStatus.REJECTED) {
responder.sendStatus(HttpResponseStatus.FORBIDDEN);
throw new ForbiddenException(String.format("Peer %s is not authorized", peer));
}
List<TetheringControlResponse> controlResponses = new ArrayList<>();
MessageFetcher fetcher = messagingContext.getMessageFetcher();
TopicId topic = new TopicId(NamespaceId.SYSTEM.getNamespace(), topicPrefix + peer);
String lastMessageId = messageId;
try (CloseableIterator<Message> iterator = fetcher.fetch(topic.getNamespace(), topic.getTopic(), 1, messageId)) {
while (iterator.hasNext()) {
Message message = iterator.next();
TetheringControlMessage controlMessage = GSON.fromJson(message.getPayloadAsString(StandardCharsets.UTF_8), TetheringControlMessage.class);
lastMessageId = message.getId();
controlResponses.add(new TetheringControlResponse(lastMessageId, controlMessage));
}
} catch (TopicNotFoundException e) {
LOG.warn("Received control connection from peer {} that's not tethered", peer);
} catch (IllegalArgumentException e) {
throw new BadRequestException(String.format("Invalid message id %s", messageId));
}
if (controlResponses.isEmpty()) {
controlResponses.add(new TetheringControlResponse(lastMessageId, new TetheringControlMessage(TetheringControlMessage.Type.KEEPALIVE)));
}
responder.sendJson(HttpResponseStatus.OK, GSON.toJson(controlResponses.toArray(new TetheringControlResponse[0]), TetheringControlResponse[].class));
}
use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.
the class DataPipelineTest method testAlertPublisher.
private void testAlertPublisher(Engine engine) throws Exception {
String sourceName = "alertSource" + engine.name();
String sinkName = "alertSink" + engine.name();
String topic = "alertTopic" + engine.name();
/*
* source --> nullAlert --> sink
* |
* |--> TMS publisher
*/
ETLBatchConfig config = ETLBatchConfig.builder().setEngine(engine).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms alert", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms alert").build();
AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest-" + engine);
ApplicationManager appManager = deployApplication(appId, appRequest);
Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
DataSetManager<Table> sourceTable = getDataset(sourceName);
MockSource.writeInput(sourceTable, ImmutableList.of(record1, record2, alertRecord));
WorkflowManager manager = appManager.getWorkflowManager(SmartWorkflow.NAME);
manager.start();
manager.waitForRun(ProgramRunStatus.COMPLETED, 3, TimeUnit.MINUTES);
DataSetManager<Table> sinkTable = getDataset(sinkName);
Set<StructuredRecord> actual = new HashSet<>(MockSink.readOutput(sinkTable));
Set<StructuredRecord> expected = ImmutableSet.of(record1, record2);
Assert.assertEquals(expected, actual);
MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
Set<Alert> actualMessages = new HashSet<>();
try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
while (iter.hasNext()) {
Message message = iter.next();
Alert alert = message.decodePayload(r -> GSON.fromJson(r, Alert.class));
actualMessages.add(alert);
}
}
Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<>()));
Assert.assertEquals(expectedMessages, actualMessages);
validateMetric(3, appId, "source.records.out");
validateMetric(3, appId, "nullAlert.records.in");
validateMetric(2, appId, "nullAlert.records.out");
validateMetric(1, appId, "nullAlert.records.alert");
validateMetric(2, appId, "sink.records.in");
validateMetric(1, appId, "tms alert.records.in");
}
use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.
the class DataStreamsTest method testAlertPublisher.
@Test
public void testAlertPublisher() throws Exception {
String sinkName = "alertSink";
String topic = "alertTopic";
Schema schema = Schema.recordOf("x", Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.LONG))));
StructuredRecord record1 = StructuredRecord.builder(schema).set("id", 1L).build();
StructuredRecord record2 = StructuredRecord.builder(schema).set("id", 2L).build();
StructuredRecord alertRecord = StructuredRecord.builder(schema).build();
/*
* source --> nullAlert --> sink
* |
* |--> TMS publisher
*/
DataStreamsConfig config = DataStreamsConfig.builder().setBatchInterval("5s").addStage(new ETLStage("source", MockSource.getPlugin(schema, ImmutableList.of(record1, record2, alertRecord)))).addStage(new ETLStage("nullAlert", NullAlertTransform.getPlugin("id"))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addStage(new ETLStage("tms", TMSAlertPublisher.getPlugin(topic, NamespaceId.DEFAULT.getNamespace()))).addConnection("source", "nullAlert").addConnection("nullAlert", "sink").addConnection("nullAlert", "tms").setCheckpointDir(checkpointDir).build();
AppRequest<DataStreamsConfig> appRequest = new AppRequest<>(APP_ARTIFACT, config);
ApplicationId appId = NamespaceId.DEFAULT.app("AlertTest");
ApplicationManager appManager = deployApplication(appId, appRequest);
SparkManager sparkManager = appManager.getSparkManager(DataStreamsSparkLauncher.NAME);
sparkManager.start();
sparkManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS);
final Set<StructuredRecord> expectedRecords = ImmutableSet.of(record1, record2);
final Set<Alert> expectedMessages = ImmutableSet.of(new Alert("nullAlert", new HashMap<String, String>()));
final DataSetManager<Table> sinkTable = getDataset(sinkName);
Tasks.waitFor(true, () -> {
// get alerts from TMS
try {
getMessagingAdmin(NamespaceId.DEFAULT.getNamespace()).getTopicProperties(topic);
} catch (TopicNotFoundException e) {
return false;
}
MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
Set<Alert> actualMessages = new HashSet<>();
try (CloseableIterator<Message> iter = messageFetcher.fetch(NamespaceId.DEFAULT.getNamespace(), topic, 5, 0)) {
while (iter.hasNext()) {
Message message = iter.next();
Alert alert = message.decodePayload(r -> GSON.fromJson(r, Alert.class));
actualMessages.add(alert);
}
}
// get records from sink
sinkTable.flush();
Set<StructuredRecord> outputRecords = new HashSet<>(MockSink.readOutput(sinkTable));
return expectedRecords.equals(outputRecords) && expectedMessages.equals(actualMessages);
}, 4, TimeUnit.MINUTES);
sparkManager.stop();
sparkManager.waitForStopped(10, TimeUnit.SECONDS);
validateMetric(appId, "source.records.out", 3);
validateMetric(appId, "nullAlert.records.in", 3);
validateMetric(appId, "nullAlert.records.out", 2);
validateMetric(appId, "nullAlert.records.alert", 1);
validateMetric(appId, "sink.records.in", 2);
validateMetric(appId, "tms.records.in", 1);
}
use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.
the class AbstractMessagingPollingService method fetchMessages.
/**
* Fetching messages from TMS.
*/
protected List<Message> fetchMessages(@Nullable String messageId) throws TopicNotFoundException, IOException {
List<Message> messages = new ArrayList<>();
LOG.trace("Fetching from topic '{}' with messageId '{}'", topicId, messageId);
MessageFetcher messageFetcher = getMessagingContext().getMessageFetcher();
try (CloseableIterator<Message> iterator = messageFetcher.fetch(getTopicId().getNamespace(), getTopicId().getTopic(), fetchSize, messageId)) {
while (iterator.hasNext() && state() == State.RUNNING) {
messages.add(iterator.next());
}
}
return messages;
}
use of io.cdap.cdap.api.messaging.MessageFetcher in project cdap by caskdata.
the class MessagingAppTestRun method testSparkMessaging.
@Test
public void testSparkMessaging() throws Exception {
ApplicationManager appManager = deployWithArtifact(NAMESPACE, MessagingApp.class, artifactJar);
final SparkManager sparkManager = appManager.getSparkManager(MessagingSpark.class.getSimpleName()).start();
final MessageFetcher fetcher = getMessagingContext().getMessageFetcher();
final AtomicReference<String> messageId = new AtomicReference<>();
// Wait for the Spark to create the topic
final MessagingAdmin messagingAdmin = getMessagingAdmin(NAMESPACE.getNamespace());
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
messagingAdmin.getTopicProperties(MessagingApp.TOPIC);
return true;
} catch (TopicNotFoundException e) {
return false;
}
}
}, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// This is to verify failed transaction is not publishing anything.
for (String expected : Arrays.asList("start", "block")) {
Tasks.waitFor(expected, new Callable<String>() {
@Override
public String call() throws Exception {
try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
if (!iterator.hasNext()) {
return null;
}
Message message = iterator.next();
messageId.set(message.getId());
return message.getPayloadAsString();
}
}
}, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
// Publish a control message to unblock the Spark execution
getMessagingContext().getMessagePublisher().publish(NAMESPACE.getNamespace(), MessagingApp.CONTROL_TOPIC, "go");
// Expects a result message as "result-15", where 15 is the sum of 1,2,3,4,5
Tasks.waitFor("result-15", new Callable<String>() {
@Override
public String call() throws Exception {
try (CloseableIterator<Message> iterator = fetcher.fetch(NAMESPACE.getNamespace(), MessagingApp.TOPIC, 1, messageId.get())) {
if (!iterator.hasNext()) {
return null;
}
Message message = iterator.next();
messageId.set(message.getId());
return message.getPayloadAsString();
}
}
}, 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
sparkManager.waitForRun(ProgramRunStatus.COMPLETED, 60, TimeUnit.SECONDS);
}
Aggregations