use of io.cdap.cdap.proto.id.TopicId in project cdap by cdapio.
the class DefaultMetricStore method getMetricsProcessorStats.
/**
* Read the metrics processing stats from meta table and return the map of topic information to stats
* @return Map of topic to metrics processing stats
* @throws Exception
*/
@Override
public Map<String, MetricsProcessorStatus> getMetricsProcessorStats() throws Exception {
MetricsConsumerMetaTable metaTable = metaTableSupplier.get();
Map<String, MetricsProcessorStatus> processMap = new HashMap<>();
for (TopicId topicId : metricsTopics) {
TopicProcessMeta topicProcessMeta = metaTable.getTopicProcessMeta(new TopicIdMetaKey(topicId));
if (topicProcessMeta != null) {
MessageId messageId = new MessageId(topicProcessMeta.getMessageId());
MetricsMessageId metricsMessageId = new MetricsMessageId(messageId.getPublishTimestamp(), messageId.getSequenceId(), messageId.getPayloadWriteTimestamp(), messageId.getPayloadSequenceId());
processMap.put(topicId.getTopic(), new MetricsProcessorStatus(metricsMessageId, topicProcessMeta.getOldestMetricsTimestamp(), topicProcessMeta.getLatestMetricsTimestamp(), topicProcessMeta.getMessagesProcessed(), topicProcessMeta.getLastProcessedTimestamp()));
}
}
return processMap;
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by cdapio.
the class TetheringServerHandler method connectControlChannel.
/**
* Sends control commands to the client.
*/
@GET
@Path("/tethering/controlchannels/{peer}")
public void connectControlChannel(FullHttpRequest request, HttpResponder responder, @PathParam("peer") String peer, @QueryParam("messageId") String messageId) throws IOException, NotImplementedException, PeerNotFoundException, ForbiddenException, BadRequestException {
checkTetheringServerEnabled();
store.updatePeerTimestamp(peer);
TetheringStatus tetheringStatus = store.getPeer(peer).getTetheringStatus();
if (tetheringStatus == TetheringStatus.PENDING) {
throw new PeerNotFoundException(String.format("Peer %s not found", peer));
} else if (tetheringStatus == TetheringStatus.REJECTED) {
responder.sendStatus(HttpResponseStatus.FORBIDDEN);
throw new ForbiddenException(String.format("Peer %s is not authorized", peer));
}
List<TetheringControlResponse> controlResponses = new ArrayList<>();
MessageFetcher fetcher = messagingContext.getMessageFetcher();
TopicId topic = new TopicId(NamespaceId.SYSTEM.getNamespace(), topicPrefix + peer);
String lastMessageId = messageId;
try (CloseableIterator<Message> iterator = fetcher.fetch(topic.getNamespace(), topic.getTopic(), 1, messageId)) {
while (iterator.hasNext()) {
Message message = iterator.next();
TetheringControlMessage controlMessage = GSON.fromJson(message.getPayloadAsString(StandardCharsets.UTF_8), TetheringControlMessage.class);
lastMessageId = message.getId();
controlResponses.add(new TetheringControlResponse(lastMessageId, controlMessage));
}
} catch (TopicNotFoundException e) {
LOG.warn("Received control connection from peer {} that's not tethered", peer);
} catch (IllegalArgumentException e) {
throw new BadRequestException(String.format("Invalid message id %s", messageId));
}
if (controlResponses.isEmpty()) {
controlResponses.add(new TetheringControlResponse(lastMessageId, new TetheringControlMessage(TetheringControlMessage.Type.KEEPALIVE)));
}
responder.sendJson(HttpResponseStatus.OK, GSON.toJson(controlResponses.toArray(new TetheringControlResponse[0]), TetheringControlResponse[].class));
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by cdapio.
the class TetheringServerHandler method createTethering.
/**
* Creates a tethering with a client.
*/
@PUT
@Path("/tethering/connections/{peer}")
public void createTethering(FullHttpRequest request, HttpResponder responder, @PathParam("peer") String peer) throws NotImplementedException, IOException {
checkTetheringServerEnabled();
contextAccessEnforcer.enforce(InstanceId.SELF, InstancePermission.TETHER);
String content = request.content().toString(StandardCharsets.UTF_8);
TetheringConnectionRequest tetherRequest = GSON.fromJson(content, TetheringConnectionRequest.class);
TopicId topicId = new TopicId(NamespaceId.SYSTEM.getNamespace(), topicPrefix + peer);
try {
messagingService.createTopic(new TopicMetadata(topicId, Collections.emptyMap()));
} catch (TopicAlreadyExistsException e) {
LOG.warn("Topic {} already exists", topicId);
} catch (IOException e) {
LOG.error("Failed to create topic {}", topicId, e);
throw e;
}
// We don't need to keep track of the client metadata on the server side.
PeerMetadata peerMetadata = new PeerMetadata(tetherRequest.getNamespaceAllocations(), Collections.emptyMap(), tetherRequest.getDescription());
// We don't store the peer endpoint on the server side because the connection is initiated by the client.
PeerInfo peerInfo = new PeerInfo(peer, null, TetheringStatus.PENDING, peerMetadata, tetherRequest.getRequestTime());
try {
store.addPeer(peerInfo);
} catch (PeerAlreadyExistsException pae) {
// Peer is already configured, treat this as a no-op.
responder.sendStatus(HttpResponseStatus.OK);
return;
} catch (Exception e) {
try {
messagingService.deleteTopic(topicId);
} catch (Exception ex) {
e.addSuppressed(ex);
}
throw new IOException("Failed to create tethering with peer " + peer, e);
}
responder.sendStatus(HttpResponseStatus.OK);
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by cdapio.
the class CoreSchedulerServiceTest method testProgramEvents.
@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
// Deploy the app
deploy(AppWithMultipleSchedules.class, 200);
CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
// These notifications should not trigger the program
ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
programStateWriter.running(anotherWorkflowRun, null);
long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.error(anotherWorkflowRun, null);
waitUntilProcessed(programEventTopic, lastProcessed);
ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
programStateWriter.running(someWorkflowRun, null);
lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.killed(someWorkflowRun);
waitUntilProcessed(programEventTopic, lastProcessed);
Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
// Enable the schedule
scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
// Start a program with user arguments
startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
// Wait for a completed run record
waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
// There should be 2 entries in tokenData
Assert.assertEquals(2, tokenData.size());
// The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
// The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
use of io.cdap.cdap.proto.id.TopicId in project cdap by cdapio.
the class RuntimeClientServerTest method testSmallMessage.
@Test
public void testSmallMessage() throws Exception {
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(RunIds.generate());
TopicId topicId = NamespaceId.SYSTEM.topic("topic");
List<Message> messages = new ArrayList<>();
messages.add(createMessage(Math.max(1, RuntimeClient.CHUNK_SIZE / 4)));
runtimeClient.sendMessages(programRunId, topicId, messages.iterator());
assertMessages(topicId, messages);
}
Aggregations