Examples with TopicId co.cask.cdap.proto.id.TopicId used on opensource projects

Example 56 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MetadataHandler method getTopic.

@GET
@Path("/topics/{topic}")
public void getTopic(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception {
    TopicId topicId = new NamespaceId(namespace).topic(topic);
    TopicMetadata metadata = messagingService.getTopic(topicId);
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(metadata.getProperties(), TOPIC_PROPERTY_TYPE));
}

Example 57 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class CoreMessagingService method startUp.

@Override
protected void startUp() throws Exception {
    Queue<TopicId> asyncCreationTopics = new LinkedList<>();
    Set<TopicId> systemTopics = MessagingServiceUtils.getSystemTopics(cConf, true);
    for (TopicId topic : systemTopics) {
        createSystemTopic(topic, asyncCreationTopics);
    }
    if (!asyncCreationTopics.isEmpty()) {
        startAsyncTopicCreation(asyncCreationTopics, 5, TimeUnit.SECONDS);
    }
    LOG.info("Core Messaging Service started");
}

Example 58 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class DefaultMetricStore method getMetricsProcessorStats.

/**
 * Read the metrics processing stats from meta table and return the map of topic information to stats
 * @return Map of topic to metrics processing stats
 * @throws Exception
 */
@Override
public Map<String, MetricsProcessorStatus> getMetricsProcessorStats() throws Exception {
    MetricsConsumerMetaTable metaTable = metaTableSupplier.get();
    Map<String, MetricsProcessorStatus> processMap = new HashMap<>();
    for (TopicId topicId : metricsTopics) {
        TopicProcessMeta topicProcessMeta = metaTable.getTopicProcessMeta(new TopicIdMetaKey(topicId));
        if (topicProcessMeta != null) {
            MessageId messageId = new MessageId(topicProcessMeta.getMessageId());
            MetricsMessageId metricsMessageId = new MetricsMessageId(messageId.getPublishTimestamp(), messageId.getSequenceId(), messageId.getPayloadWriteTimestamp(), messageId.getPayloadSequenceId());
            processMap.put(topicId.getTopic(), new MetricsProcessorStatus(metricsMessageId, topicProcessMeta.getOldestMetricsTimestamp(), topicProcessMeta.getLatestMetricsTimestamp(), topicProcessMeta.getMessagesProcessed(), topicProcessMeta.getLastProcessedTimestamp()));
        }
    }
    return processMap;
}

Example 59 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingMetricsProcessorService method run.

@Override
protected void run() {
    LOG.info("Start running MessagingMetricsProcessorService");
    MetricsConsumerMetaTable metaTable = getMetaTable();
    if (metaTable == null) {
        LOG.info("Could not get MetricsConsumerMetaTable, seems like we are being shut down");
        return;
    }
    for (TopicId topic : metricsTopics) {
        TopicProcessMeta topicProcessMeta = null;
        TopicIdMetaKey topicRowKey = new TopicIdMetaKey(topic);
        try {
            topicProcessMeta = metaTable.getTopicProcessMeta(topicRowKey);
        } catch (Exception e) {
            LOG.warn("Cannot retrieve last processed MessageId for topic: {}", topic, e);
        }
        processMetricsThreads.add(new ProcessMetricsThread(topicRowKey, topicProcessMeta));
    }
    if (!isRunning()) {
        return;
    }
    for (ProcessMetricsThread thread : processMetricsThreads) {
        thread.start();
    }
    if (instanceId == 0) {
        if (!skipMigration) {
            List<Integer> resolutions = new ArrayList<>();
            resolutions.add(Integer.MAX_VALUE);
            resolutions.add(3600);
            resolutions.add(60);
            String v2TableNamePrefix = cConfiguration.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_TABLE_PREFIX) + ".ts.";
            String v3TableNamePrefix = cConfiguration.get(Constants.Metrics.METRICS_TABLE_PREFIX, Constants.Metrics.DEFAULT_METRIC_V3_TABLE_PREFIX) + ".ts.";
            int migrationSleepMillis = Integer.valueOf(cConfiguration.get(Constants.Metrics.METRICS_MIGRATION_SLEEP_MILLIS));
            metricsDataMigrator = new DataMigrator(datasetFramework, metricDatasetFactory, resolutions, v2TableNamePrefix, v3TableNamePrefix, migrationSleepMillis);
            metricsDataMigrator.start();
            ScheduledExecutorService metricsTableDeleterExecutor = Executors.newSingleThreadScheduledExecutor(Threads.createDaemonThreadFactory("metrics-table-deleter"));
            DatasetId v2metrics1sResolutionTable = NamespaceId.SYSTEM.dataset(v2TableNamePrefix + 1);
            MetricsTableDeleter tableDeleter = new MetricsTableDeleter(datasetFramework, v2metrics1sResolutionTable);
            // just schedule deletion of 1 second table to run after 2 hours
            metricsTableDeleterExecutor.schedule(tableDeleter, 2, TimeUnit.HOURS);
        } else {
            LOG.info("Skipping Metrics Data Migration");
        }
    }
    for (ProcessMetricsThread thread : processMetricsThreads) {
        try {
            thread.join();
        } catch (InterruptedException e) {
            LOG.info("Thread {} is being terminated while waiting for it to finish.", thread.getName());
            Thread.currentThread().interrupt();
        }
    }
    try {
        // wait upto 5 seconds for the migration to exit cleanly
        if (metricsDataMigrator != null) {
            metricsDataMigrator.join(5000);
        }
    } catch (InterruptedException e) {
        LOG.info("Thread {} is being terminated while waiting for it to finish.", metricsDataMigrator.getName());
        Thread.currentThread().interrupt();
    }
    // Persist metricsFromAllTopics and messageId's after all ProcessMetricsThread's complete.
    // No need to make a copy of metricsFromAllTopics and topicProcessMetaMap because no thread is writing to them
    persistMetricsAndTopicProcessMeta(metricsFromAllTopics, topicProcessMetaMap);
}

Example 60 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class BasicMapReduceTaskContext method getMessagingContext.

@Override
protected MessagingContext getMessagingContext() {
    // Override to have transactional publisher use "store" instead of "payload"
    // since a task is executed with long transaction.
    // The actual publish will be done in the MR driver.
    final MessagingContext context = super.getMessagingContext();
    // TODO: CDAP-7807 Make it available for any topic
    final TopicId allowedTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.Dataset.DATA_EVENT_TOPIC));
    return new MessagingContext() {

        @Override
        public MessagePublisher getMessagePublisher() {
            return new AbstractMessagePublisher() {

                @Override
                protected void publish(TopicId topicId, Iterator<byte[]> payloads) throws IOException, TopicNotFoundException {
                    if (!allowedTopic.equals(topicId)) {
                        throw new UnsupportedOperationException("Publish to topic '" + topicId.getTopic() + "' is not supported");
                    }
                    // Use storePayload
                    getMessagingService().storePayload(StoreRequestBuilder.of(topicId).setTransaction(transaction.getWritePointer()).addPayloads(payloads).build());
                }
            };
        }

        @Override
        public MessagePublisher getDirectMessagePublisher() {
            return context.getDirectMessagePublisher();
        }

        @Override
        public MessageFetcher getMessageFetcher() {
            return context.getMessageFetcher();
        }
    };
}