Examples with RunningQuery io.confluent.ksql.rest.entity.RunningQuery used on opensource projects

Example 1 with RunningQuery

use of io.confluent.ksql.rest.entity.RunningQuery in project ksql by confluentinc.

the class ListSourceExecutor method describeSource.

private static SourceDescriptionWithWarnings describeSource(final KsqlConfig ksqlConfig, final KsqlExecutionContext ksqlExecutionContext, final ServiceContext serviceContext, final SourceName name, final boolean extended, final ConfiguredStatement<? extends StatementWithExtendedClause> statement, final SessionProperties sessionProperties, final Collection<SourceDescription> remoteSourceDescriptions) {
    final DataSource dataSource = ksqlExecutionContext.getMetaStore().getSource(name);
    if (dataSource == null) {
        throw new KsqlStatementException(String.format("Could not find STREAM/TABLE '%s' in the Metastore", name.text()), statement.getStatementText());
    }
    final List<RunningQuery> readQueries = getQueries(ksqlExecutionContext, q -> q.getSourceNames().contains(dataSource.getName()));
    final List<RunningQuery> writeQueries = getQueries(ksqlExecutionContext, q -> q.getSinkName().equals(Optional.of(dataSource.getName())));
    Optional<TopicDescription> topicDescription = Optional.empty();
    List<QueryOffsetSummary> queryOffsetSummaries = Collections.emptyList();
    List<String> sourceConstraints = Collections.emptyList();
    final List<KsqlWarning> warnings = new LinkedList<>();
    try {
        topicDescription = Optional.of(serviceContext.getTopicClient().describeTopic(dataSource.getKafkaTopicName()));
        sourceConstraints = getSourceConstraints(name, ksqlExecutionContext.getMetaStore());
    } catch (final KafkaException | KafkaResponseGetFailedException e) {
        warnings.add(new KsqlWarning("Error from Kafka: " + e.getMessage()));
    }
    if (extended) {
        queryOffsetSummaries = queryOffsetSummaries(ksqlConfig, serviceContext, writeQueries);
        return new SourceDescriptionWithWarnings(warnings, SourceDescriptionFactory.create(dataSource, extended, readQueries, writeQueries, topicDescription, queryOffsetSummaries, sourceConstraints, remoteSourceDescriptions.stream().flatMap(sd -> sd.getClusterStatistics().stream()), remoteSourceDescriptions.stream().flatMap(sd -> sd.getClusterErrorStats().stream()), sessionProperties.getKsqlHostInfo(), ksqlExecutionContext.metricCollectors()));
    }
    return new SourceDescriptionWithWarnings(warnings, SourceDescriptionFactory.create(dataSource, extended, readQueries, writeQueries, topicDescription, queryOffsetSummaries, sourceConstraints, java.util.stream.Stream.empty(), java.util.stream.Stream.empty(), sessionProperties.getKsqlHostInfo(), ksqlExecutionContext.metricCollectors()));
}

Example 2 with RunningQuery

use of io.confluent.ksql.rest.entity.RunningQuery in project ksql by confluentinc.

the class ListSourceExecutor method queryOffsetSummaries.

private static List<QueryOffsetSummary> queryOffsetSummaries(final KsqlConfig ksqlConfig, final ServiceContext serviceContext, final List<RunningQuery> writeQueries) {
    final Map<String, Map<TopicPartition, OffsetAndMetadata>> offsetsPerQuery = new HashMap<>(writeQueries.size());
    final Map<String, Set<String>> topicsPerQuery = new HashMap<>();
    final Set<String> allTopics = new HashSet<>();
    // Get topics and offsets per running query
    for (RunningQuery query : writeQueries) {
        final QueryId queryId = query.getId();
        final String applicationId = QueryApplicationId.build(ksqlConfig, true, queryId);
        final Map<TopicPartition, OffsetAndMetadata> topicAndConsumerOffsets = serviceContext.getConsumerGroupClient().listConsumerGroupOffsets(applicationId);
        offsetsPerQuery.put(applicationId, topicAndConsumerOffsets);
        final Set<String> topics = topicAndConsumerOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
        topicsPerQuery.put(applicationId, topics);
        allTopics.addAll(topics);
    }
    // Get topics descriptions and start/end offsets
    final Map<String, TopicDescription> sourceTopicDescriptions = serviceContext.getTopicClient().describeTopics(allTopics);
    final Map<TopicPartition, Long> topicAndStartOffsets = serviceContext.getTopicClient().listTopicsStartOffsets(allTopics);
    final Map<TopicPartition, Long> topicAndEndOffsets = serviceContext.getTopicClient().listTopicsEndOffsets(allTopics);
    // Build consumer offsets summary
    final List<QueryOffsetSummary> offsetSummaries = new ArrayList<>();
    for (Entry<String, Set<String>> entry : topicsPerQuery.entrySet()) {
        final List<QueryTopicOffsetSummary> topicSummaries = new ArrayList<>();
        for (String topic : entry.getValue()) {
            topicSummaries.add(new QueryTopicOffsetSummary(topic, consumerPartitionOffsets(sourceTopicDescriptions.get(topic), topicAndStartOffsets, topicAndEndOffsets, offsetsPerQuery.get(entry.getKey()))));
        }
        offsetSummaries.add(new QueryOffsetSummary(entry.getKey(), topicSummaries));
    }
    return offsetSummaries;
}

Example 3 with RunningQuery

use of io.confluent.ksql.rest.entity.RunningQuery in project ksql by confluentinc.

the class Console method printQueries.

private void printQueries(final List<RunningQuery> queries, final String type, final String operation) {
    if (!queries.isEmpty()) {
        writer().println(String.format("%n%-20s%n%-20s", "Queries that " + operation + " from this " + type, "-----------------------------------"));
        for (final RunningQuery writeQuery : queries) {
            writer().println(writeQuery.getId() + " (" + writeQuery.getState().orElse("N/A") + ") : " + writeQuery.getQuerySingleLine());
        }
        writer().println("\nFor query topology and execution plan please run: EXPLAIN <QueryId>");
    }
}

Example 4 with RunningQuery

use of io.confluent.ksql.rest.entity.RunningQuery in project ksql by confluentinc.

the class TestKsqlRestApp method getQueries.

private Set<RunningQuery> getQueries(final KsqlRestClient client, final KsqlQueryType queryType) {
    final RestResponse<KsqlEntityList> response = makeKsqlRequest(client, "SHOW QUERIES;");
    if (response.isErroneous()) {
        throw new AssertionError("Failed to get persistent queries." + " msg:" + response.getErrorMessage());
    }
    final Queries queries = (Queries) response.getResponse().get(0);
    return queries.getQueries().stream().filter(query -> query.getQueryType() == queryType).collect(Collectors.toSet());
}

Example 5 with RunningQuery

use of io.confluent.ksql.rest.entity.RunningQuery in project ksql by confluentinc.

the class ShowQueriesMultiNodeFunctionalTest method getShowQueriesResult.

private static String getShowQueriesResult(final TestKsqlRestApp restApp) {
    final List<KsqlEntity> results = RestIntegrationTestUtil.makeKsqlRequest(restApp, "Show Queries;");
    if (results.size() != 1) {
        return "Expected 1 response, got " + results.size();
    }
    final KsqlEntity result = results.get(0);
    if (!(result instanceof Queries)) {
        return "Expected Queries, got " + result;
    }
    final List<RunningQuery> runningQueries = ((Queries) result).getQueries();
    if (runningQueries.size() != 1) {
        return "Expected 1 running query, got " + runningQueries.size();
    }
    return runningQueries.get(0).getStatusCount().toString();
}