Examples with StreamPullQueryMetadata io.confluent.ksql.util.StreamPullQueryMetadata used on opensource projects

Example 1 with StreamPullQueryMetadata

use of io.confluent.ksql.util.StreamPullQueryMetadata in project ksql by confluentinc.

the class KsqlEngine method createStreamPullQuery.

public StreamPullQueryMetadata createStreamPullQuery(final ServiceContext serviceContext, final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statementOrig, final boolean excludeTombstones) {
    final boolean streamPullQueriesEnabled = statementOrig.getSessionConfig().getConfig(true).getBoolean(KsqlConfig.KSQL_QUERY_STREAM_PULL_QUERY_ENABLED);
    if (!streamPullQueriesEnabled) {
        throw new KsqlStatementException("Pull queries on streams are disabled. To create a push query on the stream," + " add EMIT CHANGES to the end. To enable pull queries on streams, set" + " the " + KsqlConfig.KSQL_QUERY_STREAM_PULL_QUERY_ENABLED + " config to 'true'.", statementOrig.getStatementText());
    }
    // Stream pull query overrides.
    final Map<String, Object> overrides = new HashMap<>(statementOrig.getSessionConfig().getOverrides());
    // Starting from earliest is semantically necessary.
    overrides.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    // Using a single thread keeps these queries as lightweight as possible, since we are
    // not counting them against the transient query limit.
    overrides.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1);
    // There's no point in EOS, since this query only produces side effects.
    overrides.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.AT_LEAST_ONCE);
    final ConfiguredStatement<Query> statement = statementOrig.withConfigOverrides(overrides);
    final ImmutableMap<TopicPartition, Long> endOffsets = getQueryInputEndOffsets(analysis, serviceContext.getAdminClient());
    final TransientQueryMetadata transientQueryMetadata = EngineExecutor.create(primaryContext, serviceContext, statement.getSessionConfig()).executeStreamPullQuery(statement, excludeTombstones, endOffsets);
    QueryLogger.info("Streaming stream pull query results '{}' from earliest to " + endOffsets, statement.getStatementText());
    return new StreamPullQueryMetadata(transientQueryMetadata, endOffsets);
}

Example 2 with StreamPullQueryMetadata

use of io.confluent.ksql.util.StreamPullQueryMetadata in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullStreamMetricsCallback.

public static MetricsCallback initializePullStreamMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final ImmutableAnalysis analysis, final AtomicReference<StreamPullQueryMetadata> resultForMetrics, final AtomicReference<Decrementer> refDecrementer) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final StreamPullQueryMetadata m = resultForMetrics.get();
        final KafkaStreams.State state = m == null ? null : m.getTransientQueryMetadata().getKafkaStreams().state();
        if (m == null || state == null || state.equals(State.ERROR) || state.equals(State.PENDING_ERROR)) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final boolean isWindowed = analysis.getFrom().getDataSource().getKsqlTopic().getKeyFormat().isWindowed();
            final QuerySourceType sourceType = isWindowed ? QuerySourceType.WINDOWED_STREAM : QuerySourceType.NON_WINDOWED_STREAM;
            // There is no WHERE clause constraint information in the persistent logical plan
            final PullPhysicalPlanType planType = PullPhysicalPlanType.UNKNOWN;
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            final TransientQueryQueue rowQueue = (TransientQueryQueue) m.getTransientQueryMetadata().getRowQueue();
            // The rows read from the underlying data source equal the rows read by the user
            // since the WHERE condition is pushed to the data source
            metrics.recordRowsReturned(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(rowQueue.getTotalRowsQueued(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
        // Decrement on happy or exception path
        final Decrementer decrementer = refDecrementer.get();
        if (decrementer != null) {
            decrementer.decrementAtMostOnce();
        }
    });
    return metricsCallback;
}

Example 3 with StreamPullQueryMetadata

use of io.confluent.ksql.util.StreamPullQueryMetadata in project ksql by confluentinc.

the class QueryExecutor method handleStreamPullQuery.

private QueryMetadataHolder handleStreamPullQuery(final ImmutableAnalysis analysis, final ServiceContext serviceContext, final ConfiguredStatement<Query> configured, final AtomicReference<StreamPullQueryMetadata> resultForMetrics, final AtomicReference<Decrementer> refDecrementer) {
    // Apply the same rate, bandwidth and concurrency limits as with table pull queries
    rateLimiter.checkLimit();
    pullBandRateLimiter.allow(KsqlQueryType.PULL);
    refDecrementer.set(concurrencyLimiter.increment());
    final StreamPullQueryMetadata streamPullQueryMetadata = ksqlEngine.createStreamPullQuery(serviceContext, analysis, configured, false);
    resultForMetrics.set(streamPullQueryMetadata);
    localCommands.ifPresent(lc -> lc.write(streamPullQueryMetadata.getTransientQueryMetadata()));
    return QueryMetadataHolder.of(streamPullQueryMetadata);
}