Examples with PullQueryResult io.confluent.ksql.physical.pull.PullQueryResult used on opensource projects

Example 1 with PullQueryResult

use of io.confluent.ksql.physical.pull.PullQueryResult in project ksql by confluentinc.

the class StreamedQueryResource method handleQuery.

private EndpointResponse handleQuery(final PreparedStatement<Query> statement, final CompletableFuture<Void> connectionClosedFuture, final QueryMetadataHolder queryMetadataHolder) {
    if (queryMetadataHolder.getPullQueryResult().isPresent()) {
        final PullQueryResult result = queryMetadataHolder.getPullQueryResult().get();
        final PullQueryStreamWriter pullQueryStreamWriter = new PullQueryStreamWriter(result, disconnectCheckInterval.toMillis(), OBJECT_MAPPER, result.getPullQueryQueue(), Clock.systemUTC(), connectionClosedFuture, statement);
        return EndpointResponse.ok(pullQueryStreamWriter);
    } else if (queryMetadataHolder.getPushQueryMetadata().isPresent()) {
        final PushQueryMetadata query = queryMetadataHolder.getPushQueryMetadata().get();
        final boolean emptyStream = queryMetadataHolder.getStreamPullQueryMetadata().map(streamPullQueryMetadata -> streamPullQueryMetadata.getEndOffsets().isEmpty()).orElse(false);
        final QueryStreamWriter queryStreamWriter = new QueryStreamWriter(query, disconnectCheckInterval.toMillis(), OBJECT_MAPPER, connectionClosedFuture, emptyStream);
        return EndpointResponse.ok(queryStreamWriter);
    } else {
        return Errors.badRequest(String.format("Statement type `%s' not supported for this resource", statement.getClass().getName()));
    }
}

Example 2 with PullQueryResult

use of io.confluent.ksql.physical.pull.PullQueryResult in project ksql by confluentinc.

the class QueryMetricsUtil method initializePullTableMetricsCallback.

public static MetricsCallback initializePullTableMetricsCallback(final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final SlidingWindowRateLimiter pullBandRateLimiter, final AtomicReference<PullQueryResult> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> pullQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final PullQueryResult r = resultForMetrics.get();
        if (r == null) {
            recordErrorMetrics(pullQueryMetrics, responseBytes, startTimeNanos);
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final PullPhysicalPlanType planType = r.getPlanType();
            final RoutingNodeType routingNodeType = RoutingNodeType.SOURCE_NODE;
            metrics.recordResponseSize(responseBytes, sourceType, planType, routingNodeType);
            metrics.recordLatency(startTimeNanos, sourceType, planType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, planType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, planType, routingNodeType);
        }
        pullBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}

Example 3 with PullQueryResult

use of io.confluent.ksql.physical.pull.PullQueryResult in project ksql by confluentinc.

the class EngineExecutor method executeTablePullQuery.

/**
 * Evaluates a pull query by first analyzing it, then building the logical plan and finally
 * the physical plan. The execution is then done using the physical plan in a pipelined manner.
 * @param statement The pull query
 * @param routingOptions Configuration parameters used for HA routing
 * @param pullQueryMetrics JMX metrics
 * @return the rows that are the result of evaluating the pull query
 */
PullQueryResult executeTablePullQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final HARouting routing, final RoutingOptions routingOptions, final QueryPlannerOptions queryPlannerOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final boolean startImmediately, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (!statement.getStatement().isPullQuery()) {
        throw new IllegalArgumentException("Executor can only handle pull queries");
    }
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = routingOptions.getIsSkipForwardRequest() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PullPhysicalPlan plan = null;
    try {
        // Do not set sessionConfig.getConfig to true! The copying is inefficient and slows down pull
        // query performance significantly.  Instead use QueryPlannerOptions which check overrides
        // deliberately.
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, false);
        // This is a cancel signal that is used to stop both local operations and requests
        final CompletableFuture<Void> shouldCancelRequests = new CompletableFuture<>();
        plan = buildPullPhysicalPlan(logicalPlan, analysis, queryPlannerOptions, shouldCancelRequests, consistencyOffsetVector);
        final PullPhysicalPlan physicalPlan = plan;
        final PullQueryQueue pullQueryQueue = new PullQueryQueue(analysis.getLimitClause());
        final PullQueryQueuePopulator populator = () -> routing.handlePullQuery(serviceContext, physicalPlan, statement, routingOptions, physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
        final PullQueryResult result = new PullQueryResult(physicalPlan.getOutputSchema(), populator, physicalPlan.getQueryId(), pullQueryQueue, pullQueryMetrics, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType, physicalPlan::getRowsReadFromDataSource, shouldCancelRequests, consistencyOffsetVector);
        if (startImmediately) {
            result.start();
        }
        return result;
    } catch (final Exception e) {
        if (plan == null) {
            pullQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PullPhysicalPlan physicalPlan = plan;
            pullQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute pull query {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information. If you " + "see this LOG message, please submit a GitHub ticket and we will scrub " + "the statement text from the error at {}", routingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute pull query. {} {}", routingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed pull query text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 4 with PullQueryResult

use of io.confluent.ksql.physical.pull.PullQueryResult in project ksql by confluentinc.

the class QueryEndpoint method createQueryPublisher.

public QueryPublisher createQueryPublisher(final String sql, final Map<String, Object> properties, final Map<String, Object> sessionVariables, final Map<String, Object> requestProperties, final Context context, final WorkerExecutor workerExecutor, final ServiceContext serviceContext, final MetricsCallbackHolder metricsCallbackHolder, final Optional<Boolean> isInternalRequest) {
    // Must be run on worker as all this stuff is slow
    VertxUtils.checkIsWorker();
    final ConfiguredStatement<Query> statement = createStatement(sql, properties, sessionVariables);
    final QueryMetadataHolder queryMetadataHolder = queryExecutor.handleStatement(serviceContext, properties, requestProperties, statement.getPreparedStatement(), isInternalRequest, metricsCallbackHolder, context, false);
    if (queryMetadataHolder.getPullQueryResult().isPresent()) {
        final PullQueryResult result = queryMetadataHolder.getPullQueryResult().get();
        final BlockingQueryPublisher publisher = new BlockingQueryPublisher(context, workerExecutor);
        publisher.setQueryHandle(new KsqlPullQueryHandle(result, pullQueryMetrics, statement.getPreparedStatement().getStatementText()), true, false);
        // Start from the worker thread so that errors can bubble up, and we can get a proper response
        // code rather than waiting until later after the header has been written and all we can do
        // is write an error message.
        publisher.startFromWorkerThread();
        return publisher;
    } else if (queryMetadataHolder.getPushQueryMetadata().isPresent()) {
        final PushQueryMetadata metadata = queryMetadataHolder.getPushQueryMetadata().get();
        final BlockingQueryPublisher publisher = new BlockingQueryPublisher(context, workerExecutor);
        publisher.setQueryHandle(new KsqlQueryHandle(metadata), false, queryMetadataHolder.getScalablePushQueryMetadata().isPresent());
        return publisher;
    } else {
        throw new KsqlStatementException("Unexpected metadata for query", statement.getStatementText());
    }
}

Example 5 with PullQueryResult

use of io.confluent.ksql.physical.pull.PullQueryResult in project ksql by confluentinc.

the class QueryExecutor method handleTablePullQuery.

private QueryMetadataHolder handleTablePullQuery(final ImmutableAnalysis analysis, final ServiceContext serviceContext, final ConfiguredStatement<Query> configured, final Map<String, Object> requestProperties, final Optional<Boolean> isInternalRequest, final SlidingWindowRateLimiter pullBandRateLimiter, final AtomicReference<PullQueryResult> resultForMetrics, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final RoutingOptions routingOptions = new PullQueryConfigRoutingOptions(configured.getSessionConfig().getConfig(false), configured.getSessionConfig().getOverrides(), requestProperties);
    final PullQueryConfigPlannerOptions plannerOptions = new PullQueryConfigPlannerOptions(configured.getSessionConfig().getConfig(false), configured.getSessionConfig().getOverrides());
    // A request is considered forwarded if the request has the forwarded flag or if the request
    // is from an internal listener.
    final boolean isAlreadyForwarded = routingOptions.getIsSkipForwardRequest() && // Trust the forward request option if isInternalRequest isn't available.
    isInternalRequest.orElse(true);
    // Only check the rate limit at the forwarding host
    Decrementer decrementer = null;
    try {
        if (!isAlreadyForwarded) {
            rateLimiter.checkLimit();
            decrementer = concurrencyLimiter.increment();
        }
        pullBandRateLimiter.allow(KsqlQueryType.PULL);
        final Optional<Decrementer> optionalDecrementer = Optional.ofNullable(decrementer);
        final PullQueryResult result = ksqlEngine.executeTablePullQuery(analysis, serviceContext, configured, routing, routingOptions, plannerOptions, pullQueryMetrics, false, consistencyOffsetVector);
        resultForMetrics.set(result);
        result.onCompletionOrException((v, t) -> optionalDecrementer.ifPresent(Decrementer::decrementAtMostOnce));
        return QueryMetadataHolder.of(result);
    } catch (final Throwable t) {
        if (decrementer != null) {
            decrementer.decrementAtMostOnce();
        }
        throw t;
    }
}