Examples with ScalablePushQueryMetadata io.confluent.ksql.util.ScalablePushQueryMetadata used on opensource projects

Example 1 with ScalablePushQueryMetadata

use of io.confluent.ksql.util.ScalablePushQueryMetadata in project ksql by confluentinc.

the class QueryExecutor method handleScalablePushQuery.

private QueryMetadataHolder handleScalablePushQuery(final ImmutableAnalysis analysis, final ServiceContext serviceContext, final PreparedStatement<Query> statement, final Map<String, Object> configOverrides, final Map<String, Object> requestProperties, final Context context, final SlidingWindowRateLimiter scalablePushBandRateLimiter, final AtomicReference<ScalablePushQueryMetadata> resultForMetrics) {
    final ConfiguredStatement<Query> configured = ConfiguredStatement.of(statement, SessionConfig.of(ksqlConfig, configOverrides));
    final PushQueryConfigRoutingOptions routingOptions = new PushQueryConfigRoutingOptions(ksqlConfig, configOverrides, requestProperties);
    final PushQueryConfigPlannerOptions plannerOptions = new PushQueryConfigPlannerOptions(ksqlConfig, configOverrides);
    scalablePushBandRateLimiter.allow(KsqlQueryType.PUSH);
    final ScalablePushQueryMetadata query = ksqlEngine.executeScalablePushQuery(analysis, serviceContext, configured, pushRouting, routingOptions, plannerOptions, context, scalablePushQueryMetrics);
    query.prepare();
    resultForMetrics.set(query);
    QueryLogger.info("Streaming scalable push query", statement.getStatementText());
    return QueryMetadataHolder.of(query);
}

Example 2 with ScalablePushQueryMetadata

use of io.confluent.ksql.util.ScalablePushQueryMetadata in project ksql by confluentinc.

the class EngineExecutor method executeScalablePushQuery.

ScalablePushQueryMetadata executeScalablePushQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final PushRouting pushRouting, final PushRoutingOptions pushRoutingOptions, final QueryPlannerOptions queryPlannerOptions, final Context context, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics) {
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = pushRoutingOptions.getHasBeenForwarded() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PushPhysicalPlan plan = null;
    try {
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, true);
        final PushPhysicalPlanCreator pushPhysicalPlanCreator = (offsetRange, catchupConsumerGroup) -> buildScalablePushPhysicalPlan(logicalPlan, analysis, context, offsetRange, catchupConsumerGroup);
        final Optional<PushOffsetRange> offsetRange = pushRoutingOptions.getContinuationToken().map(PushOffsetRange::deserialize);
        final Optional<String> catchupConsumerGroup = pushRoutingOptions.getCatchupConsumerGroup();
        final PushPhysicalPlanManager physicalPlanManager = new PushPhysicalPlanManager(pushPhysicalPlanCreator, catchupConsumerGroup, offsetRange);
        final PushPhysicalPlan physicalPlan = physicalPlanManager.getPhysicalPlan();
        plan = physicalPlan;
        final TransientQueryQueue transientQueryQueue = new TransientQueryQueue(analysis.getLimitClause());
        final PushQueryMetadata.ResultType resultType = physicalPlan.getScalablePushRegistry().isTable() ? physicalPlan.getScalablePushRegistry().isWindowed() ? ResultType.WINDOWED_TABLE : ResultType.TABLE : ResultType.STREAM;
        final PushQueryQueuePopulator populator = () -> pushRouting.handlePushQuery(serviceContext, physicalPlanManager, statement, pushRoutingOptions, physicalPlan.getOutputSchema(), transientQueryQueue, scalablePushQueryMetrics, offsetRange);
        final PushQueryPreparer preparer = () -> pushRouting.preparePushQuery(physicalPlanManager, statement, pushRoutingOptions);
        final ScalablePushQueryMetadata metadata = new ScalablePushQueryMetadata(physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), transientQueryQueue, scalablePushQueryMetrics, resultType, populator, preparer, physicalPlan.getSourceType(), routingNodeType, physicalPlan::getRowsReadFromDataSource);
        return metadata;
    } catch (final Exception e) {
        if (plan == null) {
            scalablePushQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PushPhysicalPlan pushPhysicalPlan = plan;
            scalablePushQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, pushPhysicalPlan.getSourceType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute push query V2 {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information." + " If you see this LOG message, please submit a GitHub ticket and" + " we will scrub the statement text from the error at {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute push query V2. {} {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed push query V2 text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 3 with ScalablePushQueryMetadata

use of io.confluent.ksql.util.ScalablePushQueryMetadata in project ksql by confluentinc.

the class QueryMetricsUtil method initializeScalablePushMetricsCallback.

public static MetricsCallback initializeScalablePushMetricsCallback(final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final SlidingWindowRateLimiter scalablePushBandRateLimiter, final AtomicReference<ScalablePushQueryMetadata> resultForMetrics) {
    final MetricsCallback metricsCallback = (statusCode, requestBytes, responseBytes, startTimeNanos) -> scalablePushQueryMetrics.ifPresent(metrics -> {
        metrics.recordStatusCode(statusCode);
        metrics.recordRequestSize(requestBytes);
        final ScalablePushQueryMetadata r = resultForMetrics.get();
        if (r == null) {
            metrics.recordResponseSizeForError(responseBytes);
            metrics.recordConnectionDurationForError(startTimeNanos);
            metrics.recordZeroRowsReturnedForError();
            metrics.recordZeroRowsProcessedForError();
        } else {
            final QuerySourceType sourceType = r.getSourceType();
            final RoutingNodeType routingNodeType = r.getRoutingNodeType();
            metrics.recordResponseSize(responseBytes, sourceType, routingNodeType);
            metrics.recordConnectionDuration(startTimeNanos, sourceType, routingNodeType);
            metrics.recordRowsReturned(r.getTotalRowsReturned(), sourceType, routingNodeType);
            metrics.recordRowsProcessed(r.getTotalRowsProcessed(), sourceType, routingNodeType);
        }
        scalablePushBandRateLimiter.add(responseBytes);
    });
    return metricsCallback;
}