Examples with LogicalPlanNode io.confluent.ksql.planner.LogicalPlanNode used on opensource projects

Example 1 with LogicalPlanNode

use of io.confluent.ksql.planner.LogicalPlanNode in project ksql by confluentinc.

the class EngineExecutor method executeScalablePushQuery.

ScalablePushQueryMetadata executeScalablePushQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final PushRouting pushRouting, final PushRoutingOptions pushRoutingOptions, final QueryPlannerOptions queryPlannerOptions, final Context context, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics) {
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = pushRoutingOptions.getHasBeenForwarded() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PushPhysicalPlan plan = null;
    try {
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, true);
        final PushPhysicalPlanCreator pushPhysicalPlanCreator = (offsetRange, catchupConsumerGroup) -> buildScalablePushPhysicalPlan(logicalPlan, analysis, context, offsetRange, catchupConsumerGroup);
        final Optional<PushOffsetRange> offsetRange = pushRoutingOptions.getContinuationToken().map(PushOffsetRange::deserialize);
        final Optional<String> catchupConsumerGroup = pushRoutingOptions.getCatchupConsumerGroup();
        final PushPhysicalPlanManager physicalPlanManager = new PushPhysicalPlanManager(pushPhysicalPlanCreator, catchupConsumerGroup, offsetRange);
        final PushPhysicalPlan physicalPlan = physicalPlanManager.getPhysicalPlan();
        plan = physicalPlan;
        final TransientQueryQueue transientQueryQueue = new TransientQueryQueue(analysis.getLimitClause());
        final PushQueryMetadata.ResultType resultType = physicalPlan.getScalablePushRegistry().isTable() ? physicalPlan.getScalablePushRegistry().isWindowed() ? ResultType.WINDOWED_TABLE : ResultType.TABLE : ResultType.STREAM;
        final PushQueryQueuePopulator populator = () -> pushRouting.handlePushQuery(serviceContext, physicalPlanManager, statement, pushRoutingOptions, physicalPlan.getOutputSchema(), transientQueryQueue, scalablePushQueryMetrics, offsetRange);
        final PushQueryPreparer preparer = () -> pushRouting.preparePushQuery(physicalPlanManager, statement, pushRoutingOptions);
        final ScalablePushQueryMetadata metadata = new ScalablePushQueryMetadata(physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), transientQueryQueue, scalablePushQueryMetrics, resultType, populator, preparer, physicalPlan.getSourceType(), routingNodeType, physicalPlan::getRowsReadFromDataSource);
        return metadata;
    } catch (final Exception e) {
        if (plan == null) {
            scalablePushQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PushPhysicalPlan pushPhysicalPlan = plan;
            scalablePushQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, pushPhysicalPlan.getSourceType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute push query V2 {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information." + " If you see this LOG message, please submit a GitHub ticket and" + " we will scrub the statement text from the error at {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute push query V2. {} {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed push query V2 text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 2 with LogicalPlanNode

use of io.confluent.ksql.planner.LogicalPlanNode in project ksql by confluentinc.

the class PullPhysicalPlanBuilder method buildPullPhysicalPlan.

/**
 * Visits the logical plan top-down to build the physical plan.
 * @param logicalPlanNode the logical plan root node
 * @return the root node of the tree of physical operators
 */
// CHECKSTYLE_RULES.OFF: CyclomaticComplexity
public PullPhysicalPlan buildPullPhysicalPlan(final LogicalPlanNode logicalPlanNode) {
    // CHECKSTYLE_RULES.ON: CyclomaticComplexity
    DataSourceOperator dataSourceOperator = null;
    final OutputNode outputNode = logicalPlanNode.getNode().orElseThrow(() -> new IllegalArgumentException("Need an output node to build a plan"));
    if (!(outputNode instanceof KsqlBareOutputNode)) {
        throw new KsqlException("Pull queries expect the root of the logical plan to be a " + "KsqlBareOutputNode.");
    }
    // We skip KsqlBareOutputNode in the translation since it only applies the LIMIT
    PlanNode currentLogicalNode = outputNode.getSource();
    AbstractPhysicalOperator prevPhysicalOp = null;
    AbstractPhysicalOperator rootPhysicalOp = null;
    while (true) {
        AbstractPhysicalOperator currentPhysicalOp = null;
        if (currentLogicalNode instanceof QueryProjectNode) {
            currentPhysicalOp = translateProjectNode((QueryProjectNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof QueryFilterNode) {
            currentPhysicalOp = translateFilterNode((QueryFilterNode) currentLogicalNode);
            seenSelectOperator = true;
        } else if (currentLogicalNode instanceof QueryLimitNode) {
            currentPhysicalOp = new LimitOperator((QueryLimitNode) currentLogicalNode);
        } else if (currentLogicalNode instanceof DataSourceNode) {
            currentPhysicalOp = translateDataSourceNode((DataSourceNode) currentLogicalNode);
            dataSourceOperator = (DataSourceOperator) currentPhysicalOp;
        } else {
            throw new KsqlException(String.format("Error in translating logical to physical plan for pull queries: unrecognized logical" + " node %s.", currentLogicalNode));
        }
        if (prevPhysicalOp == null) {
            rootPhysicalOp = currentPhysicalOp;
        } else {
            prevPhysicalOp.addChild(currentPhysicalOp);
        }
        prevPhysicalOp = currentPhysicalOp;
        // Exit the loop when a leaf node is reached
        if (currentLogicalNode.getSources().isEmpty()) {
            break;
        }
        if (currentLogicalNode.getSources().size() > 1) {
            throw new KsqlException("Pull queries do not support joins or nested sub-queries yet.");
        }
        currentLogicalNode = currentLogicalNode.getSources().get(0);
    }
    if (dataSourceOperator == null) {
        throw new IllegalStateException("DataSourceOperator cannot be null in Pull physical plan");
    }
    return new PullPhysicalPlan(rootPhysicalOp, (rootPhysicalOp).getLogicalNode().getSchema(), queryId, lookupConstraints, pullPhysicalPlanType, querySourceType, mat, dataSourceOperator);
}

Example 3 with LogicalPlanNode

use of io.confluent.ksql.planner.LogicalPlanNode in project ksql by confluentinc.

the class PushPhysicalPlanBuilderTest method shouldThrowOnUnknownLogicalNode.

@Test
public void shouldThrowOnUnknownLogicalNode() {
    // Given:
    when(ksqlBareOutputNode.getSource()).thenReturn(mock(PlanNode.class));
    final PushPhysicalPlanBuilder builder = new PushPhysicalPlanBuilder(logContext, persistentQueryMetadata);
    // When:
    final Exception e = assertThrows(KsqlException.class, () -> builder.buildPushPhysicalPlan(logicalPlanNode, context, Optional.empty(), Optional.empty()));
    // Then:
    assertThat(e.getMessage(), containsString("unrecognized logical node"));
}

Example 4 with LogicalPlanNode

use of io.confluent.ksql.planner.LogicalPlanNode in project ksql by confluentinc.

the class EngineExecutor method planQuery.

private ExecutorPlans planQuery(final ConfiguredStatement<?> statement, final Query query, final Optional<Sink> sink, final Optional<String> withQueryId, final MetaStore metaStore) {
    final QueryEngine queryEngine = engineContext.createQueryEngine(serviceContext);
    final KsqlConfig ksqlConfig = config.getConfig(true);
    final OutputNode outputNode = QueryEngine.buildQueryLogicalPlan(query, sink, metaStore, ksqlConfig, getRowpartitionRowoffsetEnabled(ksqlConfig, statement.getSessionConfig().getOverrides()), statement.getStatementText());
    final LogicalPlanNode logicalPlan = new LogicalPlanNode(statement.getStatementText(), Optional.of(outputNode));
    final QueryId queryId = QueryIdUtil.buildId(statement.getStatement(), engineContext, engineContext.idGenerator(), outputNode, ksqlConfig.getBoolean(KsqlConfig.KSQL_CREATE_OR_REPLACE_ENABLED), withQueryId);
    if (withQueryId.isPresent() && engineContext.getQueryRegistry().getPersistentQuery(queryId).isPresent()) {
        throw new KsqlException(String.format("Query ID '%s' already exists.", queryId));
    }
    final Optional<PersistentQueryMetadata> persistentQueryMetadata = engineContext.getQueryRegistry().getPersistentQuery(queryId);
    final Optional<PlanInfo> oldPlanInfo;
    if (persistentQueryMetadata.isPresent()) {
        final ExecutionStep<?> oldPlan = persistentQueryMetadata.get().getPhysicalPlan();
        oldPlanInfo = Optional.of(oldPlan.extractPlanInfo(new PlanInfoExtractor()));
    } else {
        oldPlanInfo = Optional.empty();
    }
    final PhysicalPlan physicalPlan = queryEngine.buildPhysicalPlan(logicalPlan, config, metaStore, queryId, oldPlanInfo);
    return new ExecutorPlans(logicalPlan, physicalPlan);
}

Example 5 with LogicalPlanNode

use of io.confluent.ksql.planner.LogicalPlanNode in project ksql by confluentinc.

the class EngineExecutor method executeTablePullQuery.

/**
 * Evaluates a pull query by first analyzing it, then building the logical plan and finally
 * the physical plan. The execution is then done using the physical plan in a pipelined manner.
 * @param statement The pull query
 * @param routingOptions Configuration parameters used for HA routing
 * @param pullQueryMetrics JMX metrics
 * @return the rows that are the result of evaluating the pull query
 */
PullQueryResult executeTablePullQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final HARouting routing, final RoutingOptions routingOptions, final QueryPlannerOptions queryPlannerOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final boolean startImmediately, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (!statement.getStatement().isPullQuery()) {
        throw new IllegalArgumentException("Executor can only handle pull queries");
    }
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = routingOptions.getIsSkipForwardRequest() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PullPhysicalPlan plan = null;
    try {
        // Do not set sessionConfig.getConfig to true! The copying is inefficient and slows down pull
        // query performance significantly.  Instead use QueryPlannerOptions which check overrides
        // deliberately.
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, false);
        // This is a cancel signal that is used to stop both local operations and requests
        final CompletableFuture<Void> shouldCancelRequests = new CompletableFuture<>();
        plan = buildPullPhysicalPlan(logicalPlan, analysis, queryPlannerOptions, shouldCancelRequests, consistencyOffsetVector);
        final PullPhysicalPlan physicalPlan = plan;
        final PullQueryQueue pullQueryQueue = new PullQueryQueue(analysis.getLimitClause());
        final PullQueryQueuePopulator populator = () -> routing.handlePullQuery(serviceContext, physicalPlan, statement, routingOptions, physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
        final PullQueryResult result = new PullQueryResult(physicalPlan.getOutputSchema(), populator, physicalPlan.getQueryId(), pullQueryQueue, pullQueryMetrics, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType, physicalPlan::getRowsReadFromDataSource, shouldCancelRequests, consistencyOffsetVector);
        if (startImmediately) {
            result.start();
        }
        return result;
    } catch (final Exception e) {
        if (plan == null) {
            pullQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PullPhysicalPlan physicalPlan = plan;
            pullQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute pull query {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information. If you " + "see this LOG message, please submit a GitHub ticket and we will scrub " + "the statement text from the error at {}", routingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute pull query. {} {}", routingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed pull query text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}