Search in sources :

Example 1 with ExecutableDdlStatement

use of io.confluent.ksql.parser.tree.ExecutableDdlStatement in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SourceName(io.confluent.ksql.name.SourceName) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) PushPhysicalPlanBuilder(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanBuilder) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Map(java.util.Map) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) QueryId(io.confluent.ksql.query.QueryId) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Sink(io.confluent.ksql.parser.tree.Sink) Set(java.util.Set) Relation(io.confluent.ksql.parser.tree.Relation) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlException(io.confluent.ksql.util.KsqlException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) PushRoutingOptions(io.confluent.ksql.physical.scalablepush.PushRoutingOptions) PlanInfoExtractor(io.confluent.ksql.execution.plan.PlanInfoExtractor) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer) OutputNode(io.confluent.ksql.planner.plan.OutputNode) Throwables(com.google.common.base.Throwables) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) ValueFormat(io.confluent.ksql.serde.ValueFormat) Table(io.confluent.ksql.parser.tree.Table) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) Query(io.confluent.ksql.parser.tree.Query) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) Formats(io.confluent.ksql.execution.plan.Formats) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Context(io.vertx.core.Context) CreateTable(io.confluent.ksql.parser.tree.CreateTable) Locale(java.util.Locale) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) KsqlConfig(io.confluent.ksql.util.KsqlConfig) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) Objects(java.util.Objects) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) QueryPlannerOptions(io.confluent.ksql.planner.QueryPlannerOptions) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Optional(java.util.Optional) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) PullQueryQueuePopulator(io.confluent.ksql.physical.pull.PullQueryQueuePopulator) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) HARouting(io.confluent.ksql.physical.pull.HARouting) PlanInfo(io.confluent.ksql.execution.plan.PlanInfo) PullPhysicalPlanBuilder(io.confluent.ksql.physical.pull.PullPhysicalPlanBuilder) KeyFormat(io.confluent.ksql.serde.KeyFormat) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) CompletableFuture(java.util.concurrent.CompletableFuture) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) OptionalInt(java.util.OptionalInt) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) LogicalPlanner(io.confluent.ksql.planner.LogicalPlanner) Logger(org.slf4j.Logger) PhysicalPlan(io.confluent.ksql.physical.PhysicalPlan) PlanSummary(io.confluent.ksql.util.PlanSummary) PullPhysicalPlan(io.confluent.ksql.physical.pull.PullPhysicalPlan) PlanNode(io.confluent.ksql.planner.plan.PlanNode) QueryRegistry(io.confluent.ksql.query.QueryRegistry) Collections(java.util.Collections) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Select(io.confluent.ksql.parser.tree.Select) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) Table(io.confluent.ksql.parser.tree.Table) CreateTable(io.confluent.ksql.parser.tree.CreateTable) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) Query(io.confluent.ksql.parser.tree.Query) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) CreateTable(io.confluent.ksql.parser.tree.CreateTable) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Formats(io.confluent.ksql.execution.plan.Formats) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Relation(io.confluent.ksql.parser.tree.Relation) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) Select(io.confluent.ksql.parser.tree.Select) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings)

Example 2 with ExecutableDdlStatement

use of io.confluent.ksql.parser.tree.ExecutableDdlStatement in project ksql by confluentinc.

the class EngineExecutor method plan.

// Known to be non-empty
@SuppressWarnings("OptionalGetWithoutIsPresent")
KsqlPlan plan(final ConfiguredStatement<?> statement) {
    try {
        throwOnNonExecutableStatement(statement);
        if (statement.getStatement() instanceof ExecutableDdlStatement) {
            final boolean isSourceStream = statement.getStatement() instanceof CreateStream && ((CreateStream) statement.getStatement()).isSource();
            final boolean isSourceTable = statement.getStatement() instanceof CreateTable && ((CreateTable) statement.getStatement()).isSource();
            if ((isSourceStream || isSourceTable) && !isSourceTableMaterializationEnabled()) {
                throw new KsqlStatementException("Cannot execute command because source table " + "materialization is disabled.", statement.getStatementText());
            }
            if (isSourceTable) {
                return sourceTablePlan(statement);
            } else {
                final DdlCommand ddlCommand = engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
                return KsqlPlan.ddlPlanCurrent(statement.getStatementText(), ddlCommand);
            }
        }
        final QueryContainer queryContainer = (QueryContainer) statement.getStatement();
        final ExecutorPlans plans = planQuery(statement, queryContainer.getQuery(), Optional.of(queryContainer.getSink()), queryContainer.getQueryId(), engineContext.getMetaStore());
        final KsqlStructuredDataOutputNode outputNode = (KsqlStructuredDataOutputNode) plans.logicalPlan.getNode().get();
        final Optional<DdlCommand> ddlCommand = maybeCreateSinkDdl(statement, outputNode);
        validateResultType(outputNode.getNodeOutputType(), statement);
        final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), outputNode.getSinkName(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
        engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
        return KsqlPlan.queryPlanCurrent(statement.getStatementText(), ddlCommand, queryPlan);
    } catch (final KsqlStatementException e) {
        throw e;
    } catch (final Exception e) {
        throw new KsqlStatementException(e.getMessage(), statement.getStatementText(), e);
    }
}
Also used : DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) CreateTable(io.confluent.ksql.parser.tree.CreateTable) CreateStream(io.confluent.ksql.parser.tree.CreateStream) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) KsqlException(io.confluent.ksql.util.KsqlException) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer)

Example 3 with ExecutableDdlStatement

use of io.confluent.ksql.parser.tree.ExecutableDdlStatement in project ksql by confluentinc.

the class CommandFactoriesTest method shouldThrowOnUnsupportedStatementType.

@Test(expected = KsqlException.class)
public void shouldThrowOnUnsupportedStatementType() {
    // Given:
    final ExecutableDdlStatement ddlStatement = new ExecutableDdlStatement() {
    };
    // Then:
    commandFactories.create(sqlExpression, ddlStatement, SessionConfig.of(ksqlConfig, emptyMap()));
}
Also used : ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) Test(org.junit.Test)

Example 4 with ExecutableDdlStatement

use of io.confluent.ksql.parser.tree.ExecutableDdlStatement in project ksql by confluentinc.

the class InteractiveStatementExecutor method executeStatement.

@SuppressWarnings("unchecked")
private void executeStatement(final PreparedStatement<?> statement, final CommandId commandId, final Optional<CommandStatusFuture> commandStatusFuture) {
    if (statement.getStatement() instanceof TerminateQuery) {
        terminateQuery((PreparedStatement<TerminateQuery>) statement);
        final String successMessage = "Query terminated.";
        final CommandStatus successStatus = new CommandStatus(CommandStatus.Status.SUCCESS, successMessage, Optional.empty());
        putFinalStatus(commandId, commandStatusFuture, successStatus);
    } else if (statement.getStatement() instanceof ExecutableDdlStatement) {
        throwUnsupportedStatementError();
    } else if (statement.getStatement() instanceof CreateAsSelect) {
        throwUnsupportedStatementError();
    } else if (statement.getStatement() instanceof InsertInto) {
        throwUnsupportedStatementError();
    } else if (statement.getStatement() instanceof AlterSystemProperty) {
        final PreparedStatement<AlterSystemProperty> alterSystemQuery = (PreparedStatement<AlterSystemProperty>) statement;
        final String propertyName = alterSystemQuery.getStatement().getPropertyName();
        final String propertyValue = alterSystemQuery.getStatement().getPropertyValue();
        ksqlEngine.alterSystemProperty(propertyName, propertyValue);
        ksqlEngine.updateStreamsPropertiesAndRestartRuntime();
        final String successMessage = String.format("System property %s was set to %s.", propertyName, propertyValue);
        final CommandStatus successStatus = new CommandStatus(CommandStatus.Status.SUCCESS, successMessage, Optional.empty());
        putFinalStatus(commandId, commandStatusFuture, successStatus);
    } else {
        throw new KsqlException(String.format("Unexpected statement type: %s", statement.getClass().getName()));
    }
}
Also used : TerminateQuery(io.confluent.ksql.parser.tree.TerminateQuery) AlterSystemProperty(io.confluent.ksql.parser.tree.AlterSystemProperty) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) InsertInto(io.confluent.ksql.parser.tree.InsertInto) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlException(io.confluent.ksql.util.KsqlException)

Aggregations

ExecutableDdlStatement (io.confluent.ksql.parser.tree.ExecutableDdlStatement)4 KsqlException (io.confluent.ksql.util.KsqlException)3 DdlCommand (io.confluent.ksql.execution.ddl.commands.DdlCommand)2 CreateAsSelect (io.confluent.ksql.parser.tree.CreateAsSelect)2 CreateStream (io.confluent.ksql.parser.tree.CreateStream)2 CreateTable (io.confluent.ksql.parser.tree.CreateTable)2 QueryContainer (io.confluent.ksql.parser.tree.QueryContainer)2 KsqlStructuredDataOutputNode (io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode)2 KsqlStatementException (io.confluent.ksql.util.KsqlStatementException)2 Throwables (com.google.common.base.Throwables)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)1 ExecuteResult (io.confluent.ksql.KsqlExecutionContext.ExecuteResult)1 ImmutableAnalysis (io.confluent.ksql.analyzer.ImmutableAnalysis)1 SessionConfig (io.confluent.ksql.config.SessionConfig)1 CreateTableCommand (io.confluent.ksql.execution.ddl.commands.CreateTableCommand)1 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)1 UnqualifiedColumnReferenceExp (io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp)1