Search in sources :

Example 1 with CreateTableCommand

use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SourceName(io.confluent.ksql.name.SourceName) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) PushPhysicalPlanBuilder(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanBuilder) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Map(java.util.Map) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) QueryId(io.confluent.ksql.query.QueryId) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Sink(io.confluent.ksql.parser.tree.Sink) Set(java.util.Set) Relation(io.confluent.ksql.parser.tree.Relation) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlException(io.confluent.ksql.util.KsqlException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) PushRoutingOptions(io.confluent.ksql.physical.scalablepush.PushRoutingOptions) PlanInfoExtractor(io.confluent.ksql.execution.plan.PlanInfoExtractor) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer) OutputNode(io.confluent.ksql.planner.plan.OutputNode) Throwables(com.google.common.base.Throwables) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) ValueFormat(io.confluent.ksql.serde.ValueFormat) Table(io.confluent.ksql.parser.tree.Table) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) Query(io.confluent.ksql.parser.tree.Query) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) Formats(io.confluent.ksql.execution.plan.Formats) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Context(io.vertx.core.Context) CreateTable(io.confluent.ksql.parser.tree.CreateTable) Locale(java.util.Locale) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) KsqlConfig(io.confluent.ksql.util.KsqlConfig) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) Objects(java.util.Objects) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) QueryPlannerOptions(io.confluent.ksql.planner.QueryPlannerOptions) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Optional(java.util.Optional) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) PullQueryQueuePopulator(io.confluent.ksql.physical.pull.PullQueryQueuePopulator) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) HARouting(io.confluent.ksql.physical.pull.HARouting) PlanInfo(io.confluent.ksql.execution.plan.PlanInfo) PullPhysicalPlanBuilder(io.confluent.ksql.physical.pull.PullPhysicalPlanBuilder) KeyFormat(io.confluent.ksql.serde.KeyFormat) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) CompletableFuture(java.util.concurrent.CompletableFuture) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) OptionalInt(java.util.OptionalInt) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) LogicalPlanner(io.confluent.ksql.planner.LogicalPlanner) Logger(org.slf4j.Logger) PhysicalPlan(io.confluent.ksql.physical.PhysicalPlan) PlanSummary(io.confluent.ksql.util.PlanSummary) PullPhysicalPlan(io.confluent.ksql.physical.pull.PullPhysicalPlan) PlanNode(io.confluent.ksql.planner.plan.PlanNode) QueryRegistry(io.confluent.ksql.query.QueryRegistry) Collections(java.util.Collections) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Select(io.confluent.ksql.parser.tree.Select) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) Table(io.confluent.ksql.parser.tree.Table) CreateTable(io.confluent.ksql.parser.tree.CreateTable) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) Query(io.confluent.ksql.parser.tree.Query) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) CreateTable(io.confluent.ksql.parser.tree.CreateTable) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Formats(io.confluent.ksql.execution.plan.Formats) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Relation(io.confluent.ksql.parser.tree.Relation) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) Select(io.confluent.ksql.parser.tree.Select) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings)

Example 2 with CreateTableCommand

use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.

the class CreateSourceFactory method createTableCommand.

// This method is called by simple CREATE statements
public CreateTableCommand createTableCommand(final CreateTable statement, final KsqlConfig ksqlConfig) {
    final SourceName sourceName = statement.getName();
    final CreateSourceProperties props = statement.getProperties();
    final String topicName = ensureTopicExists(props, serviceContext);
    final LogicalSchema schema = buildSchema(statement.getElements(), ksqlConfig);
    final DataSource dataSource = metaStore.getSource(sourceName);
    if (dataSource != null && !statement.isOrReplace() && !statement.isNotExists()) {
        final String sourceType = dataSource.getDataSourceType().getKsqlType();
        throw new KsqlException(String.format("Cannot add table '%s': A %s with the same name already exists", sourceName.text(), sourceType.toLowerCase()));
    }
    if (schema.key().isEmpty()) {
        final boolean usingSchemaInference = props.getValueSchemaId().isPresent();
        final String additional = usingSchemaInference ? System.lineSeparator() + "Use a partial schema to define the primary key and still load the value columns from " + "the Schema Registry, for example:" + System.lineSeparator() + "\tCREATE TABLE " + sourceName.text() + " (ID INT PRIMARY KEY) WITH (...);" : "";
        throw new KsqlException("Tables require a PRIMARY KEY. Please define the PRIMARY KEY." + additional);
    }
    throwIfCreateOrReplaceOnSourceStreamOrTable(statement, dataSource);
    final Optional<TimestampColumn> timestampColumn = buildTimestampColumn(ksqlConfig, props, schema);
    return new CreateTableCommand(sourceName, schema, timestampColumn, topicName, buildFormats(statement.getName(), schema, props, ksqlConfig), getWindowInfo(props), Optional.of(statement.isOrReplace()), Optional.of(statement.isSource()));
}
Also used : SourceName(io.confluent.ksql.name.SourceName) TimestampColumn(io.confluent.ksql.execution.timestamp.TimestampColumn) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) KsqlException(io.confluent.ksql.util.KsqlException) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties) DataSource(io.confluent.ksql.metastore.model.DataSource)

Example 3 with CreateTableCommand

use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.

the class CreateSourceFactoryTest method shouldNotThrowOnCreateTableIfNotExistsIsSet.

@Test
public void shouldNotThrowOnCreateTableIfNotExistsIsSet() {
    // Given
    final CreateTable ddlStatement = new CreateTable(TABLE_NAME, TableElements.of(tableElement("COL1", new Type(BIGINT), PRIMARY_KEY_CONSTRAINT), tableElement("COL2", new Type(SqlTypes.STRING))), false, true, withProperties, false);
    // When:
    final CreateTableCommand result = createSourceFactory.createTableCommand(ddlStatement, ksqlConfig);
    // Then:
    assertThat(result.getSourceName(), is(TABLE_NAME));
}
Also used : DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) Type(io.confluent.ksql.execution.expression.tree.Type) CreateTable(io.confluent.ksql.parser.tree.CreateTable) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Test(org.junit.Test)

Example 4 with CreateTableCommand

use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.

the class CreateSourceFactoryTest method shouldCreateTableCommandWithSingleValueWrappingFromDefaultConfig.

@Test
public void shouldCreateTableCommandWithSingleValueWrappingFromDefaultConfig() {
    // Given:
    final CreateTable statement = new CreateTable(SOME_NAME, TABLE_ELEMENTS_1_VALUE, false, true, withProperties, false);
    // When:
    final CreateTableCommand cmd = createSourceFactory.createTableCommand(statement, ksqlConfig);
    // Then:
    assertThat(cmd.getFormats().getValueFeatures(), is(SerdeFeatures.of()));
}
Also used : CreateTable(io.confluent.ksql.parser.tree.CreateTable) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Test(org.junit.Test)

Example 5 with CreateTableCommand

use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.

the class CreateSourceFactoryTest method shouldCreateTableCommandFromNodeOutput.

@Test
public void shouldCreateTableCommandFromNodeOutput() {
    // Given:
    final KsqlTopic ksqlTopic = mock(KsqlTopic.class);
    when(ksqlTopic.getKafkaTopicName()).thenReturn(TOPIC_NAME);
    when(ksqlTopic.getKeyFormat()).thenReturn(SOME_KEY_FORMAT);
    when(ksqlTopic.getValueFormat()).thenReturn(SOME_VALUE_FORMAT);
    final KsqlStructuredDataOutputNode outputNode = mock(KsqlStructuredDataOutputNode.class);
    when(outputNode.getSinkName()).thenReturn(Optional.of(SOME_NAME));
    when(outputNode.getSchema()).thenReturn(EXPECTED_SCHEMA);
    when(outputNode.getTimestampColumn()).thenReturn(Optional.of(TIMESTAMP_COLUMN));
    when(outputNode.getKsqlTopic()).thenReturn(ksqlTopic);
    // When:
    final CreateTableCommand result = createSourceFactory.createTableCommand(outputNode);
    // Then:
    assertThat(result.getSourceName(), is(SOME_NAME));
    assertThat(result.getSchema(), is(EXPECTED_SCHEMA));
    assertThat(result.getTimestampColumn(), is(Optional.of(TIMESTAMP_COLUMN)));
    assertThat(result.getTopicName(), is(TOPIC_NAME));
    assertThat(result.getFormats(), is(Formats.from(ksqlTopic)));
    assertThat(result.getWindowInfo(), is(Optional.empty()));
    assertThat(result.isOrReplace(), is(false));
}
Also used : KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Test(org.junit.Test)

Aggregations

CreateTableCommand (io.confluent.ksql.execution.ddl.commands.CreateTableCommand)15 Test (org.junit.Test)13 CreateTable (io.confluent.ksql.parser.tree.CreateTable)8 DataSourceType (io.confluent.ksql.metastore.model.DataSource.DataSourceType)4 Type (io.confluent.ksql.execution.expression.tree.Type)3 KsqlTable (io.confluent.ksql.metastore.model.KsqlTable)3 KsqlException (io.confluent.ksql.util.KsqlException)3 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)2 TimestampColumn (io.confluent.ksql.execution.timestamp.TimestampColumn)2 DataSource (io.confluent.ksql.metastore.model.DataSource)2 SourceName (io.confluent.ksql.name.SourceName)2 KsqlStructuredDataOutputNode (io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode)2 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)2 KsqlConfig (io.confluent.ksql.util.KsqlConfig)2 Throwables (com.google.common.base.Throwables)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)1 ExecuteResult (io.confluent.ksql.KsqlExecutionContext.ExecuteResult)1