use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.
the class EngineExecutor method sourceTablePlan.
@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
final CreateTable createTable = (CreateTable) statement.getStatement();
final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
// Only VALUE or HEADER columns must be selected from the source table. When running a
// pull query, the keys are added if selecting all columns.
final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
// Source table need to keep emitting changes so every new record is materialized for
// pull query availability.
final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
// This is a plan for a `select * from <source-table> emit changes` statement,
// without a sink topic to write the results. The query is just made to materialize the
// source table.
final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
// The source table does not exist in the current metastore, so a temporary metastore that
// contains only the source table is created here. This metastore is used later to create
// ExecutorsPlan.
final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
final Formats formats = ddlCommand.getFormats();
tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}
use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.
the class CreateSourceFactory method createTableCommand.
// This method is called by simple CREATE statements
public CreateTableCommand createTableCommand(final CreateTable statement, final KsqlConfig ksqlConfig) {
final SourceName sourceName = statement.getName();
final CreateSourceProperties props = statement.getProperties();
final String topicName = ensureTopicExists(props, serviceContext);
final LogicalSchema schema = buildSchema(statement.getElements(), ksqlConfig);
final DataSource dataSource = metaStore.getSource(sourceName);
if (dataSource != null && !statement.isOrReplace() && !statement.isNotExists()) {
final String sourceType = dataSource.getDataSourceType().getKsqlType();
throw new KsqlException(String.format("Cannot add table '%s': A %s with the same name already exists", sourceName.text(), sourceType.toLowerCase()));
}
if (schema.key().isEmpty()) {
final boolean usingSchemaInference = props.getValueSchemaId().isPresent();
final String additional = usingSchemaInference ? System.lineSeparator() + "Use a partial schema to define the primary key and still load the value columns from " + "the Schema Registry, for example:" + System.lineSeparator() + "\tCREATE TABLE " + sourceName.text() + " (ID INT PRIMARY KEY) WITH (...);" : "";
throw new KsqlException("Tables require a PRIMARY KEY. Please define the PRIMARY KEY." + additional);
}
throwIfCreateOrReplaceOnSourceStreamOrTable(statement, dataSource);
final Optional<TimestampColumn> timestampColumn = buildTimestampColumn(ksqlConfig, props, schema);
return new CreateTableCommand(sourceName, schema, timestampColumn, topicName, buildFormats(statement.getName(), schema, props, ksqlConfig), getWindowInfo(props), Optional.of(statement.isOrReplace()), Optional.of(statement.isSource()));
}
use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.
the class CreateSourceFactoryTest method shouldNotThrowOnCreateTableIfNotExistsIsSet.
@Test
public void shouldNotThrowOnCreateTableIfNotExistsIsSet() {
// Given
final CreateTable ddlStatement = new CreateTable(TABLE_NAME, TableElements.of(tableElement("COL1", new Type(BIGINT), PRIMARY_KEY_CONSTRAINT), tableElement("COL2", new Type(SqlTypes.STRING))), false, true, withProperties, false);
// When:
final CreateTableCommand result = createSourceFactory.createTableCommand(ddlStatement, ksqlConfig);
// Then:
assertThat(result.getSourceName(), is(TABLE_NAME));
}
use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.
the class CreateSourceFactoryTest method shouldCreateTableCommandWithSingleValueWrappingFromDefaultConfig.
@Test
public void shouldCreateTableCommandWithSingleValueWrappingFromDefaultConfig() {
// Given:
final CreateTable statement = new CreateTable(SOME_NAME, TABLE_ELEMENTS_1_VALUE, false, true, withProperties, false);
// When:
final CreateTableCommand cmd = createSourceFactory.createTableCommand(statement, ksqlConfig);
// Then:
assertThat(cmd.getFormats().getValueFeatures(), is(SerdeFeatures.of()));
}
use of io.confluent.ksql.execution.ddl.commands.CreateTableCommand in project ksql by confluentinc.
the class CreateSourceFactoryTest method shouldCreateTableCommandFromNodeOutput.
@Test
public void shouldCreateTableCommandFromNodeOutput() {
// Given:
final KsqlTopic ksqlTopic = mock(KsqlTopic.class);
when(ksqlTopic.getKafkaTopicName()).thenReturn(TOPIC_NAME);
when(ksqlTopic.getKeyFormat()).thenReturn(SOME_KEY_FORMAT);
when(ksqlTopic.getValueFormat()).thenReturn(SOME_VALUE_FORMAT);
final KsqlStructuredDataOutputNode outputNode = mock(KsqlStructuredDataOutputNode.class);
when(outputNode.getSinkName()).thenReturn(Optional.of(SOME_NAME));
when(outputNode.getSchema()).thenReturn(EXPECTED_SCHEMA);
when(outputNode.getTimestampColumn()).thenReturn(Optional.of(TIMESTAMP_COLUMN));
when(outputNode.getKsqlTopic()).thenReturn(ksqlTopic);
// When:
final CreateTableCommand result = createSourceFactory.createTableCommand(outputNode);
// Then:
assertThat(result.getSourceName(), is(SOME_NAME));
assertThat(result.getSchema(), is(EXPECTED_SCHEMA));
assertThat(result.getTimestampColumn(), is(Optional.of(TIMESTAMP_COLUMN)));
assertThat(result.getTopicName(), is(TOPIC_NAME));
assertThat(result.getFormats(), is(Formats.from(ksqlTopic)));
assertThat(result.getWindowInfo(), is(Optional.empty()));
assertThat(result.isOrReplace(), is(false));
}
Aggregations