Search in sources :

Example 1 with MutableMetaStore

use of io.confluent.ksql.metastore.MutableMetaStore in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) PushPhysicalPlanCreator(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanCreator) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SourceName(io.confluent.ksql.name.SourceName) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) PushPhysicalPlanManager(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanManager) PushPhysicalPlanBuilder(io.confluent.ksql.physical.scalablepush.PushPhysicalPlanBuilder) RoutingNodeType(io.confluent.ksql.util.KsqlConstants.RoutingNodeType) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Map(java.util.Map) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) QueryId(io.confluent.ksql.query.QueryId) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) ImmutableAnalysis(io.confluent.ksql.analyzer.ImmutableAnalysis) Sink(io.confluent.ksql.parser.tree.Sink) Set(java.util.Set) Relation(io.confluent.ksql.parser.tree.Relation) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlException(io.confluent.ksql.util.KsqlException) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) PullQueryResult(io.confluent.ksql.physical.pull.PullQueryResult) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStructuredDataOutputNode(io.confluent.ksql.planner.plan.KsqlStructuredDataOutputNode) PushRoutingOptions(io.confluent.ksql.physical.scalablepush.PushRoutingOptions) PlanInfoExtractor(io.confluent.ksql.execution.plan.PlanInfoExtractor) DataSourceNode(io.confluent.ksql.planner.plan.DataSourceNode) QueryContainer(io.confluent.ksql.parser.tree.QueryContainer) OutputNode(io.confluent.ksql.planner.plan.OutputNode) Throwables(com.google.common.base.Throwables) PushQueryMetadata(io.confluent.ksql.util.PushQueryMetadata) PushQueryQueuePopulator(io.confluent.ksql.physical.scalablepush.PushQueryQueuePopulator) ValueFormat(io.confluent.ksql.serde.ValueFormat) Table(io.confluent.ksql.parser.tree.Table) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) OutputRefinement(io.confluent.ksql.parser.OutputRefinement) LogicalPlanNode(io.confluent.ksql.planner.LogicalPlanNode) Query(io.confluent.ksql.parser.tree.Query) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) Formats(io.confluent.ksql.execution.plan.Formats) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Context(io.vertx.core.Context) CreateTable(io.confluent.ksql.parser.tree.CreateTable) Locale(java.util.Locale) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ScalablePushQueryMetadata(io.confluent.ksql.util.ScalablePushQueryMetadata) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) KsqlConfig(io.confluent.ksql.util.KsqlConfig) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) DdlCommand(io.confluent.ksql.execution.ddl.commands.DdlCommand) Objects(java.util.Objects) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) QueryPlannerOptions(io.confluent.ksql.planner.QueryPlannerOptions) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Optional(java.util.Optional) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) PullQueryQueuePopulator(io.confluent.ksql.physical.pull.PullQueryQueuePopulator) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) PushPhysicalPlan(io.confluent.ksql.physical.scalablepush.PushPhysicalPlan) HARouting(io.confluent.ksql.physical.pull.HARouting) PlanInfo(io.confluent.ksql.execution.plan.PlanInfo) PullPhysicalPlanBuilder(io.confluent.ksql.physical.pull.PullPhysicalPlanBuilder) KeyFormat(io.confluent.ksql.serde.KeyFormat) ResultType(io.confluent.ksql.util.PushQueryMetadata.ResultType) CompletableFuture(java.util.concurrent.CompletableFuture) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) OptionalInt(java.util.OptionalInt) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) LogicalPlanner(io.confluent.ksql.planner.LogicalPlanner) Logger(org.slf4j.Logger) PhysicalPlan(io.confluent.ksql.physical.PhysicalPlan) PlanSummary(io.confluent.ksql.util.PlanSummary) PullPhysicalPlan(io.confluent.ksql.physical.pull.PullPhysicalPlan) PlanNode(io.confluent.ksql.planner.plan.PlanNode) QueryRegistry(io.confluent.ksql.query.QueryRegistry) Collections(java.util.Collections) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Select(io.confluent.ksql.parser.tree.Select) PushQueryPreparer(io.confluent.ksql.physical.scalablepush.PushQueryPreparer) Table(io.confluent.ksql.parser.tree.Table) CreateTable(io.confluent.ksql.parser.tree.CreateTable) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) Query(io.confluent.ksql.parser.tree.Query) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) CreateTable(io.confluent.ksql.parser.tree.CreateTable) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) Formats(io.confluent.ksql.execution.plan.Formats) UnqualifiedColumnReferenceExp(io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp) CreateTableCommand(io.confluent.ksql.execution.ddl.commands.CreateTableCommand) Relation(io.confluent.ksql.parser.tree.Relation) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlBareOutputNode(io.confluent.ksql.planner.plan.KsqlBareOutputNode) RefinementInfo(io.confluent.ksql.serde.RefinementInfo) CreateTableAsSelect(io.confluent.ksql.parser.tree.CreateTableAsSelect) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) Select(io.confluent.ksql.parser.tree.Select) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) ExecutableDdlStatement(io.confluent.ksql.parser.tree.ExecutableDdlStatement) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) AliasedRelation(io.confluent.ksql.parser.tree.AliasedRelation) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings)

Example 2 with MutableMetaStore

use of io.confluent.ksql.metastore.MutableMetaStore in project ksql by confluentinc.

the class TestExecutor method getKsqlEngine.

static KsqlEngine getKsqlEngine(final ServiceContext serviceContext, final Optional<String> extensionDir) {
    final FunctionRegistry functionRegistry;
    if (extensionDir.isPresent()) {
        final MutableFunctionRegistry mutable = new InternalFunctionRegistry();
        UdfLoaderUtil.load(mutable, extensionDir.get());
        functionRegistry = mutable;
    } else {
        functionRegistry = TestFunctionRegistry.INSTANCE.get();
    }
    final MutableMetaStore metaStore = new MetaStoreImpl(functionRegistry);
    final MetricCollectors metricCollectors = new MetricCollectors();
    return new KsqlEngine(serviceContext, ProcessingLogContext.create(), "test_instance_", metaStore, (engine) -> new KsqlEngineMetrics("", engine, Collections.emptyMap(), Optional.empty(), metricCollectors), new SequentialQueryIdGenerator(), KsqlConfig.empty(), Collections.emptyList(), metricCollectors);
}
Also used : KsqlEngineMetrics(io.confluent.ksql.internal.KsqlEngineMetrics) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) FunctionRegistry(io.confluent.ksql.function.FunctionRegistry) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry)

Example 3 with MutableMetaStore

use of io.confluent.ksql.metastore.MutableMetaStore in project ksql by confluentinc.

the class TestCaseBuilderUtil method getAllTopics.

public static Collection<Topic> getAllTopics(final Collection<String> statements, final Collection<Topic> topics, final Collection<Record> outputs, final Collection<Record> inputs, final FunctionRegistry functionRegistry, final KsqlConfig ksqlConfig) {
    final Map<String, Topic> allTopics = new HashMap<>();
    // Add all topics from topic nodes to the map:
    topics.forEach(topic -> allTopics.put(topic.getName(), topic));
    // Infer topics if not added already:
    final MutableMetaStore metaStore = new MetaStoreImpl(functionRegistry);
    for (String sql : statements) {
        final Topic topicFromStatement = createTopicFromStatement(sql, metaStore, ksqlConfig);
        if (topicFromStatement != null) {
            allTopics.computeIfPresent(topicFromStatement.getName(), (key, topic) -> {
                final Optional<ParsedSchema> keySchema = Optional.of(topic.getKeySchema()).filter(Optional::isPresent).orElse(topicFromStatement.getKeySchema());
                final Optional<ParsedSchema> valueSchema = Optional.of(topic.getValueSchema()).filter(Optional::isPresent).orElse(topicFromStatement.getValueSchema());
                topic = new Topic(topic.getName(), topic.getNumPartitions(), topic.getReplicas(), keySchema, valueSchema, topic.getKeyFeatures(), topic.getValueFeatures());
                return topic;
            });
            if (allTopics.containsKey(topicFromStatement.getName())) {
                // If the topic already exists, just add the key/value serde features
                final Topic existingTopic = allTopics.get(topicFromStatement.getName());
                allTopics.put(topicFromStatement.getName(), new Topic(existingTopic.getName(), existingTopic.getNumPartitions(), existingTopic.getReplicas(), existingTopic.getKeySchema(), existingTopic.getValueSchema(), topicFromStatement.getKeyFeatures(), topicFromStatement.getValueFeatures()));
            } else {
                allTopics.put(topicFromStatement.getName(), topicFromStatement);
            }
        }
    }
    // Get topics from inputs and outputs fields:
    Streams.concat(inputs.stream(), outputs.stream()).map(record -> new Topic(record.getTopicName(), Optional.empty(), Optional.empty())).forEach(topic -> allTopics.putIfAbsent(topic.getName(), topic));
    return allTopics.values();
}
Also used : FormatFactory(io.confluent.ksql.serde.FormatFactory) KsqlParser(io.confluent.ksql.parser.KsqlParser) SerdeFeaturesFactory(io.confluent.ksql.serde.SerdeFeaturesFactory) HashMap(java.util.HashMap) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Function(java.util.function.Function) ArrayList(java.util.ArrayList) SessionConfig(io.confluent.ksql.config.SessionConfig) DefaultKsqlParser(io.confluent.ksql.parser.DefaultKsqlParser) CreateSource(io.confluent.ksql.parser.tree.CreateSource) Map(java.util.Map) MetaStore(io.confluent.ksql.metastore.MetaStore) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) DefaultFormatInjector(io.confluent.ksql.format.DefaultFormatInjector) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) RegisterType(io.confluent.ksql.parser.tree.RegisterType) Path(java.nio.file.Path) Files.getNameWithoutExtension(com.google.common.io.Files.getNameWithoutExtension) FunctionRegistry(io.confluent.ksql.function.FunctionRegistry) Collection(java.util.Collection) SerdeFeature(io.confluent.ksql.serde.SerdeFeature) SchemaTranslator(io.confluent.ksql.serde.SchemaTranslator) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) Streams(com.google.common.collect.Streams) ParsedSchema(io.confluent.kafka.schemaregistry.ParsedSchema) SourcePropertiesUtil(io.confluent.ksql.parser.properties.with.SourcePropertiesUtil) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties) List(java.util.List) Format(io.confluent.ksql.serde.Format) Optional(java.util.Optional) Column(io.confluent.ksql.schema.ksql.Column) PersistenceSchema(io.confluent.ksql.schema.ksql.PersistenceSchema) FormatInfo(io.confluent.ksql.serde.FormatInfo) Collections(java.util.Collections) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) SqlBaseParser(io.confluent.ksql.parser.SqlBaseParser) HashMap(java.util.HashMap) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) ParsedSchema(io.confluent.kafka.schemaregistry.ParsedSchema)

Example 4 with MutableMetaStore

use of io.confluent.ksql.metastore.MutableMetaStore in project ksql by confluentinc.

the class MetaStoreFixture method getNewMetaStore.

public static MutableMetaStore getNewMetaStore(final FunctionRegistry functionRegistry, final ValueFormat valueFormat) {
    final MutableMetaStore metaStore = new MetaStoreImpl(functionRegistry);
    final KeyFormat keyFormat = KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of());
    final LogicalSchema test1Schema = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.array(SqlTypes.DOUBLE)).valueColumn(ColumnName.of("COL5"), SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).headerColumn(ColumnName.of("HEAD"), Optional.empty()).build();
    final KsqlTopic ksqlTopic0 = new KsqlTopic("test0", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream0 = new KsqlStream<>("sqlexpression", SourceName.of("TEST0"), test1Schema, Optional.empty(), false, ksqlTopic0, false);
    metaStore.putSource(ksqlStream0, false);
    final KsqlTopic ksqlTopic1 = new KsqlTopic("test1", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream1 = new KsqlStream<>("sqlexpression", SourceName.of("TEST1"), test1Schema, Optional.empty(), false, ksqlTopic1, false);
    metaStore.putSource(ksqlStream1, false);
    final LogicalSchema test2Schema = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic2 = new KsqlTopic("test2", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable = new KsqlTable<>("sqlexpression", SourceName.of("TEST2"), test2Schema, Optional.empty(), false, ksqlTopic2, false);
    metaStore.putSource(ksqlTable, false);
    final SqlType addressSchema = SqlTypes.struct().field("NUMBER", SqlTypes.BIGINT).field("STREET", SqlTypes.STRING).field("CITY", SqlTypes.STRING).field("STATE", SqlTypes.STRING).field("ZIPCODE", SqlTypes.BIGINT).build();
    final SqlType categorySchema = SqlTypes.struct().field("ID", SqlTypes.BIGINT).field("NAME", SqlTypes.STRING).build();
    final SqlType itemInfoSchema = SqlTypes.struct().field("ITEMID", SqlTypes.BIGINT).field("NAME", SqlTypes.STRING).field("CATEGORY", categorySchema).build();
    final LogicalSchema ordersSchema = LogicalSchema.builder().keyColumn(ColumnName.of("ORDERTIME"), SqlTypes.BIGINT).valueColumn(ColumnName.of("ORDERID"), SqlTypes.BIGINT).valueColumn(ColumnName.of("ITEMID"), SqlTypes.STRING).valueColumn(ColumnName.of("ITEMINFO"), itemInfoSchema).valueColumn(ColumnName.of("ORDERUNITS"), SqlTypes.INTEGER).valueColumn(ColumnName.of("ARRAYCOL"), SqlTypes.array(SqlTypes.DOUBLE)).valueColumn(ColumnName.of("MAPCOL"), SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).valueColumn(ColumnName.of("ADDRESS"), addressSchema).valueColumn(ColumnName.of("TIMESTAMPCOL"), SqlTypes.TIMESTAMP).valueColumn(ColumnName.of("TIMECOL"), SqlTypes.TIME).valueColumn(ColumnName.of("DATECOL"), SqlTypes.DATE).valueColumn(ColumnName.of("BYTESCOL"), SqlTypes.BYTES).build();
    final KsqlTopic ksqlTopicOrders = new KsqlTopic("orders_topic", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStreamOrders = new KsqlStream<>("sqlexpression", SourceName.of("ORDERS"), ordersSchema, Optional.empty(), false, ksqlTopicOrders, false);
    metaStore.putSource(ksqlStreamOrders, false);
    final LogicalSchema testTable3 = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic3 = new KsqlTopic("test3", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable3 = new KsqlTable<>("sqlexpression", SourceName.of("TEST3"), testTable3, Optional.empty(), false, ksqlTopic3, false);
    metaStore.putSource(ksqlTable3, false);
    final SqlType nestedOrdersSchema = SqlTypes.struct().field("ORDERTIME", SqlTypes.BIGINT).field("ORDERID", SqlTypes.BIGINT).field("ITEMID", SqlTypes.STRING).field("ITEMINFO", itemInfoSchema).field("ORDERUNITS", SqlTypes.INTEGER).field("ARRAYCOL", SqlTypes.array(SqlTypes.DOUBLE)).field("MAPCOL", SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).field("ADDRESS", addressSchema).build();
    final LogicalSchema nestedArrayStructMapSchema = LogicalSchema.builder().keyColumn(ColumnName.of("K"), SqlTypes.STRING).valueColumn(ColumnName.of("ARRAYCOL"), SqlTypes.array(itemInfoSchema)).valueColumn(ColumnName.of("MAPCOL"), SqlTypes.map(SqlTypes.STRING, itemInfoSchema)).valueColumn(ColumnName.of("NESTED_ORDER_COL"), nestedOrdersSchema).valueColumn(ColumnName.of("ITEM"), itemInfoSchema).build();
    final KsqlTopic nestedArrayStructMapTopic = new KsqlTopic("NestedArrayStructMap_topic", keyFormat, valueFormat);
    final KsqlStream<?> nestedArrayStructMapOrders = new KsqlStream<>("sqlexpression", SourceName.of("NESTED_STREAM"), nestedArrayStructMapSchema, Optional.empty(), false, nestedArrayStructMapTopic, false);
    metaStore.putSource(nestedArrayStructMapOrders, false);
    final KsqlTopic ksqlTopic4 = new KsqlTopic("test4", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream4 = new KsqlStream<>("sqlexpression4", SourceName.of("TEST4"), test1Schema, Optional.empty(), false, ksqlTopic4, false);
    metaStore.putSource(ksqlStream4, false);
    final LogicalSchema sensorReadingsSchema = LogicalSchema.builder().keyColumn(ColumnName.of("ID"), SqlTypes.BIGINT).valueColumn(ColumnName.of("SENSOR_NAME"), SqlTypes.STRING).valueColumn(ColumnName.of("ARR1"), SqlTypes.array(SqlTypes.BIGINT)).valueColumn(ColumnName.of("ARR2"), SqlTypes.array(SqlTypes.STRING)).build();
    final KsqlTopic ksqlTopicSensorReadings = new KsqlTopic("sensor_readings_topic", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStreamSensorReadings = new KsqlStream<>("sqlexpression", SourceName.of("SENSOR_READINGS"), sensorReadingsSchema, Optional.empty(), false, ksqlTopicSensorReadings, false);
    metaStore.putSource(ksqlStreamSensorReadings, false);
    final LogicalSchema testTable5 = LogicalSchema.builder().keyColumn(ColumnName.of("A"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("B"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("C"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("D"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("E"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("F"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("G"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic5 = new KsqlTopic("test5", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable5 = new KsqlTable<>("sqlexpression", SourceName.of("TEST5"), testTable5, Optional.empty(), false, ksqlTopic5, false);
    metaStore.putSource(ksqlTable5, false);
    return metaStore;
}
Also used : KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) KeyFormat(io.confluent.ksql.serde.KeyFormat) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic)

Example 5 with MutableMetaStore

use of io.confluent.ksql.metastore.MutableMetaStore in project ksql by confluentinc.

the class AnalyzerFunctionalTest method shouldNotInheritNamespaceExplicitlySetUpstreamForAvro.

@Test
public void shouldNotInheritNamespaceExplicitlySetUpstreamForAvro() {
    final String simpleQuery = "create stream s1 as select * from S0;";
    final MutableMetaStore newAvroMetaStore = avroMetaStore.copy();
    final KsqlTopic ksqlTopic = new KsqlTopic("s0", KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of()), ValueFormat.of(FormatInfo.of(FormatFactory.AVRO.name(), ImmutableMap.of(ConnectProperties.FULL_SCHEMA_NAME, "org.ac.s1")), SerdeFeatures.of()));
    final LogicalSchema schema = LogicalSchema.builder().keyColumn(SystemColumns.ROWKEY_NAME, SqlTypes.STRING).valueColumn(ColumnName.of("FIELD1"), SqlTypes.BIGINT).build();
    final KsqlStream<?> ksqlStream = new KsqlStream<>("create stream s0 with(KAFKA_TOPIC='s0', VALUE_AVRO_SCHEMA_FULL_NAME='org.ac.s1', VALUE_FORMAT='avro');", SourceName.of("S0"), schema, Optional.empty(), false, ksqlTopic, false);
    newAvroMetaStore.putSource(ksqlStream, false);
    final List<Statement> statements = parse(simpleQuery, newAvroMetaStore);
    final CreateStreamAsSelect createStreamAsSelect = (CreateStreamAsSelect) statements.get(0);
    final Query query = createStreamAsSelect.getQuery();
    final Analyzer analyzer = new Analyzer(newAvroMetaStore, "", ROWPARTITION_ROWOFFSET_ENABLED, PULL_LIMIT_CLAUSE_ENABLED);
    final Analysis analysis = analyzer.analyze(query, Optional.of(createStreamAsSelect.getSink()));
    assertThat(analysis.getInto(), is(not(Optional.empty())));
    assertThat(analysis.getInto().get().getNewTopic().get().getValueFormat(), is(FormatInfo.of(FormatFactory.AVRO.name())));
}
Also used : KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) Query(io.confluent.ksql.parser.tree.Query) Statement(io.confluent.ksql.parser.tree.Statement) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Matchers.containsString(org.hamcrest.Matchers.containsString) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Test(org.junit.Test)

Aggregations

MutableMetaStore (io.confluent.ksql.metastore.MutableMetaStore)5 MetaStoreImpl (io.confluent.ksql.metastore.MetaStoreImpl)4 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)3 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)3 SessionConfig (io.confluent.ksql.config.SessionConfig)2 FunctionRegistry (io.confluent.ksql.function.FunctionRegistry)2 InternalFunctionRegistry (io.confluent.ksql.function.InternalFunctionRegistry)2 MetaStore (io.confluent.ksql.metastore.MetaStore)2 KsqlStream (io.confluent.ksql.metastore.model.KsqlStream)2 KsqlTable (io.confluent.ksql.metastore.model.KsqlTable)2 PreparedStatement (io.confluent.ksql.parser.KsqlParser.PreparedStatement)2 CreateStreamAsSelect (io.confluent.ksql.parser.tree.CreateStreamAsSelect)2 Throwables (com.google.common.base.Throwables)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 Streams (com.google.common.collect.Streams)1 Files.getNameWithoutExtension (com.google.common.io.Files.getNameWithoutExtension)1 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)1 ParsedSchema (io.confluent.kafka.schemaregistry.ParsedSchema)1