Search in sources :

Example 11 with ExecuteResult

use of io.confluent.ksql.KsqlExecutionContext.ExecuteResult in project ksql by confluentinc.

the class TestExecutorUtil method executePlan.

private static ExecuteResultAndSources executePlan(final KsqlExecutionContext executionContext, final ConfiguredKsqlPlan plan) {
    final ExecuteResult executeResult = executionContext.execute(executionContext.getServiceContext(), plan);
    final Optional<List<DataSource>> dataSources = plan.getPlan().getQueryPlan().map(queryPlan -> getSources(queryPlan.getSources(), executionContext.getMetaStore()));
    return new ExecuteResultAndSources(executeResult, dataSources);
}
Also used : List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult)

Example 12 with ExecuteResult

use of io.confluent.ksql.KsqlExecutionContext.ExecuteResult in project ksql by confluentinc.

the class KsqlTesterTest method execute.

@SuppressWarnings("unchecked")
private void execute(final ParsedStatement parsedStatement) {
    final PreparedStatement<?> engineStatement = engine.prepare(parsedStatement);
    final ConfiguredStatement<?> configured = ConfiguredStatement.of(engineStatement, SessionConfig.of(config, overrides));
    createTopics(engineStatement);
    if (engineStatement.getStatement() instanceof InsertValues) {
        pipeInput((ConfiguredStatement<InsertValues>) configured);
        return;
    } else if (engineStatement.getStatement() instanceof SetProperty) {
        PropertyOverrider.set((ConfiguredStatement<SetProperty>) configured, overrides);
        return;
    } else if (engineStatement.getStatement() instanceof UnsetProperty) {
        PropertyOverrider.unset((ConfiguredStatement<UnsetProperty>) configured, overrides);
        return;
    }
    final ConfiguredStatement<?> injected = formatInjector.inject(configured);
    final ExecuteResult result = engine.execute(serviceContext, injected);
    // is DDL statement
    if (!result.getQuery().isPresent()) {
        return;
    }
    final PersistentQueryMetadata query = (PersistentQueryMetadata) result.getQuery().get();
    final Topology topology = query.getTopology();
    final Properties properties = new Properties();
    properties.putAll(query.getStreamsProperties());
    properties.put(StreamsConfig.STATE_DIR_CONFIG, tmpFolder.getRoot().getAbsolutePath());
    final TopologyTestDriver driver = new TopologyTestDriver(topology, properties);
    final List<TopicInfo> inputTopics = query.getSourceNames().stream().map(sn -> engine.getMetaStore().getSource(sn)).map(ds -> new TopicInfo(ds.getKafkaTopicName(), keySerde(ds), valueSerde(ds))).collect(Collectors.toList());
    // Sink may be Optional for source tables. Once source table query execution is supported, then
    // we would need have a condition to not create an output topic info
    final DataSource output = engine.getMetaStore().getSource(query.getSinkName().get());
    final TopicInfo outputInfo = new TopicInfo(output.getKafkaTopicName(), keySerde(output), valueSerde(output));
    driverPipeline.addDriver(driver, inputTopics, outputInfo);
    drivers.put(query.getQueryId(), new DriverAndProperties(driver, properties));
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) NoSuchFileException(java.nio.file.NoSuchFileException) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) InsertValues(io.confluent.ksql.parser.tree.InsertValues) Serde(org.apache.kafka.common.serialization.Serde) Map(java.util.Map) After(org.junit.After) DefaultFormatInjector(io.confluent.ksql.format.DefaultFormatInjector) QueryId(io.confluent.ksql.query.QueryId) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) Path(java.nio.file.Path) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) Parameterized(org.junit.runners.Parameterized) QueryMetadata(io.confluent.ksql.util.QueryMetadata) MockSchemaRegistryClient(io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient) ImmutableMap(com.google.common.collect.ImmutableMap) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TestDirective(io.confluent.ksql.test.parser.TestDirective) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) Collectors(java.util.stream.Collectors) TestServiceContext(io.confluent.ksql.services.TestServiceContext) Objects(java.util.Objects) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) List(java.util.List) SetProperty(io.confluent.ksql.parser.tree.SetProperty) NoopProcessingLogContext(io.confluent.ksql.logging.processing.NoopProcessingLogContext) PropertyOverrider(io.confluent.ksql.properties.PropertyOverrider) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) Topology(org.apache.kafka.streams.Topology) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) AssertTable(io.confluent.ksql.parser.AssertTable) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) KsqlGenericRecord(io.confluent.ksql.engine.generic.KsqlGenericRecord) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) AssertStream(io.confluent.ksql.parser.tree.AssertStream) AssertStatement(io.confluent.ksql.parser.tree.AssertStatement) SessionConfig(io.confluent.ksql.config.SessionConfig) ImmutableList(com.google.common.collect.ImmutableList) Injector(io.confluent.ksql.statement.Injector) KsqlTestException(io.confluent.ksql.test.KsqlTestException) CreateSource(io.confluent.ksql.parser.tree.CreateSource) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) AssertValues(io.confluent.ksql.parser.tree.AssertValues) KsqlTestFolder(io.confluent.ksql.test.util.KsqlTestFolder) GenericRowSerDe(io.confluent.ksql.serde.GenericRowSerDe) Before(org.junit.Before) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) TestStatement(io.confluent.ksql.test.parser.TestStatement) TestFunctionRegistry(io.confluent.ksql.test.tools.TestFunctionRegistry) Properties(java.util.Properties) Logger(org.slf4j.Logger) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) GenericRecordFactory(io.confluent.ksql.engine.generic.GenericRecordFactory) ServiceInfo(io.confluent.ksql.ServiceInfo) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) SqlTestLoader(io.confluent.ksql.test.parser.SqlTestLoader) AssertTombstone(io.confluent.ksql.parser.tree.AssertTombstone) File(java.io.File) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) GenericKeySerDe(io.confluent.ksql.serde.GenericKeySerDe) Rule(org.junit.Rule) Paths(java.nio.file.Paths) GenericRow(io.confluent.ksql.GenericRow) QueryEventListener(io.confluent.ksql.engine.QueryEventListener) GenericKey(io.confluent.ksql.GenericKey) PersistenceSchema(io.confluent.ksql.schema.ksql.PersistenceSchema) Collections(java.util.Collections) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) TemporaryFolder(org.junit.rules.TemporaryFolder) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Topology(org.apache.kafka.streams.Topology) Properties(java.util.Properties) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) DataSource(io.confluent.ksql.metastore.model.DataSource) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) InsertValues(io.confluent.ksql.parser.tree.InsertValues) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) SetProperty(io.confluent.ksql.parser.tree.SetProperty) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata)

Example 13 with ExecuteResult

use of io.confluent.ksql.KsqlExecutionContext.ExecuteResult in project ksql by confluentinc.

the class ListSourceExecutorTest method shouldShowColumnsSource.

@Test
public void shouldShowColumnsSource() {
    // Given:
    engine.givenSource(DataSourceType.KSTREAM, "SOURCE");
    final ExecuteResult result = engine.getEngine().execute(engine.getServiceContext(), engine.configure("CREATE STREAM SINK AS SELECT * FROM source;"));
    final PersistentQueryMetadata metadata = (PersistentQueryMetadata) result.getQuery().orElseThrow(IllegalArgumentException::new);
    final DataSource stream = engine.getEngine().getMetaStore().getSource(SourceName.of("SINK"));
    // When:
    final SourceDescriptionEntity sourceDescription = (SourceDescriptionEntity) CUSTOM_EXECUTORS.showColumns().execute(ConfiguredStatement.of(PreparedStatement.of("DESCRIBE SINK;", new ShowColumns(SourceName.of("SINK"), false)), SessionConfig.of(engine.getKsqlConfig(), ImmutableMap.of())), SESSION_PROPERTIES, engine.getEngine(), engine.getServiceContext()).getEntity().orElseThrow(IllegalStateException::new);
    // Then:
    final QueryStatusCount queryStatusCount = QueryStatusCount.fromStreamsStateCounts(Collections.singletonMap(metadata.getState(), 1));
    assertThat(sourceDescription.getSourceDescription(), equalTo(SourceDescriptionFactory.create(stream, false, ImmutableList.of(), ImmutableList.of(new RunningQuery(metadata.getStatementString(), ImmutableSet.of(metadata.getSinkName().get().toString(FormatOptions.noEscape())), ImmutableSet.of(metadata.getResultTopic().get().getKafkaTopicName()), metadata.getQueryId(), queryStatusCount, KsqlConstants.KsqlQueryType.PERSISTENT)), Optional.empty(), ImmutableList.of(), ImmutableList.of(), new MetricCollectors())));
}
Also used : RunningQuery(io.confluent.ksql.rest.entity.RunningQuery) SourceDescriptionEntity(io.confluent.ksql.rest.entity.SourceDescriptionEntity) ShowColumns(io.confluent.ksql.parser.tree.ShowColumns) QueryStatusCount(io.confluent.ksql.rest.entity.QueryStatusCount) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) DataSource(io.confluent.ksql.metastore.model.DataSource) Test(org.junit.Test)

Example 14 with ExecuteResult

use of io.confluent.ksql.KsqlExecutionContext.ExecuteResult in project ksql by confluentinc.

the class InteractiveStatementExecutor method executePlan.

private void executePlan(final Command command, final CommandId commandId, final Optional<CommandStatusFuture> commandStatusFuture, final KsqlPlan plan, final Mode mode, final long offset, final boolean restoreInProgress) {
    final KsqlConfig mergedConfig = buildMergedConfig(command);
    final ConfiguredKsqlPlan configured = ConfiguredKsqlPlan.of(plan, SessionConfig.of(mergedConfig, command.getOverwriteProperties()));
    putStatus(commandId, commandStatusFuture, new CommandStatus(CommandStatus.Status.EXECUTING, "Executing statement"));
    final ExecuteResult result = ksqlEngine.execute(serviceContext, configured, restoreInProgress);
    queryIdGenerator.setNextId(offset + 1);
    if (result.getQuery().isPresent()) {
        if (mode == Mode.EXECUTE) {
            result.getQuery().get().start();
        }
    }
    final String successMessage = getSuccessMessage(result);
    final Optional<QueryId> queryId = result.getQuery().map(QueryMetadata::getQueryId);
    final CommandStatus successStatus = new CommandStatus(CommandStatus.Status.SUCCESS, successMessage, queryId);
    putFinalStatus(commandId, commandStatusFuture, successStatus);
}
Also used : ConfiguredKsqlPlan(io.confluent.ksql.planner.plan.ConfiguredKsqlPlan) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryMetadata(io.confluent.ksql.util.QueryMetadata) QueryId(io.confluent.ksql.query.QueryId) KsqlConfig(io.confluent.ksql.util.KsqlConfig) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult)

Aggregations

ExecuteResult (io.confluent.ksql.KsqlExecutionContext.ExecuteResult)14 Test (org.junit.Test)10 ParsedStatement (io.confluent.ksql.parser.KsqlParser.ParsedStatement)6 QueryId (io.confluent.ksql.query.QueryId)6 KsqlConfig (io.confluent.ksql.util.KsqlConfig)5 SessionConfig (io.confluent.ksql.config.SessionConfig)4 ServiceContext (io.confluent.ksql.services.ServiceContext)4 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)4 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)4 Collections (java.util.Collections)4 Map (java.util.Map)4 Optional (java.util.Optional)4 Collectors (java.util.stream.Collectors)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 Iterables (com.google.common.collect.Iterables)3 QueryMetadata (io.confluent.ksql.util.QueryMetadata)3 List (java.util.List)3 ImmutableList (com.google.common.collect.ImmutableList)2 MockSchemaRegistryClient (io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient)2 SchemaRegistryClient (io.confluent.kafka.schemaregistry.client.SchemaRegistryClient)2