Search in sources :

Example 31 with PreparedStatement

use of io.confluent.ksql.parser.KsqlParser.PreparedStatement in project ksql by confluentinc.

the class KsqlEngineTest method shouldHandleMultipleStatements.

@SuppressWarnings("unchecked")
@Test
public void shouldHandleMultipleStatements() {
    // Given:
    final String sql = "" + "-- single line comment\n" + "/*\n" + "   Multi-line comment\n" + "*/\n" + "CREATE STREAM S0 (a INT, b VARCHAR) " + "      WITH (kafka_topic='s0_topic', value_format='DELIMITED', key_format='KAFKA');\n" + "\n" + "CREATE TABLE T1 (f0 BIGINT PRIMARY KEY, f1 DOUBLE) " + "     WITH (kafka_topic='t1_topic', value_format='JSON', key_format='KAFKA');\n" + "\n" + "CREATE STREAM S1 AS SELECT * FROM S0;\n" + "\n" + "CREATE STREAM S2 AS SELECT * FROM S0;\n" + "\n" + "DROP TABLE T1;";
    givenTopicsExist("s0_topic", "t1_topic");
    final List<QueryMetadata> queries = new ArrayList<>();
    // When:
    final List<PreparedStatement<?>> preparedStatements = ksqlEngine.parse(sql).stream().map(stmt -> {
        final PreparedStatement<?> prepared = ksqlEngine.prepare(stmt);
        final ExecuteResult result = ksqlEngine.execute(serviceContext, ConfiguredStatement.of(prepared, SessionConfig.of(ksqlConfig, new HashMap<>())));
        result.getQuery().ifPresent(queries::add);
        return prepared;
    }).collect(Collectors.toList());
    // Then:
    final List<?> statements = preparedStatements.stream().map(PreparedStatement::getStatement).collect(Collectors.toList());
    assertThat(statements, contains(instanceOf(CreateStream.class), instanceOf(CreateTable.class), instanceOf(CreateStreamAsSelect.class), instanceOf(CreateStreamAsSelect.class), instanceOf(DropTable.class)));
    assertThat(queries, hasSize(2));
}
Also used : Query(io.confluent.ksql.parser.tree.Query) Arrays(java.util.Arrays) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) SourceName(io.confluent.ksql.name.SourceName) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) Matchers.not(org.hamcrest.Matchers.not) ServiceContext(io.confluent.ksql.services.ServiceContext) KsqlExceptionMatcher.statementText(io.confluent.ksql.util.KsqlExceptionMatcher.statementText) SandboxedTransientQueryMetadata(io.confluent.ksql.util.SandboxedTransientQueryMetadata) MutableMetaStore(io.confluent.ksql.metastore.MutableMetaStore) Matchers.hasItems(org.hamcrest.Matchers.hasItems) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) QueryCleanupTask(io.confluent.ksql.engine.QueryCleanupService.QueryCleanupTask) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) CreateTable(io.confluent.ksql.parser.tree.CreateTable) KsqlEngineTestUtil.execute(io.confluent.ksql.engine.KsqlEngineTestUtil.execute) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Map(java.util.Map) After(org.junit.After) Spy(org.mockito.Spy) SandboxedPersistentQueryMetadataImpl(io.confluent.ksql.util.SandboxedPersistentQueryMetadataImpl) Assert.fail(org.junit.Assert.fail) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) QueryId(io.confluent.ksql.query.QueryId) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryMetadata(io.confluent.ksql.util.QueryMetadata) MockSchemaRegistryClient(io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient) FakeKafkaConsumerGroupClient(io.confluent.ksql.services.FakeKafkaConsumerGroupClient) SystemColumns(io.confluent.ksql.schema.ksql.SystemColumns) Schema(org.apache.avro.Schema) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) MetaStoreFixture(io.confluent.ksql.util.MetaStoreFixture) Collectors(java.util.stream.Collectors) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TestServiceContext(io.confluent.ksql.services.TestServiceContext) List(java.util.List) ParseFailedException(io.confluent.ksql.parser.exception.ParseFailedException) Matchers.contains(org.hamcrest.Matchers.contains) KsqlExecutionContext(io.confluent.ksql.KsqlExecutionContext) Matchers.equalTo(org.hamcrest.Matchers.equalTo) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) KsqlConstants(io.confluent.ksql.util.KsqlConstants) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) Mockito.mock(org.mockito.Mockito.mock) KsqlExceptionMatcher.rawMessage(io.confluent.ksql.util.KsqlExceptionMatcher.rawMessage) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Iterables(com.google.common.collect.Iterables) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) CreateStreamAsSelect(io.confluent.ksql.parser.tree.CreateStreamAsSelect) ReservedInternalTopics(io.confluent.ksql.util.ReservedInternalTopics) Supplier(java.util.function.Supplier) AvroSchema(io.confluent.kafka.schemaregistry.avro.AvroSchema) ArrayList(java.util.ArrayList) SessionConfig(io.confluent.ksql.config.SessionConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) Collections.emptyMap(java.util.Collections.emptyMap) DropTable(io.confluent.ksql.parser.tree.DropTable) Matchers.empty(org.hamcrest.Matchers.empty) Assert.assertTrue(org.junit.Assert.assertTrue) FieldMatchers.hasFullName(io.confluent.ksql.metastore.model.MetaStoreMatchers.FieldMatchers.hasFullName) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) SandboxedBinPackedPersistentQueryMetadataImpl(io.confluent.ksql.util.SandboxedBinPackedPersistentQueryMetadataImpl) ConfigException(org.apache.kafka.common.config.ConfigException) Mockito.verify(org.mockito.Mockito.verify) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) SchemaBuilder(org.apache.avro.SchemaBuilder) KsqlConfigTestUtil(io.confluent.ksql.KsqlConfigTestUtil) Mockito.never(org.mockito.Mockito.never) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaStreams(org.apache.kafka.streams.KafkaStreams) Assert(org.junit.Assert) Collections(java.util.Collections) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) SandboxedTransientQueryMetadata(io.confluent.ksql.util.SandboxedTransientQueryMetadata) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryMetadata(io.confluent.ksql.util.QueryMetadata) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) Matchers.containsString(org.hamcrest.Matchers.containsString) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) Test(org.junit.Test)

Example 32 with PreparedStatement

use of io.confluent.ksql.parser.KsqlParser.PreparedStatement in project ksql by confluentinc.

the class KsqlTesterTest method execute.

@SuppressWarnings("unchecked")
private void execute(final ParsedStatement parsedStatement) {
    final PreparedStatement<?> engineStatement = engine.prepare(parsedStatement);
    final ConfiguredStatement<?> configured = ConfiguredStatement.of(engineStatement, SessionConfig.of(config, overrides));
    createTopics(engineStatement);
    if (engineStatement.getStatement() instanceof InsertValues) {
        pipeInput((ConfiguredStatement<InsertValues>) configured);
        return;
    } else if (engineStatement.getStatement() instanceof SetProperty) {
        PropertyOverrider.set((ConfiguredStatement<SetProperty>) configured, overrides);
        return;
    } else if (engineStatement.getStatement() instanceof UnsetProperty) {
        PropertyOverrider.unset((ConfiguredStatement<UnsetProperty>) configured, overrides);
        return;
    }
    final ConfiguredStatement<?> injected = formatInjector.inject(configured);
    final ExecuteResult result = engine.execute(serviceContext, injected);
    // is DDL statement
    if (!result.getQuery().isPresent()) {
        return;
    }
    final PersistentQueryMetadata query = (PersistentQueryMetadata) result.getQuery().get();
    final Topology topology = query.getTopology();
    final Properties properties = new Properties();
    properties.putAll(query.getStreamsProperties());
    properties.put(StreamsConfig.STATE_DIR_CONFIG, tmpFolder.getRoot().getAbsolutePath());
    final TopologyTestDriver driver = new TopologyTestDriver(topology, properties);
    final List<TopicInfo> inputTopics = query.getSourceNames().stream().map(sn -> engine.getMetaStore().getSource(sn)).map(ds -> new TopicInfo(ds.getKafkaTopicName(), keySerde(ds), valueSerde(ds))).collect(Collectors.toList());
    // Sink may be Optional for source tables. Once source table query execution is supported, then
    // we would need have a condition to not create an output topic info
    final DataSource output = engine.getMetaStore().getSource(query.getSinkName().get());
    final TopicInfo outputInfo = new TopicInfo(output.getKafkaTopicName(), keySerde(output), valueSerde(output));
    driverPipeline.addDriver(driver, inputTopics, outputInfo);
    drivers.put(query.getQueryId(), new DriverAndProperties(driver, properties));
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) NoSuchFileException(java.nio.file.NoSuchFileException) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) InsertValues(io.confluent.ksql.parser.tree.InsertValues) Serde(org.apache.kafka.common.serialization.Serde) Map(java.util.Map) After(org.junit.After) DefaultFormatInjector(io.confluent.ksql.format.DefaultFormatInjector) QueryId(io.confluent.ksql.query.QueryId) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) Path(java.nio.file.Path) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) Parameterized(org.junit.runners.Parameterized) QueryMetadata(io.confluent.ksql.util.QueryMetadata) MockSchemaRegistryClient(io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient) ImmutableMap(com.google.common.collect.ImmutableMap) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TestDirective(io.confluent.ksql.test.parser.TestDirective) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) Collectors(java.util.stream.Collectors) TestServiceContext(io.confluent.ksql.services.TestServiceContext) Objects(java.util.Objects) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) List(java.util.List) SetProperty(io.confluent.ksql.parser.tree.SetProperty) NoopProcessingLogContext(io.confluent.ksql.logging.processing.NoopProcessingLogContext) PropertyOverrider(io.confluent.ksql.properties.PropertyOverrider) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) Topology(org.apache.kafka.streams.Topology) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) AssertTable(io.confluent.ksql.parser.AssertTable) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) KsqlGenericRecord(io.confluent.ksql.engine.generic.KsqlGenericRecord) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) AssertStream(io.confluent.ksql.parser.tree.AssertStream) AssertStatement(io.confluent.ksql.parser.tree.AssertStatement) SessionConfig(io.confluent.ksql.config.SessionConfig) ImmutableList(com.google.common.collect.ImmutableList) Injector(io.confluent.ksql.statement.Injector) KsqlTestException(io.confluent.ksql.test.KsqlTestException) CreateSource(io.confluent.ksql.parser.tree.CreateSource) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) AssertValues(io.confluent.ksql.parser.tree.AssertValues) KsqlTestFolder(io.confluent.ksql.test.util.KsqlTestFolder) GenericRowSerDe(io.confluent.ksql.serde.GenericRowSerDe) Before(org.junit.Before) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) TestStatement(io.confluent.ksql.test.parser.TestStatement) TestFunctionRegistry(io.confluent.ksql.test.tools.TestFunctionRegistry) Properties(java.util.Properties) Logger(org.slf4j.Logger) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) GenericRecordFactory(io.confluent.ksql.engine.generic.GenericRecordFactory) ServiceInfo(io.confluent.ksql.ServiceInfo) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) SqlTestLoader(io.confluent.ksql.test.parser.SqlTestLoader) AssertTombstone(io.confluent.ksql.parser.tree.AssertTombstone) File(java.io.File) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) GenericKeySerDe(io.confluent.ksql.serde.GenericKeySerDe) Rule(org.junit.Rule) Paths(java.nio.file.Paths) GenericRow(io.confluent.ksql.GenericRow) QueryEventListener(io.confluent.ksql.engine.QueryEventListener) GenericKey(io.confluent.ksql.GenericKey) PersistenceSchema(io.confluent.ksql.schema.ksql.PersistenceSchema) Collections(java.util.Collections) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) TemporaryFolder(org.junit.rules.TemporaryFolder) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Topology(org.apache.kafka.streams.Topology) Properties(java.util.Properties) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) DataSource(io.confluent.ksql.metastore.model.DataSource) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) InsertValues(io.confluent.ksql.parser.tree.InsertValues) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) SetProperty(io.confluent.ksql.parser.tree.SetProperty) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata)

Aggregations

PreparedStatement (io.confluent.ksql.parser.KsqlParser.PreparedStatement)32 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)21 Test (org.junit.Test)20 Statement (io.confluent.ksql.parser.tree.Statement)18 KsqlException (io.confluent.ksql.util.KsqlException)15 Query (io.confluent.ksql.parser.tree.Query)9 Matchers.containsString (org.hamcrest.Matchers.containsString)8 KsqlTopicAuthorizationException (io.confluent.ksql.exception.KsqlTopicAuthorizationException)7 ParsedStatement (io.confluent.ksql.parser.KsqlParser.ParsedStatement)7 InsertInto (io.confluent.ksql.parser.tree.InsertInto)7 KsqlRestException (io.confluent.ksql.rest.server.resources.KsqlRestException)7 ServiceContext (io.confluent.ksql.services.ServiceContext)7 KsqlConfig (io.confluent.ksql.util.KsqlConfig)7 Optional (java.util.Optional)7 KsqlExecutionContext (io.confluent.ksql.KsqlExecutionContext)6 SessionConfig (io.confluent.ksql.config.SessionConfig)6 DataSource (io.confluent.ksql.metastore.model.DataSource)6 KsqlServerException (io.confluent.ksql.util.KsqlServerException)6 KsqlStatementException (io.confluent.ksql.util.KsqlStatementException)6 List (java.util.List)6