Search in sources :

Example 26 with ParsedStatement

use of io.confluent.ksql.parser.KsqlParser.ParsedStatement in project ksql by confluentinc.

the class KsqlTesterTest method execute.

@SuppressWarnings("unchecked")
private void execute(final ParsedStatement parsedStatement) {
    final PreparedStatement<?> engineStatement = engine.prepare(parsedStatement);
    final ConfiguredStatement<?> configured = ConfiguredStatement.of(engineStatement, SessionConfig.of(config, overrides));
    createTopics(engineStatement);
    if (engineStatement.getStatement() instanceof InsertValues) {
        pipeInput((ConfiguredStatement<InsertValues>) configured);
        return;
    } else if (engineStatement.getStatement() instanceof SetProperty) {
        PropertyOverrider.set((ConfiguredStatement<SetProperty>) configured, overrides);
        return;
    } else if (engineStatement.getStatement() instanceof UnsetProperty) {
        PropertyOverrider.unset((ConfiguredStatement<UnsetProperty>) configured, overrides);
        return;
    }
    final ConfiguredStatement<?> injected = formatInjector.inject(configured);
    final ExecuteResult result = engine.execute(serviceContext, injected);
    // is DDL statement
    if (!result.getQuery().isPresent()) {
        return;
    }
    final PersistentQueryMetadata query = (PersistentQueryMetadata) result.getQuery().get();
    final Topology topology = query.getTopology();
    final Properties properties = new Properties();
    properties.putAll(query.getStreamsProperties());
    properties.put(StreamsConfig.STATE_DIR_CONFIG, tmpFolder.getRoot().getAbsolutePath());
    final TopologyTestDriver driver = new TopologyTestDriver(topology, properties);
    final List<TopicInfo> inputTopics = query.getSourceNames().stream().map(sn -> engine.getMetaStore().getSource(sn)).map(ds -> new TopicInfo(ds.getKafkaTopicName(), keySerde(ds), valueSerde(ds))).collect(Collectors.toList());
    // Sink may be Optional for source tables. Once source table query execution is supported, then
    // we would need have a condition to not create an output topic info
    final DataSource output = engine.getMetaStore().getSource(query.getSinkName().get());
    final TopicInfo outputInfo = new TopicInfo(output.getKafkaTopicName(), keySerde(output), valueSerde(output));
    driverPipeline.addDriver(driver, inputTopics, outputInfo);
    drivers.put(query.getQueryId(), new DriverAndProperties(driver, properties));
}
Also used : DataSource(io.confluent.ksql.metastore.model.DataSource) NoSuchFileException(java.nio.file.NoSuchFileException) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) InsertValues(io.confluent.ksql.parser.tree.InsertValues) Serde(org.apache.kafka.common.serialization.Serde) Map(java.util.Map) After(org.junit.After) DefaultFormatInjector(io.confluent.ksql.format.DefaultFormatInjector) QueryId(io.confluent.ksql.query.QueryId) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) Path(java.nio.file.Path) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) Parameterized(org.junit.runners.Parameterized) QueryMetadata(io.confluent.ksql.util.QueryMetadata) MockSchemaRegistryClient(io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient) ImmutableMap(com.google.common.collect.ImmutableMap) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TestDirective(io.confluent.ksql.test.parser.TestDirective) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) Collectors(java.util.stream.Collectors) TestServiceContext(io.confluent.ksql.services.TestServiceContext) Objects(java.util.Objects) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) List(java.util.List) SetProperty(io.confluent.ksql.parser.tree.SetProperty) NoopProcessingLogContext(io.confluent.ksql.logging.processing.NoopProcessingLogContext) PropertyOverrider(io.confluent.ksql.properties.PropertyOverrider) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) Topology(org.apache.kafka.streams.Topology) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Iterables(com.google.common.collect.Iterables) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) AssertTable(io.confluent.ksql.parser.AssertTable) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) KsqlGenericRecord(io.confluent.ksql.engine.generic.KsqlGenericRecord) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) AssertStream(io.confluent.ksql.parser.tree.AssertStream) AssertStatement(io.confluent.ksql.parser.tree.AssertStatement) SessionConfig(io.confluent.ksql.config.SessionConfig) ImmutableList(com.google.common.collect.ImmutableList) Injector(io.confluent.ksql.statement.Injector) KsqlTestException(io.confluent.ksql.test.KsqlTestException) CreateSource(io.confluent.ksql.parser.tree.CreateSource) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) AssertValues(io.confluent.ksql.parser.tree.AssertValues) KsqlTestFolder(io.confluent.ksql.test.util.KsqlTestFolder) GenericRowSerDe(io.confluent.ksql.serde.GenericRowSerDe) Before(org.junit.Before) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) TestStatement(io.confluent.ksql.test.parser.TestStatement) TestFunctionRegistry(io.confluent.ksql.test.tools.TestFunctionRegistry) Properties(java.util.Properties) Logger(org.slf4j.Logger) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) GenericRecordFactory(io.confluent.ksql.engine.generic.GenericRecordFactory) ServiceInfo(io.confluent.ksql.ServiceInfo) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) Test(org.junit.Test) SqlTestLoader(io.confluent.ksql.test.parser.SqlTestLoader) AssertTombstone(io.confluent.ksql.parser.tree.AssertTombstone) File(java.io.File) CreateAsSelect(io.confluent.ksql.parser.tree.CreateAsSelect) GenericKeySerDe(io.confluent.ksql.serde.GenericKeySerDe) Rule(org.junit.Rule) Paths(java.nio.file.Paths) GenericRow(io.confluent.ksql.GenericRow) QueryEventListener(io.confluent.ksql.engine.QueryEventListener) GenericKey(io.confluent.ksql.GenericKey) PersistenceSchema(io.confluent.ksql.schema.ksql.PersistenceSchema) Collections(java.util.Collections) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) TemporaryFolder(org.junit.rules.TemporaryFolder) UnsetProperty(io.confluent.ksql.parser.tree.UnsetProperty) TopologyTestDriver(org.apache.kafka.streams.TopologyTestDriver) Topology(org.apache.kafka.streams.Topology) Properties(java.util.Properties) TopicInfo(io.confluent.ksql.test.driver.TestDriverPipeline.TopicInfo) DataSource(io.confluent.ksql.metastore.model.DataSource) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) InsertValues(io.confluent.ksql.parser.tree.InsertValues) ExecuteResult(io.confluent.ksql.KsqlExecutionContext.ExecuteResult) SetProperty(io.confluent.ksql.parser.tree.SetProperty) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata)

Example 27 with ParsedStatement

use of io.confluent.ksql.parser.KsqlParser.ParsedStatement in project ksql by confluentinc.

the class ExpressionParseTestUtil method parseExpression.

public static Expression parseExpression(final String asText, final MetaStore metaStore) {
    final KsqlParser parser = new DefaultKsqlParser();
    final String ksql = String.format("SELECT %s FROM test1;", asText);
    final ParsedStatement parsedStatement = parser.parse(ksql).get(0);
    final PreparedStatement preparedStatement = parser.prepare(parsedStatement, metaStore);
    final SingleColumn singleColumn = (SingleColumn) ((Query) preparedStatement.getStatement()).getSelect().getSelectItems().get(0);
    return singleColumn.getExpression();
}
Also used : DefaultKsqlParser(io.confluent.ksql.parser.DefaultKsqlParser) KsqlParser(io.confluent.ksql.parser.KsqlParser) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) SingleColumn(io.confluent.ksql.parser.tree.SingleColumn) DefaultKsqlParser(io.confluent.ksql.parser.DefaultKsqlParser)

Example 28 with ParsedStatement

use of io.confluent.ksql.parser.KsqlParser.ParsedStatement in project ksql by confluentinc.

the class RequestHandlerTest method shouldUseCustomExecutor.

@Test
public void shouldUseCustomExecutor() {
    // Given
    final KsqlEntity entity = mock(KsqlEntity.class);
    final StatementExecutor<CreateStream> customExecutor = givenReturningExecutor(CreateStream.class, entity);
    givenRequestHandler(ImmutableMap.of(CreateStream.class, customExecutor));
    // When
    final List<ParsedStatement> statements = KSQL_PARSER.parse(SOME_STREAM_SQL);
    final KsqlEntityList entities = handler.execute(securityContext, statements, sessionProperties);
    // Then
    assertThat(entities, contains(entity));
    verify(customExecutor, times(1)).execute(argThat(is(configured(preparedStatement(instanceOf(CreateStream.class)), ImmutableMap.of(), ksqlConfig))), eq(sessionProperties), eq(ksqlEngine), eq(serviceContext));
}
Also used : KsqlEntityList(io.confluent.ksql.rest.entity.KsqlEntityList) CreateStream(io.confluent.ksql.parser.tree.CreateStream) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) KsqlEntity(io.confluent.ksql.rest.entity.KsqlEntity) Test(org.junit.Test)

Example 29 with ParsedStatement

use of io.confluent.ksql.parser.KsqlParser.ParsedStatement in project ksql by confluentinc.

the class RequestHandlerTest method shouldDistributeProperties.

@Test
public void shouldDistributeProperties() {
    // Given
    givenRequestHandler(ImmutableMap.of());
    when(sessionProperties.getMutableScopedProperties()).thenReturn(ImmutableMap.of("x", "y"));
    // When
    final List<ParsedStatement> statements = KSQL_PARSER.parse(SOME_STREAM_SQL);
    final KsqlEntityList entities = handler.execute(securityContext, statements, sessionProperties);
    // Then
    assertThat(entities, contains(entity));
    verify(distributor, times(2)).execute(argThat(is(configured(preparedStatement(instanceOf(CreateStream.class)), ImmutableMap.of("x", "y"), ksqlConfig))), eq(ksqlEngine), eq(securityContext));
}
Also used : KsqlEntityList(io.confluent.ksql.rest.entity.KsqlEntityList) CreateStream(io.confluent.ksql.parser.tree.CreateStream) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) Test(org.junit.Test)

Example 30 with ParsedStatement

use of io.confluent.ksql.parser.KsqlParser.ParsedStatement in project ksql by confluentinc.

the class RequestValidatorTest method shouldThrowIfNoValidatorAvailable.

@Test
public void shouldThrowIfNoValidatorAvailable() {
    // Given:
    final List<ParsedStatement> statements = givenParsed("EXPLAIN X;");
    // When:
    final Exception e = assertThrows(KsqlStatementException.class, () -> validator.validate(serviceContext, statements, sessionProperties, "sql"));
    // Then:
    assertThat(e.getMessage(), containsString("Do not know how to validate statement"));
}
Also used : ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) KsqlException(io.confluent.ksql.util.KsqlException) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) Test(org.junit.Test)

Aggregations

ParsedStatement (io.confluent.ksql.parser.KsqlParser.ParsedStatement)37 Test (org.junit.Test)22 KsqlStatementException (io.confluent.ksql.util.KsqlStatementException)15 KsqlException (io.confluent.ksql.util.KsqlException)10 CreateStream (io.confluent.ksql.parser.tree.CreateStream)7 ExecuteResult (io.confluent.ksql.KsqlExecutionContext.ExecuteResult)6 KsqlEntityList (io.confluent.ksql.rest.entity.KsqlEntityList)6 Optional (java.util.Optional)6 DefaultKsqlParser (io.confluent.ksql.parser.DefaultKsqlParser)5 PreparedStatement (io.confluent.ksql.parser.KsqlParser.PreparedStatement)5 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)5 Injector (io.confluent.ksql.statement.Injector)5 KsqlConfig (io.confluent.ksql.util.KsqlConfig)5 QueryMetadata (io.confluent.ksql.util.QueryMetadata)5 IOException (java.io.IOException)5 HashMap (java.util.HashMap)5 KsqlExecutionContext (io.confluent.ksql.KsqlExecutionContext)4 KsqlEntity (io.confluent.ksql.rest.entity.KsqlEntity)4 ServiceContext (io.confluent.ksql.services.ServiceContext)4 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)4