Search in sources :

Example 31 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class KsqlResourceTest method shouldThrowOnHandleStatementIfNotConfigured.

@Test
public void shouldThrowOnHandleStatementIfNotConfigured() {
    // Given:
    ksqlResource = new KsqlResource(ksqlEngine, commandRunner, DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT, activenessRegistrar, (ec, sc) -> InjectorChain.of(schemaInjectorFactory.apply(sc), topicInjectorFactory.apply(ec), new TopicDeleteInjector(ec, sc)), Optional.of(authorizationValidator), errorsHandler, connectErrorHandler, denyListPropertyValidator, commandRunnerWarning);
    // When:
    final KsqlRestException e = assertThrows(KsqlRestException.class, () -> ksqlResource.handleKsqlStatements(securityContext, new KsqlRequest("query", emptyMap(), emptyMap(), null)));
    // Then:
    assertThat(e, exceptionStatusCode(CoreMatchers.is(SERVICE_UNAVAILABLE.code())));
    assertThat(e, exceptionErrorMessage(errorMessage(Matchers.is("Server initializing"))));
}
Also used : ClusterTerminateRequest(io.confluent.ksql.rest.entity.ClusterTerminateRequest) DROP(io.confluent.ksql.rest.entity.CommandId.Action.DROP) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) ColumnName(io.confluent.ksql.name.ColumnName) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) KsqlRestExceptionMatchers.exceptionStatementErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatementErrorMessage) Mockito.doThrow(org.mockito.Mockito.doThrow) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Duration(java.time.Duration) Map(java.util.Map) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) SystemColumns(io.confluent.ksql.schema.ksql.SystemColumns) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) EXECUTE(io.confluent.ksql.rest.entity.CommandId.Action.EXECUTE) ActivenessRegistrar(io.confluent.ksql.version.metrics.ActivenessRegistrar) KsqlRestConfig(io.confluent.ksql.rest.server.KsqlRestConfig) QueryDescription(io.confluent.ksql.rest.entity.QueryDescription) Metrics(org.apache.kafka.common.metrics.Metrics) SERVICE_UNAVAILABLE(io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE) KsqlException(io.confluent.ksql.util.KsqlException) FunctionCategory(io.confluent.ksql.function.FunctionCategory) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) Mockito.mock(org.mockito.Mockito.mock) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) TOPIC(io.confluent.ksql.rest.entity.CommandId.Type.TOPIC) RunWith(org.junit.runner.RunWith) KsqlIdentifierTestUtil(io.confluent.ksql.test.util.KsqlIdentifierTestUtil) KsqlSecurityContext(io.confluent.ksql.security.KsqlSecurityContext) Supplier(java.util.function.Supplier) SessionConfig(io.confluent.ksql.config.SessionConfig) Answer(org.mockito.stubbing.Answer) QueryStatusCount(io.confluent.ksql.rest.entity.QueryStatusCount) Injector(io.confluent.ksql.statement.Injector) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) UserFunctionLoader(io.confluent.ksql.function.UserFunctionLoader) CoreMatchers.nullValue(org.hamcrest.CoreMatchers.nullValue) Before(org.junit.Before) Producer(org.apache.kafka.clients.producer.Producer) SourceDescriptionFactory(io.confluent.ksql.rest.entity.SourceDescriptionFactory) SandboxedServiceContext(io.confluent.ksql.services.SandboxedServiceContext) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) ValueFormat(io.confluent.ksql.serde.ValueFormat) ConnectServerErrors(io.confluent.ksql.rest.server.execution.ConnectServerErrors) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) Mockito.never(org.mockito.Mockito.never) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Matcher(org.hamcrest.Matcher) DefaultErrorMessages(io.confluent.ksql.rest.DefaultErrorMessages) Mockito.reset(org.mockito.Mockito.reset) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ParserMatchers.configured(io.confluent.ksql.parser.ParserMatchers.configured) KsqlEngineTestUtil(io.confluent.ksql.engine.KsqlEngineTestUtil) ParserMatchers.preparedStatementText(io.confluent.ksql.parser.ParserMatchers.preparedStatementText) SimpleFunctionInfo(io.confluent.ksql.rest.entity.SimpleFunctionInfo) StringLiteral(io.confluent.ksql.execution.expression.tree.StringLiteral) KsqlAuthorizationValidator(io.confluent.ksql.security.KsqlAuthorizationValidator) MockitoHamcrest.argThat(org.mockito.hamcrest.MockitoHamcrest.argThat) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryDescriptionList(io.confluent.ksql.rest.entity.QueryDescriptionList) ImmutableSet(com.google.common.collect.ImmutableSet) Command(io.confluent.ksql.rest.server.computation.Command) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TypeSafeMatcher(org.hamcrest.TypeSafeMatcher) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) Assert.assertThrows(org.junit.Assert.assertThrows) KsqlEntityList(io.confluent.ksql.rest.entity.KsqlEntityList) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) Function(java.util.function.Function) CommandStatusFuture(io.confluent.ksql.rest.server.computation.CommandStatusFuture) BAD_REQUEST(io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST) Sandbox(io.confluent.ksql.util.Sandbox) ImmutableList(com.google.common.collect.ImmutableList) ConfiguredKsqlPlan(io.confluent.ksql.planner.plan.ConfiguredKsqlPlan) TerminateCluster(io.confluent.ksql.rest.util.TerminateCluster) Queries(io.confluent.ksql.rest.entity.Queries) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) CoreMatchers.hasItems(org.hamcrest.CoreMatchers.hasItems) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) EndpointResponse(io.confluent.ksql.rest.EndpointResponse) Mockito.when(org.mockito.Mockito.when) TABLE(io.confluent.ksql.rest.entity.CommandId.Type.TABLE) SourceInfo(io.confluent.ksql.rest.entity.SourceInfo) TopicDeleteInjector(io.confluent.ksql.topic.TopicDeleteInjector) KsqlStatementErrorMessage(io.confluent.ksql.rest.entity.KsqlStatementErrorMessage) DataSource(io.confluent.ksql.metastore.model.DataSource) StreamsList(io.confluent.ksql.rest.entity.StreamsList) SourceName(io.confluent.ksql.name.SourceName) RunningQuery(io.confluent.ksql.rest.entity.RunningQuery) Matchers.not(org.hamcrest.Matchers.not) KsqlTopicAuthorizationException(io.confluent.ksql.exception.KsqlTopicAuthorizationException) SourceDescription(io.confluent.ksql.rest.entity.SourceDescription) QueuedCommandStatus(io.confluent.ksql.rest.server.computation.QueuedCommandStatus) InjectorChain(io.confluent.ksql.statement.InjectorChain) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) Mockito.doReturn(org.mockito.Mockito.doReturn) CREATE(io.confluent.ksql.rest.entity.CommandId.Action.CREATE) QueryMetadata(io.confluent.ksql.util.QueryMetadata) FakeKafkaConsumerGroupClient(io.confluent.ksql.services.FakeKafkaConsumerGroupClient) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) MockedStatic(org.mockito.MockedStatic) Matchers.contains(org.hamcrest.Matchers.contains) KsqlExecutionContext(io.confluent.ksql.KsqlExecutionContext) Matchers.is(org.hamcrest.Matchers.is) OK(io.netty.handler.codec.http.HttpResponseStatus.OK) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) FormatFactory(io.confluent.ksql.serde.FormatFactory) Mock(org.mockito.Mock) KsqlRestExceptionMatchers.exceptionErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionErrorMessage) CreateStream(io.confluent.ksql.parser.tree.CreateStream) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DropSourceCommand(io.confluent.ksql.execution.ddl.commands.DropSourceCommand) Properties(java.util.Properties) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) AclOperation(org.apache.kafka.common.acl.AclOperation) File(java.io.File) KsqlConfigTestUtil(io.confluent.ksql.KsqlConfigTestUtil) TableElements(io.confluent.ksql.parser.tree.TableElements) ERROR_CODE_FORBIDDEN_KAFKA_ACCESS(io.confluent.ksql.rest.Errors.ERROR_CODE_FORBIDDEN_KAFKA_ACCESS) SqlTypes(io.confluent.ksql.schema.ksql.types.SqlTypes) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) KsqlStatementErrorMessageMatchers.statement(io.confluent.ksql.rest.entity.KsqlStatementErrorMessageMatchers.statement) CoreMatchers(org.hamcrest.CoreMatchers) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) KsqlPlan(io.confluent.ksql.engine.KsqlPlan) FunctionNameList(io.confluent.ksql.rest.entity.FunctionNameList) ServiceContext(io.confluent.ksql.services.ServiceContext) TimeoutException(java.util.concurrent.TimeoutException) CommandId(io.confluent.ksql.rest.entity.CommandId) TableElement(io.confluent.ksql.parser.tree.TableElement) DenyListPropertyValidator(io.confluent.ksql.properties.DenyListPropertyValidator) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) After(org.junit.After) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) QueryLogger(io.confluent.ksql.logging.query.QueryLogger) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) EntityUtil(io.confluent.ksql.rest.util.EntityUtil) SourceDescriptionList(io.confluent.ksql.rest.entity.SourceDescriptionList) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) PropertiesList(io.confluent.ksql.rest.entity.PropertiesList) ImmutableMap(com.google.common.collect.ImmutableMap) Errors(io.confluent.ksql.rest.Errors) KsqlConfig(io.confluent.ksql.util.KsqlConfig) QueryDescriptionFactory(io.confluent.ksql.rest.entity.QueryDescriptionFactory) TestServiceContext(io.confluent.ksql.services.TestServiceContext) List(java.util.List) KsqlErrorMessageMatchers.errorCode(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorCode) KsqlEngineMetrics(io.confluent.ksql.internal.KsqlEngineMetrics) Optional(java.util.Optional) FormatInfo(io.confluent.ksql.serde.FormatInfo) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) StreamsConfig(org.apache.kafka.streams.StreamsConfig) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) FORBIDDEN(io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN) INTERNAL_SERVER_ERROR(io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR) KeyFormat(io.confluent.ksql.serde.KeyFormat) STREAM(io.confluent.ksql.rest.entity.CommandId.Type.STREAM) QueryDescriptionEntity(io.confluent.ksql.rest.entity.QueryDescriptionEntity) HashMap(java.util.HashMap) AvroSchema(io.confluent.kafka.schemaregistry.avro.AvroSchema) KsqlEntity(io.confluent.ksql.rest.entity.KsqlEntity) FunctionType(io.confluent.ksql.rest.entity.FunctionType) SourceDescriptionEntity(io.confluent.ksql.rest.entity.SourceDescriptionEntity) KsqlErrorMessageMatchers.errorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorMessage) TablesList(io.confluent.ksql.rest.entity.TablesList) DELETE_TOPIC_LIST_PROP(io.confluent.ksql.rest.entity.ClusterTerminateRequest.DELETE_TOPIC_LIST_PROP) Description(org.hamcrest.Description) CommandStatusEntity(io.confluent.ksql.rest.entity.CommandStatusEntity) Matchers(org.hamcrest.Matchers) KsqlRestExceptionMatchers.exceptionStatusCode(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatusCode) Property(io.confluent.ksql.rest.entity.PropertiesList.Property) Mockito.verify(org.mockito.Mockito.verify) Mockito(org.mockito.Mockito) Collections(java.util.Collections) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) TopicDeleteInjector(io.confluent.ksql.topic.TopicDeleteInjector) Test(org.junit.Test)

Example 32 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class KsqlResourceTest method setUpKsqlResource.

private void setUpKsqlResource() {
    ksqlResource = new KsqlResource(ksqlEngine, commandRunner, DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT, activenessRegistrar, (ec, sc) -> InjectorChain.of(schemaInjectorFactory.apply(sc), topicInjectorFactory.apply(ec), new TopicDeleteInjector(ec, sc)), Optional.of(authorizationValidator), errorsHandler, connectErrorHandler, denyListPropertyValidator, commandRunnerWarning);
    ksqlResource.configure(ksqlConfig);
}
Also used : ClusterTerminateRequest(io.confluent.ksql.rest.entity.ClusterTerminateRequest) DROP(io.confluent.ksql.rest.entity.CommandId.Action.DROP) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) ColumnName(io.confluent.ksql.name.ColumnName) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) KsqlRestExceptionMatchers.exceptionStatementErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatementErrorMessage) Mockito.doThrow(org.mockito.Mockito.doThrow) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Duration(java.time.Duration) Map(java.util.Map) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) SystemColumns(io.confluent.ksql.schema.ksql.SystemColumns) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) EXECUTE(io.confluent.ksql.rest.entity.CommandId.Action.EXECUTE) ActivenessRegistrar(io.confluent.ksql.version.metrics.ActivenessRegistrar) KsqlRestConfig(io.confluent.ksql.rest.server.KsqlRestConfig) QueryDescription(io.confluent.ksql.rest.entity.QueryDescription) Metrics(org.apache.kafka.common.metrics.Metrics) SERVICE_UNAVAILABLE(io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE) KsqlException(io.confluent.ksql.util.KsqlException) FunctionCategory(io.confluent.ksql.function.FunctionCategory) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) Mockito.mock(org.mockito.Mockito.mock) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) TOPIC(io.confluent.ksql.rest.entity.CommandId.Type.TOPIC) RunWith(org.junit.runner.RunWith) KsqlIdentifierTestUtil(io.confluent.ksql.test.util.KsqlIdentifierTestUtil) KsqlSecurityContext(io.confluent.ksql.security.KsqlSecurityContext) Supplier(java.util.function.Supplier) SessionConfig(io.confluent.ksql.config.SessionConfig) Answer(org.mockito.stubbing.Answer) QueryStatusCount(io.confluent.ksql.rest.entity.QueryStatusCount) Injector(io.confluent.ksql.statement.Injector) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) UserFunctionLoader(io.confluent.ksql.function.UserFunctionLoader) CoreMatchers.nullValue(org.hamcrest.CoreMatchers.nullValue) Before(org.junit.Before) Producer(org.apache.kafka.clients.producer.Producer) SourceDescriptionFactory(io.confluent.ksql.rest.entity.SourceDescriptionFactory) SandboxedServiceContext(io.confluent.ksql.services.SandboxedServiceContext) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) ValueFormat(io.confluent.ksql.serde.ValueFormat) ConnectServerErrors(io.confluent.ksql.rest.server.execution.ConnectServerErrors) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) Mockito.never(org.mockito.Mockito.never) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Matcher(org.hamcrest.Matcher) DefaultErrorMessages(io.confluent.ksql.rest.DefaultErrorMessages) Mockito.reset(org.mockito.Mockito.reset) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ParserMatchers.configured(io.confluent.ksql.parser.ParserMatchers.configured) KsqlEngineTestUtil(io.confluent.ksql.engine.KsqlEngineTestUtil) ParserMatchers.preparedStatementText(io.confluent.ksql.parser.ParserMatchers.preparedStatementText) SimpleFunctionInfo(io.confluent.ksql.rest.entity.SimpleFunctionInfo) StringLiteral(io.confluent.ksql.execution.expression.tree.StringLiteral) KsqlAuthorizationValidator(io.confluent.ksql.security.KsqlAuthorizationValidator) MockitoHamcrest.argThat(org.mockito.hamcrest.MockitoHamcrest.argThat) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryDescriptionList(io.confluent.ksql.rest.entity.QueryDescriptionList) ImmutableSet(com.google.common.collect.ImmutableSet) Command(io.confluent.ksql.rest.server.computation.Command) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TypeSafeMatcher(org.hamcrest.TypeSafeMatcher) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) Assert.assertThrows(org.junit.Assert.assertThrows) KsqlEntityList(io.confluent.ksql.rest.entity.KsqlEntityList) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) Function(java.util.function.Function) CommandStatusFuture(io.confluent.ksql.rest.server.computation.CommandStatusFuture) BAD_REQUEST(io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST) Sandbox(io.confluent.ksql.util.Sandbox) ImmutableList(com.google.common.collect.ImmutableList) ConfiguredKsqlPlan(io.confluent.ksql.planner.plan.ConfiguredKsqlPlan) TerminateCluster(io.confluent.ksql.rest.util.TerminateCluster) Queries(io.confluent.ksql.rest.entity.Queries) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) CoreMatchers.hasItems(org.hamcrest.CoreMatchers.hasItems) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) EndpointResponse(io.confluent.ksql.rest.EndpointResponse) Mockito.when(org.mockito.Mockito.when) TABLE(io.confluent.ksql.rest.entity.CommandId.Type.TABLE) SourceInfo(io.confluent.ksql.rest.entity.SourceInfo) TopicDeleteInjector(io.confluent.ksql.topic.TopicDeleteInjector) KsqlStatementErrorMessage(io.confluent.ksql.rest.entity.KsqlStatementErrorMessage) DataSource(io.confluent.ksql.metastore.model.DataSource) StreamsList(io.confluent.ksql.rest.entity.StreamsList) SourceName(io.confluent.ksql.name.SourceName) RunningQuery(io.confluent.ksql.rest.entity.RunningQuery) Matchers.not(org.hamcrest.Matchers.not) KsqlTopicAuthorizationException(io.confluent.ksql.exception.KsqlTopicAuthorizationException) SourceDescription(io.confluent.ksql.rest.entity.SourceDescription) QueuedCommandStatus(io.confluent.ksql.rest.server.computation.QueuedCommandStatus) InjectorChain(io.confluent.ksql.statement.InjectorChain) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) Mockito.doReturn(org.mockito.Mockito.doReturn) CREATE(io.confluent.ksql.rest.entity.CommandId.Action.CREATE) QueryMetadata(io.confluent.ksql.util.QueryMetadata) FakeKafkaConsumerGroupClient(io.confluent.ksql.services.FakeKafkaConsumerGroupClient) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) MockedStatic(org.mockito.MockedStatic) Matchers.contains(org.hamcrest.Matchers.contains) KsqlExecutionContext(io.confluent.ksql.KsqlExecutionContext) Matchers.is(org.hamcrest.Matchers.is) OK(io.netty.handler.codec.http.HttpResponseStatus.OK) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) FormatFactory(io.confluent.ksql.serde.FormatFactory) Mock(org.mockito.Mock) KsqlRestExceptionMatchers.exceptionErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionErrorMessage) CreateStream(io.confluent.ksql.parser.tree.CreateStream) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DropSourceCommand(io.confluent.ksql.execution.ddl.commands.DropSourceCommand) Properties(java.util.Properties) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) AclOperation(org.apache.kafka.common.acl.AclOperation) File(java.io.File) KsqlConfigTestUtil(io.confluent.ksql.KsqlConfigTestUtil) TableElements(io.confluent.ksql.parser.tree.TableElements) ERROR_CODE_FORBIDDEN_KAFKA_ACCESS(io.confluent.ksql.rest.Errors.ERROR_CODE_FORBIDDEN_KAFKA_ACCESS) SqlTypes(io.confluent.ksql.schema.ksql.types.SqlTypes) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) KsqlStatementErrorMessageMatchers.statement(io.confluent.ksql.rest.entity.KsqlStatementErrorMessageMatchers.statement) CoreMatchers(org.hamcrest.CoreMatchers) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) KsqlPlan(io.confluent.ksql.engine.KsqlPlan) FunctionNameList(io.confluent.ksql.rest.entity.FunctionNameList) ServiceContext(io.confluent.ksql.services.ServiceContext) TimeoutException(java.util.concurrent.TimeoutException) CommandId(io.confluent.ksql.rest.entity.CommandId) TableElement(io.confluent.ksql.parser.tree.TableElement) DenyListPropertyValidator(io.confluent.ksql.properties.DenyListPropertyValidator) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) After(org.junit.After) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) QueryLogger(io.confluent.ksql.logging.query.QueryLogger) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) EntityUtil(io.confluent.ksql.rest.util.EntityUtil) SourceDescriptionList(io.confluent.ksql.rest.entity.SourceDescriptionList) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) PropertiesList(io.confluent.ksql.rest.entity.PropertiesList) ImmutableMap(com.google.common.collect.ImmutableMap) Errors(io.confluent.ksql.rest.Errors) KsqlConfig(io.confluent.ksql.util.KsqlConfig) QueryDescriptionFactory(io.confluent.ksql.rest.entity.QueryDescriptionFactory) TestServiceContext(io.confluent.ksql.services.TestServiceContext) List(java.util.List) KsqlErrorMessageMatchers.errorCode(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorCode) KsqlEngineMetrics(io.confluent.ksql.internal.KsqlEngineMetrics) Optional(java.util.Optional) FormatInfo(io.confluent.ksql.serde.FormatInfo) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) StreamsConfig(org.apache.kafka.streams.StreamsConfig) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) FORBIDDEN(io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN) INTERNAL_SERVER_ERROR(io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR) KeyFormat(io.confluent.ksql.serde.KeyFormat) STREAM(io.confluent.ksql.rest.entity.CommandId.Type.STREAM) QueryDescriptionEntity(io.confluent.ksql.rest.entity.QueryDescriptionEntity) HashMap(java.util.HashMap) AvroSchema(io.confluent.kafka.schemaregistry.avro.AvroSchema) KsqlEntity(io.confluent.ksql.rest.entity.KsqlEntity) FunctionType(io.confluent.ksql.rest.entity.FunctionType) SourceDescriptionEntity(io.confluent.ksql.rest.entity.SourceDescriptionEntity) KsqlErrorMessageMatchers.errorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorMessage) TablesList(io.confluent.ksql.rest.entity.TablesList) DELETE_TOPIC_LIST_PROP(io.confluent.ksql.rest.entity.ClusterTerminateRequest.DELETE_TOPIC_LIST_PROP) Description(org.hamcrest.Description) CommandStatusEntity(io.confluent.ksql.rest.entity.CommandStatusEntity) Matchers(org.hamcrest.Matchers) KsqlRestExceptionMatchers.exceptionStatusCode(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatusCode) Property(io.confluent.ksql.rest.entity.PropertiesList.Property) Mockito.verify(org.mockito.Mockito.verify) Mockito(org.mockito.Mockito) Collections(java.util.Collections) TopicDeleteInjector(io.confluent.ksql.topic.TopicDeleteInjector)

Example 33 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class KsqlResourceTest method givenKsqlConfigWith.

private void givenKsqlConfigWith(final Map<String, Object> additionalConfig) {
    final Map<String, Object> config = ksqlRestConfig.getKsqlConfigProperties();
    config.putAll(additionalConfig);
    ksqlConfig = new KsqlConfig(config);
    final MetricCollectors metricCollectors = new MetricCollectors();
    ksqlEngine = KsqlEngineTestUtil.createKsqlEngine(serviceContext, metaStore, (engine) -> new KsqlEngineMetrics("", engine, Collections.emptyMap(), Optional.empty(), metricCollectors), new SequentialQueryIdGenerator(), ksqlConfig, metricCollectors);
    setUpKsqlResource();
}
Also used : ClusterTerminateRequest(io.confluent.ksql.rest.entity.ClusterTerminateRequest) DROP(io.confluent.ksql.rest.entity.CommandId.Action.DROP) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) ColumnName(io.confluent.ksql.name.ColumnName) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) KsqlRestExceptionMatchers.exceptionStatementErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatementErrorMessage) Mockito.doThrow(org.mockito.Mockito.doThrow) TransientQueryMetadata(io.confluent.ksql.util.TransientQueryMetadata) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Duration(java.time.Duration) Map(java.util.Map) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CommandStatus(io.confluent.ksql.rest.entity.CommandStatus) SystemColumns(io.confluent.ksql.schema.ksql.SystemColumns) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) EXECUTE(io.confluent.ksql.rest.entity.CommandId.Action.EXECUTE) ActivenessRegistrar(io.confluent.ksql.version.metrics.ActivenessRegistrar) KsqlRestConfig(io.confluent.ksql.rest.server.KsqlRestConfig) QueryDescription(io.confluent.ksql.rest.entity.QueryDescription) Metrics(org.apache.kafka.common.metrics.Metrics) SERVICE_UNAVAILABLE(io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE) KsqlException(io.confluent.ksql.util.KsqlException) FunctionCategory(io.confluent.ksql.function.FunctionCategory) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) Mockito.mock(org.mockito.Mockito.mock) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) TOPIC(io.confluent.ksql.rest.entity.CommandId.Type.TOPIC) RunWith(org.junit.runner.RunWith) KsqlIdentifierTestUtil(io.confluent.ksql.test.util.KsqlIdentifierTestUtil) KsqlSecurityContext(io.confluent.ksql.security.KsqlSecurityContext) Supplier(java.util.function.Supplier) SessionConfig(io.confluent.ksql.config.SessionConfig) Answer(org.mockito.stubbing.Answer) QueryStatusCount(io.confluent.ksql.rest.entity.QueryStatusCount) Injector(io.confluent.ksql.statement.Injector) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) UserFunctionLoader(io.confluent.ksql.function.UserFunctionLoader) CoreMatchers.nullValue(org.hamcrest.CoreMatchers.nullValue) Before(org.junit.Before) Producer(org.apache.kafka.clients.producer.Producer) SourceDescriptionFactory(io.confluent.ksql.rest.entity.SourceDescriptionFactory) SandboxedServiceContext(io.confluent.ksql.services.SandboxedServiceContext) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) KsqlStream(io.confluent.ksql.metastore.model.KsqlStream) ValueFormat(io.confluent.ksql.serde.ValueFormat) ConnectServerErrors(io.confluent.ksql.rest.server.execution.ConnectServerErrors) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) Mockito.never(org.mockito.Mockito.never) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Matcher(org.hamcrest.Matcher) DefaultErrorMessages(io.confluent.ksql.rest.DefaultErrorMessages) Mockito.reset(org.mockito.Mockito.reset) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) ParserMatchers.configured(io.confluent.ksql.parser.ParserMatchers.configured) KsqlEngineTestUtil(io.confluent.ksql.engine.KsqlEngineTestUtil) ParserMatchers.preparedStatementText(io.confluent.ksql.parser.ParserMatchers.preparedStatementText) SimpleFunctionInfo(io.confluent.ksql.rest.entity.SimpleFunctionInfo) StringLiteral(io.confluent.ksql.execution.expression.tree.StringLiteral) KsqlAuthorizationValidator(io.confluent.ksql.security.KsqlAuthorizationValidator) MockitoHamcrest.argThat(org.mockito.hamcrest.MockitoHamcrest.argThat) FakeKafkaTopicClient(io.confluent.ksql.services.FakeKafkaTopicClient) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) QueryDescriptionList(io.confluent.ksql.rest.entity.QueryDescriptionList) ImmutableSet(com.google.common.collect.ImmutableSet) Command(io.confluent.ksql.rest.server.computation.Command) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TypeSafeMatcher(org.hamcrest.TypeSafeMatcher) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Statement(io.confluent.ksql.parser.tree.Statement) KsqlConstants(io.confluent.ksql.util.KsqlConstants) Assert.assertThrows(org.junit.Assert.assertThrows) KsqlEntityList(io.confluent.ksql.rest.entity.KsqlEntityList) DataSourceType(io.confluent.ksql.metastore.model.DataSource.DataSourceType) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) Function(java.util.function.Function) CommandStatusFuture(io.confluent.ksql.rest.server.computation.CommandStatusFuture) BAD_REQUEST(io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST) Sandbox(io.confluent.ksql.util.Sandbox) ImmutableList(com.google.common.collect.ImmutableList) ConfiguredKsqlPlan(io.confluent.ksql.planner.plan.ConfiguredKsqlPlan) TerminateCluster(io.confluent.ksql.rest.util.TerminateCluster) Queries(io.confluent.ksql.rest.entity.Queries) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) CoreMatchers.hasItems(org.hamcrest.CoreMatchers.hasItems) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) EndpointResponse(io.confluent.ksql.rest.EndpointResponse) Mockito.when(org.mockito.Mockito.when) TABLE(io.confluent.ksql.rest.entity.CommandId.Type.TABLE) SourceInfo(io.confluent.ksql.rest.entity.SourceInfo) TopicDeleteInjector(io.confluent.ksql.topic.TopicDeleteInjector) KsqlStatementErrorMessage(io.confluent.ksql.rest.entity.KsqlStatementErrorMessage) DataSource(io.confluent.ksql.metastore.model.DataSource) StreamsList(io.confluent.ksql.rest.entity.StreamsList) SourceName(io.confluent.ksql.name.SourceName) RunningQuery(io.confluent.ksql.rest.entity.RunningQuery) Matchers.not(org.hamcrest.Matchers.not) KsqlTopicAuthorizationException(io.confluent.ksql.exception.KsqlTopicAuthorizationException) SourceDescription(io.confluent.ksql.rest.entity.SourceDescription) QueuedCommandStatus(io.confluent.ksql.rest.server.computation.QueuedCommandStatus) InjectorChain(io.confluent.ksql.statement.InjectorChain) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) Mockito.doReturn(org.mockito.Mockito.doReturn) CREATE(io.confluent.ksql.rest.entity.CommandId.Action.CREATE) QueryMetadata(io.confluent.ksql.util.QueryMetadata) FakeKafkaConsumerGroupClient(io.confluent.ksql.services.FakeKafkaConsumerGroupClient) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) MockedStatic(org.mockito.MockedStatic) Matchers.contains(org.hamcrest.Matchers.contains) KsqlExecutionContext(io.confluent.ksql.KsqlExecutionContext) Matchers.is(org.hamcrest.Matchers.is) OK(io.netty.handler.codec.http.HttpResponseStatus.OK) FormatOptions(io.confluent.ksql.schema.utils.FormatOptions) FormatFactory(io.confluent.ksql.serde.FormatFactory) Mock(org.mockito.Mock) KsqlRestExceptionMatchers.exceptionErrorMessage(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionErrorMessage) CreateStream(io.confluent.ksql.parser.tree.CreateStream) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DropSourceCommand(io.confluent.ksql.execution.ddl.commands.DropSourceCommand) Properties(java.util.Properties) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) AclOperation(org.apache.kafka.common.acl.AclOperation) File(java.io.File) KsqlConfigTestUtil(io.confluent.ksql.KsqlConfigTestUtil) TableElements(io.confluent.ksql.parser.tree.TableElements) ERROR_CODE_FORBIDDEN_KAFKA_ACCESS(io.confluent.ksql.rest.Errors.ERROR_CODE_FORBIDDEN_KAFKA_ACCESS) SqlTypes(io.confluent.ksql.schema.ksql.types.SqlTypes) PreparedStatement(io.confluent.ksql.parser.KsqlParser.PreparedStatement) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) KsqlStatementErrorMessageMatchers.statement(io.confluent.ksql.rest.entity.KsqlStatementErrorMessageMatchers.statement) CoreMatchers(org.hamcrest.CoreMatchers) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) KsqlPlan(io.confluent.ksql.engine.KsqlPlan) FunctionNameList(io.confluent.ksql.rest.entity.FunctionNameList) ServiceContext(io.confluent.ksql.services.ServiceContext) TimeoutException(java.util.concurrent.TimeoutException) CommandId(io.confluent.ksql.rest.entity.CommandId) TableElement(io.confluent.ksql.parser.tree.TableElement) DenyListPropertyValidator(io.confluent.ksql.properties.DenyListPropertyValidator) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) After(org.junit.After) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) QueryLogger(io.confluent.ksql.logging.query.QueryLogger) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) EntityUtil(io.confluent.ksql.rest.util.EntityUtil) SourceDescriptionList(io.confluent.ksql.rest.entity.SourceDescriptionList) KsqlTable(io.confluent.ksql.metastore.model.KsqlTable) PropertiesList(io.confluent.ksql.rest.entity.PropertiesList) ImmutableMap(com.google.common.collect.ImmutableMap) Errors(io.confluent.ksql.rest.Errors) KsqlConfig(io.confluent.ksql.util.KsqlConfig) QueryDescriptionFactory(io.confluent.ksql.rest.entity.QueryDescriptionFactory) TestServiceContext(io.confluent.ksql.services.TestServiceContext) List(java.util.List) KsqlErrorMessageMatchers.errorCode(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorCode) KsqlEngineMetrics(io.confluent.ksql.internal.KsqlEngineMetrics) Optional(java.util.Optional) FormatInfo(io.confluent.ksql.serde.FormatInfo) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) StreamsConfig(org.apache.kafka.streams.StreamsConfig) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) FORBIDDEN(io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN) INTERNAL_SERVER_ERROR(io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR) KeyFormat(io.confluent.ksql.serde.KeyFormat) STREAM(io.confluent.ksql.rest.entity.CommandId.Type.STREAM) QueryDescriptionEntity(io.confluent.ksql.rest.entity.QueryDescriptionEntity) HashMap(java.util.HashMap) AvroSchema(io.confluent.kafka.schemaregistry.avro.AvroSchema) KsqlEntity(io.confluent.ksql.rest.entity.KsqlEntity) FunctionType(io.confluent.ksql.rest.entity.FunctionType) SourceDescriptionEntity(io.confluent.ksql.rest.entity.SourceDescriptionEntity) KsqlErrorMessageMatchers.errorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessageMatchers.errorMessage) TablesList(io.confluent.ksql.rest.entity.TablesList) DELETE_TOPIC_LIST_PROP(io.confluent.ksql.rest.entity.ClusterTerminateRequest.DELETE_TOPIC_LIST_PROP) Description(org.hamcrest.Description) CommandStatusEntity(io.confluent.ksql.rest.entity.CommandStatusEntity) Matchers(org.hamcrest.Matchers) KsqlRestExceptionMatchers.exceptionStatusCode(io.confluent.ksql.rest.server.resources.KsqlRestExceptionMatchers.exceptionStatusCode) Property(io.confluent.ksql.rest.entity.PropertiesList.Property) Mockito.verify(org.mockito.Mockito.verify) Mockito(org.mockito.Mockito) Collections(java.util.Collections) KsqlEngineMetrics(io.confluent.ksql.internal.KsqlEngineMetrics) MetricCollectors(io.confluent.ksql.metrics.MetricCollectors) KsqlConfig(io.confluent.ksql.util.KsqlConfig) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString)

Example 34 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class KsqlRestApplication method buildApplication.

@SuppressWarnings({ "checkstyle:JavaNCSS", "checkstyle:MethodLength", "checkstyle:ParameterNumber" })
static KsqlRestApplication buildApplication(final String metricsPrefix, final KsqlRestConfig restConfig, final Function<Supplier<Boolean>, VersionCheckerAgent> versionCheckerFactory, final int maxStatementRetries, final ServiceContext serviceContext, final Supplier<SchemaRegistryClient> schemaRegistryClientFactory, final ConnectClientFactory connectClientFactory, final Vertx vertx, final KsqlClient sharedClient, final DefaultServiceContextFactory defaultServiceContextFactory, final UserServiceContextFactory userServiceContextFactory, final MetricCollectors metricCollectors) {
    final String ksqlInstallDir = restConfig.getString(KsqlRestConfig.INSTALL_DIR_CONFIG);
    final KsqlConfig ksqlConfig = new KsqlConfig(restConfig.getKsqlConfigProperties());
    final ProcessingLogConfig processingLogConfig = new ProcessingLogConfig(restConfig.getOriginals());
    final ProcessingLogContext processingLogContext = ProcessingLogContext.create(processingLogConfig);
    final MutableFunctionRegistry functionRegistry = new InternalFunctionRegistry();
    if (restConfig.getBoolean(KsqlRestConfig.KSQL_SERVER_ENABLE_UNCAUGHT_EXCEPTION_HANDLER)) {
        Thread.setDefaultUncaughtExceptionHandler(new KsqlUncaughtExceptionHandler(LogManager::shutdown));
    }
    final SpecificQueryIdGenerator specificQueryIdGenerator = new SpecificQueryIdGenerator();
    final String stateDir = ksqlConfig.getKsqlStreamConfigProps().getOrDefault(StreamsConfig.STATE_DIR_CONFIG, StreamsConfig.configDef().defaultValues().get(StreamsConfig.STATE_DIR_CONFIG)).toString();
    final ServiceInfo serviceInfo = ServiceInfo.create(ksqlConfig, metricsPrefix);
    final Map<String, String> metricsTags = ImmutableMap.<String, String>builder().putAll(serviceInfo.customMetricsTags()).put(KsqlConstants.KSQL_SERVICE_ID_METRICS_TAG, serviceInfo.serviceId()).build();
    StorageUtilizationMetricsReporter.configureShared(new File(stateDir), metricCollectors.getMetrics(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("ksql-csu-metrics-reporter-%d").build());
    final ScheduledExecutorService leakedResourcesReporter = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder().setNameFormat("ksql-leaked-resources-metrics-reporter-%d").build());
    final KsqlEngine ksqlEngine = new KsqlEngine(serviceContext, processingLogContext, functionRegistry, serviceInfo, specificQueryIdGenerator, new KsqlConfig(restConfig.getKsqlConfigProperties()), Collections.emptyList(), metricCollectors);
    final PersistentQuerySaturationMetrics saturation = new PersistentQuerySaturationMetrics(ksqlEngine, new JmxDataPointsReporter(metricCollectors.getMetrics(), "ksqldb_utilization", Duration.ofMinutes(1)), Duration.ofMinutes(5), Duration.ofSeconds(30), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    executorService.scheduleAtFixedRate(saturation, 0, Duration.ofMinutes(1).toMillis(), TimeUnit.MILLISECONDS);
    final int transientQueryCleanupServicePeriod = ksqlConfig.getInt(KsqlConfig.KSQL_TRANSIENT_QUERY_CLEANUP_SERVICE_PERIOD_SECONDS);
    final LeakedResourcesMetrics leaked = new LeakedResourcesMetrics(ksqlEngine, new JmxDataPointsReporter(metricCollectors.getMetrics(), ReservedInternalTopics.KSQL_INTERNAL_TOPIC_PREFIX + ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG) + ".leaked_resources_metrics", Duration.ofSeconds(transientQueryCleanupServicePeriod)), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS));
    leakedResourcesReporter.scheduleAtFixedRate(leaked, 0, transientQueryCleanupServicePeriod, TimeUnit.SECONDS);
    UserFunctionLoader.newInstance(ksqlConfig, functionRegistry, ksqlInstallDir, metricCollectors.getMetrics()).load();
    final String commandTopicName = ReservedInternalTopics.commandTopic(ksqlConfig);
    final Admin internalAdmin = createCommandTopicAdminClient(restConfig, ksqlConfig);
    final KafkaTopicClient internalTopicClient = new KafkaTopicClientImpl(() -> internalAdmin);
    final CommandStore commandStore = CommandStore.Factory.create(ksqlConfig, commandTopicName, Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), ksqlConfig.addConfluentMetricsContextConfigsKafka(restConfig.getCommandConsumerProperties()), ksqlConfig.addConfluentMetricsContextConfigsKafka(restConfig.getCommandProducerProperties()), internalTopicClient);
    final InteractiveStatementExecutor statementExecutor = new InteractiveStatementExecutor(serviceContext, ksqlEngine, specificQueryIdGenerator);
    final StatusResource statusResource = new StatusResource(statementExecutor);
    final VersionCheckerAgent versionChecker = versionCheckerFactory.apply(ksqlEngine::hasActiveQueries);
    final ServerState serverState = new ServerState();
    final KsqlSecurityExtension securityExtension = loadSecurityExtension(ksqlConfig);
    final KsqlSecurityContextProvider ksqlSecurityContextProvider = new DefaultKsqlSecurityContextProvider(securityExtension, defaultServiceContextFactory, userServiceContextFactory, ksqlConfig, schemaRegistryClientFactory, connectClientFactory, sharedClient);
    final Optional<AuthenticationPlugin> securityHandlerPlugin = loadAuthenticationPlugin(restConfig);
    final Optional<KsqlAuthorizationValidator> authorizationValidator = KsqlAuthorizationValidatorFactory.create(ksqlConfig, serviceContext, securityExtension.getAuthorizationProvider());
    final Errors errorHandler = new Errors(restConfig.getConfiguredInstance(KsqlRestConfig.KSQL_SERVER_ERROR_MESSAGES, ErrorMessages.class));
    final ConnectServerErrors connectErrorHandler = loadConnectErrorHandler(ksqlConfig);
    final Optional<LagReportingAgent> lagReportingAgent = initializeLagReportingAgent(restConfig, ksqlEngine, serviceContext);
    final Optional<HeartbeatAgent> heartbeatAgent = initializeHeartbeatAgent(restConfig, ksqlEngine, serviceContext, lagReportingAgent);
    final RoutingFilterFactory routingFilterFactory = initializeRoutingFilterFactory(ksqlConfig, heartbeatAgent, lagReportingAgent);
    final RateLimiter pullQueryRateLimiter = new RateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_QPS_CONFIG), "pull", metricCollectors.getMetrics(), metricsTags);
    final ConcurrencyLimiter pullQueryConcurrencyLimiter = new ConcurrencyLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_CONCURRENT_REQUESTS_CONFIG), "pull", metricCollectors.getMetrics(), metricsTags);
    final SlidingWindowRateLimiter pullBandRateLimiter = new SlidingWindowRateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PULL_MAX_HOURLY_BANDWIDTH_MEGABYTES_CONFIG), NUM_MILLISECONDS_IN_HOUR, "pull", metricCollectors.getMetrics(), metricsTags);
    final SlidingWindowRateLimiter scalablePushBandRateLimiter = new SlidingWindowRateLimiter(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_HOURLY_BANDWIDTH_MEGABYTES_CONFIG), NUM_MILLISECONDS_IN_HOUR, "push", metricCollectors.getMetrics(), metricsTags);
    final DenyListPropertyValidator denyListPropertyValidator = new DenyListPropertyValidator(ksqlConfig.getList(KsqlConfig.KSQL_PROPERTIES_OVERRIDES_DENYLIST));
    final Optional<PullQueryExecutorMetrics> pullQueryMetrics = ksqlConfig.getBoolean(KsqlConfig.KSQL_QUERY_PULL_METRICS_ENABLED) ? Optional.of(new PullQueryExecutorMetrics(ksqlEngine.getServiceId(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS), Time.SYSTEM, metricCollectors.getMetrics())) : Optional.empty();
    final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics = ksqlConfig.getBoolean(KsqlConfig.KSQL_QUERY_PUSH_V2_ENABLED) ? Optional.of(new ScalablePushQueryMetrics(ksqlEngine.getServiceId(), ksqlConfig.getStringAsMap(KsqlConfig.KSQL_CUSTOM_METRICS_TAGS), Time.SYSTEM, metricCollectors.getMetrics())) : Optional.empty();
    final HARouting pullQueryRouting = new HARouting(routingFilterFactory, pullQueryMetrics, ksqlConfig);
    final PushRouting pushQueryRouting = new PushRouting();
    final Optional<LocalCommands> localCommands = createLocalCommands(restConfig, ksqlEngine);
    final QueryExecutor queryExecutor = new QueryExecutor(ksqlEngine, restConfig, ksqlConfig, pullQueryMetrics, scalablePushQueryMetrics, pullQueryRateLimiter, pullQueryConcurrencyLimiter, pullBandRateLimiter, scalablePushBandRateLimiter, pullQueryRouting, pushQueryRouting, localCommands);
    final StreamedQueryResource streamedQueryResource = new StreamedQueryResource(ksqlEngine, restConfig, commandStore, Duration.ofMillis(restConfig.getLong(KsqlRestConfig.STREAMED_QUERY_DISCONNECT_CHECK_MS_CONFIG)), Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), versionChecker::updateLastRequestTime, authorizationValidator, errorHandler, denyListPropertyValidator, queryExecutor);
    final List<String> managedTopics = new LinkedList<>();
    managedTopics.add(commandTopicName);
    if (processingLogConfig.getBoolean(ProcessingLogConfig.TOPIC_AUTO_CREATE)) {
        managedTopics.add(ProcessingLogServerUtils.getTopicName(processingLogConfig, ksqlConfig));
    }
    final CommandRunner commandRunner = new CommandRunner(statementExecutor, commandStore, maxStatementRetries, new ClusterTerminator(ksqlEngine, serviceContext, managedTopics), serverState, ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG), Duration.ofMillis(restConfig.getLong(KsqlRestConfig.KSQL_COMMAND_RUNNER_BLOCKED_THRESHHOLD_ERROR_MS)), metricsPrefix, InternalTopicSerdes.deserializer(Command.class), errorHandler, internalTopicClient, commandTopicName, metricCollectors.getMetrics());
    final KsqlResource ksqlResource = new KsqlResource(ksqlEngine, commandRunner, Duration.ofMillis(restConfig.getLong(DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG)), versionChecker::updateLastRequestTime, authorizationValidator, errorHandler, connectErrorHandler, denyListPropertyValidator);
    final List<KsqlServerPrecondition> preconditions = restConfig.getConfiguredInstances(KsqlRestConfig.KSQL_SERVER_PRECONDITIONS, KsqlServerPrecondition.class);
    final List<KsqlConfigurable> configurables = ImmutableList.of(ksqlResource, streamedQueryResource, statementExecutor);
    final Consumer<KsqlConfig> rocksDBConfigSetterHandler = RocksDBConfigSetterHandler::maybeConfigureRocksDBConfigSetter;
    return new KsqlRestApplication(serviceContext, ksqlEngine, ksqlConfig, restConfig, commandRunner, commandStore, statusResource, streamedQueryResource, ksqlResource, versionChecker, ksqlSecurityContextProvider, securityExtension, securityHandlerPlugin, serverState, processingLogContext, preconditions, configurables, rocksDBConfigSetterHandler, heartbeatAgent, lagReportingAgent, vertx, denyListPropertyValidator, pullQueryMetrics, scalablePushQueryMetrics, localCommands, queryExecutor, metricCollectors, internalTopicClient, internalAdmin);
}
Also used : KsqlEngine(io.confluent.ksql.engine.KsqlEngine) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) JmxDataPointsReporter(io.confluent.ksql.internal.JmxDataPointsReporter) ProcessingLogConfig(io.confluent.ksql.logging.processing.ProcessingLogConfig) StreamedQueryResource(io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource) ServiceInfo(io.confluent.ksql.ServiceInfo) QueryExecutor(io.confluent.ksql.rest.server.query.QueryExecutor) SpecificQueryIdGenerator(io.confluent.ksql.query.id.SpecificQueryIdGenerator) DefaultConnectServerErrors(io.confluent.ksql.rest.server.execution.DefaultConnectServerErrors) ConnectServerErrors(io.confluent.ksql.rest.server.execution.ConnectServerErrors) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) KafkaTopicClientImpl(io.confluent.ksql.services.KafkaTopicClientImpl) ConcurrencyLimiter(io.confluent.ksql.rest.util.ConcurrencyLimiter) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ListeningScheduledExecutorService(com.google.common.util.concurrent.ListeningScheduledExecutorService) KsqlAuthorizationValidator(io.confluent.ksql.security.KsqlAuthorizationValidator) KsqlConfig(io.confluent.ksql.util.KsqlConfig) KsqlVersionCheckerAgent(io.confluent.ksql.version.metrics.KsqlVersionCheckerAgent) VersionCheckerAgent(io.confluent.ksql.version.metrics.VersionCheckerAgent) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RateLimiter(io.confluent.ksql.rest.util.RateLimiter) LinkedList(java.util.LinkedList) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) ErrorMessages(io.confluent.ksql.rest.ErrorMessages) DefaultConnectServerErrors(io.confluent.ksql.rest.server.execution.DefaultConnectServerErrors) ConnectServerErrors(io.confluent.ksql.rest.server.execution.ConnectServerErrors) Errors(io.confluent.ksql.rest.Errors) HARouting(io.confluent.ksql.physical.pull.HARouting) DefaultKsqlSecurityContextProvider(io.confluent.ksql.api.impl.DefaultKsqlSecurityContextProvider) DenyListPropertyValidator(io.confluent.ksql.properties.DenyListPropertyValidator) PushRouting(io.confluent.ksql.physical.scalablepush.PushRouting) File(java.io.File) InteractiveStatementExecutor(io.confluent.ksql.rest.server.computation.InteractiveStatementExecutor) KsqlResource(io.confluent.ksql.rest.server.resources.KsqlResource) KsqlSecurityContextProvider(io.confluent.ksql.api.impl.KsqlSecurityContextProvider) DefaultKsqlSecurityContextProvider(io.confluent.ksql.api.impl.DefaultKsqlSecurityContextProvider) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) StatusResource(io.confluent.ksql.rest.server.resources.StatusResource) ClusterStatusResource(io.confluent.ksql.rest.server.resources.ClusterStatusResource) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) PersistentQuerySaturationMetrics(io.confluent.ksql.utilization.PersistentQuerySaturationMetrics) ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) LeakedResourcesMetrics(io.confluent.ksql.internal.LeakedResourcesMetrics) ProcessingLogContext(io.confluent.ksql.logging.processing.ProcessingLogContext) ClusterTerminator(io.confluent.ksql.rest.util.ClusterTerminator) ServerState(io.confluent.ksql.rest.server.state.ServerState) KsqlSecurityExtension(io.confluent.ksql.security.KsqlSecurityExtension) Admin(org.apache.kafka.clients.admin.Admin) WSQueryEndpoint(io.confluent.ksql.rest.server.resources.streaming.WSQueryEndpoint) KsqlConfigurable(io.confluent.ksql.rest.server.resources.KsqlConfigurable) KsqlUncaughtExceptionHandler(io.confluent.ksql.rest.util.KsqlUncaughtExceptionHandler) KafkaTopicClient(io.confluent.ksql.services.KafkaTopicClient) Command(io.confluent.ksql.rest.server.computation.Command) AuthenticationPlugin(io.confluent.ksql.api.auth.AuthenticationPlugin) SlidingWindowRateLimiter(io.confluent.ksql.api.server.SlidingWindowRateLimiter) RoutingFilterFactory(io.confluent.ksql.execution.streams.RoutingFilter.RoutingFilterFactory) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry)

Example 35 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class StandaloneExecutorFactory method create.

@VisibleForTesting
static StandaloneExecutor create(final Map<String, Object> properties, final String queriesFile, final String installDir, final Function<KsqlConfig, ServiceContext> serviceContextFactory, final BiFunction<String, KsqlConfig, ConfigStore> configStoreFactory, final Function<Supplier<Boolean>, VersionCheckerAgent> versionCheckerFactory, final StandaloneExecutorConstructor constructor, final MetricCollectors metricCollectors) {
    final KsqlConfig baseConfig = new KsqlConfig(properties);
    final ServiceContext serviceContext = serviceContextFactory.apply(baseConfig);
    final String configTopicName = ReservedInternalTopics.configsTopic(baseConfig);
    KsqlInternalTopicUtils.ensureTopic(configTopicName, baseConfig, serviceContext.getTopicClient());
    final ConfigStore configStore = configStoreFactory.apply(configTopicName, baseConfig);
    final KsqlConfig ksqlConfig = configStore.getKsqlConfig();
    final ProcessingLogConfig processingLogConfig = new ProcessingLogConfig(properties);
    final ProcessingLogContext processingLogContext = ProcessingLogContext.create(processingLogConfig);
    final MutableFunctionRegistry functionRegistry = new InternalFunctionRegistry();
    final KsqlEngine ksqlEngine = new KsqlEngine(serviceContext, processingLogContext, functionRegistry, ServiceInfo.create(ksqlConfig), new SequentialQueryIdGenerator(), ksqlConfig, Collections.emptyList(), metricCollectors);
    final UserFunctionLoader udfLoader = UserFunctionLoader.newInstance(ksqlConfig, functionRegistry, installDir, metricCollectors.getMetrics());
    final VersionCheckerAgent versionChecker = versionCheckerFactory.apply(ksqlEngine::hasActiveQueries);
    return constructor.create(serviceContext, processingLogConfig, ksqlConfig, ksqlEngine, queriesFile, udfLoader, true, versionChecker, Injectors.NO_TOPIC_DELETE, metricCollectors);
}
Also used : KsqlEngine(io.confluent.ksql.engine.KsqlEngine) ProcessingLogContext(io.confluent.ksql.logging.processing.ProcessingLogContext) MutableFunctionRegistry(io.confluent.ksql.function.MutableFunctionRegistry) ServiceContext(io.confluent.ksql.services.ServiceContext) UserFunctionLoader(io.confluent.ksql.function.UserFunctionLoader) ProcessingLogConfig(io.confluent.ksql.logging.processing.ProcessingLogConfig) KsqlConfig(io.confluent.ksql.util.KsqlConfig) KsqlVersionCheckerAgent(io.confluent.ksql.version.metrics.KsqlVersionCheckerAgent) VersionCheckerAgent(io.confluent.ksql.version.metrics.VersionCheckerAgent) KafkaConfigStore(io.confluent.ksql.rest.server.computation.KafkaConfigStore) ConfigStore(io.confluent.ksql.rest.server.computation.ConfigStore) SequentialQueryIdGenerator(io.confluent.ksql.query.id.SequentialQueryIdGenerator) InternalFunctionRegistry(io.confluent.ksql.function.InternalFunctionRegistry) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

KsqlEngine (io.confluent.ksql.engine.KsqlEngine)38 Test (org.junit.Test)25 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)24 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)22 InternalFunctionRegistry (io.confluent.ksql.function.InternalFunctionRegistry)14 SequentialQueryIdGenerator (io.confluent.ksql.query.id.SequentialQueryIdGenerator)14 KsqlConfig (io.confluent.ksql.util.KsqlConfig)14 MetricCollectors (io.confluent.ksql.metrics.MetricCollectors)13 ServiceContext (io.confluent.ksql.services.ServiceContext)13 MutableFunctionRegistry (io.confluent.ksql.function.MutableFunctionRegistry)11 KsqlEntity (io.confluent.ksql.rest.entity.KsqlEntity)11 KsqlHostInfoEntity (io.confluent.ksql.rest.entity.KsqlHostInfoEntity)11 Queries (io.confluent.ksql.rest.entity.Queries)11 List (java.util.List)11 MetaStoreImpl (io.confluent.ksql.metastore.MetaStoreImpl)10 MatcherAssert.assertThat (org.hamcrest.MatcherAssert.assertThat)10 Matchers.is (org.hamcrest.Matchers.is)10 ImmutableList (com.google.common.collect.ImmutableList)9 SessionConfig (io.confluent.ksql.config.SessionConfig)9 KsqlEngineMetrics (io.confluent.ksql.internal.KsqlEngineMetrics)9