Search in sources :

Example 1 with KsqlEngine

use of io.confluent.ksql.KsqlEngine in project ksql by confluentinc.

the class JsonFormatTest method before.

@Before
public void before() throws Exception {
    Map<String, Object> configMap = new HashMap<>();
    configMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    configMap.put("application.id", "KSQL");
    configMap.put("commit.interval.ms", 0);
    configMap.put("cache.max.bytes.buffering", 0);
    configMap.put("auto.offset.reset", "earliest");
    KsqlConfig ksqlConfig = new KsqlConfig(configMap);
    adminClient = AdminClient.create(ksqlConfig.getKsqlAdminClientConfigProps());
    topicClient = new KafkaTopicClientImpl(adminClient);
    ksqlEngine = new KsqlEngine(ksqlConfig, topicClient);
    metaStore = ksqlEngine.getMetaStore();
    createInitTopics();
    produceInitData();
    execInitCreateStreamQueries();
}
Also used : KsqlEngine(io.confluent.ksql.KsqlEngine) HashMap(java.util.HashMap) KsqlConfig(io.confluent.ksql.util.KsqlConfig) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl) Before(org.junit.Before)

Example 2 with KsqlEngine

use of io.confluent.ksql.KsqlEngine in project ksql by confluentinc.

the class KsqlRestApplication method buildApplication.

public static KsqlRestApplication buildApplication(KsqlRestConfig restConfig, boolean isUiEnabled, VersionCheckerAgent versionCheckerAgent) throws Exception {
    Map<String, Object> ksqlConfProperties = new HashMap<>();
    ksqlConfProperties.putAll(restConfig.getKsqlConfigProperties());
    KsqlConfig ksqlConfig = new KsqlConfig(ksqlConfProperties);
    adminClient = AdminClient.create(ksqlConfig.getKsqlAdminClientConfigProps());
    KsqlEngine ksqlEngine = new KsqlEngine(ksqlConfig, new KafkaTopicClientImpl(adminClient));
    KafkaTopicClient topicClient = ksqlEngine.getTopicClient();
    final String kafkaClusterId;
    try {
        kafkaClusterId = adminClient.describeCluster().clusterId().get();
    } catch (final UnsupportedVersionException e) {
        throw new KsqlException("The kafka brokers are incompatible with. " + "KSQL requires broker versions >= 0.10.1.x");
    }
    String commandTopic = restConfig.getCommandTopic(ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG));
    ensureCommandTopic(restConfig, topicClient, commandTopic);
    Map<String, Expression> commandTopicProperties = new HashMap<>();
    commandTopicProperties.put(DdlConfig.VALUE_FORMAT_PROPERTY, new StringLiteral("json"));
    commandTopicProperties.put(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY, new StringLiteral(commandTopic));
    ksqlEngine.getDdlCommandExec().execute(new RegisterTopicCommand(new RegisterTopic(QualifiedName.of(COMMANDS_KSQL_TOPIC_NAME), false, commandTopicProperties)));
    ksqlEngine.getDdlCommandExec().execute(new CreateStreamCommand("statementText", new CreateStream(QualifiedName.of(COMMANDS_STREAM_NAME), Collections.singletonList(new TableElement("STATEMENT", "STRING")), false, Collections.singletonMap(DdlConfig.TOPIC_NAME_PROPERTY, new StringLiteral(COMMANDS_KSQL_TOPIC_NAME))), Collections.emptyMap(), ksqlEngine.getTopicClient(), true));
    Map<String, Object> commandConsumerProperties = restConfig.getCommandConsumerProperties();
    KafkaConsumer<CommandId, Command> commandConsumer = new KafkaConsumer<>(commandConsumerProperties, getJsonDeserializer(CommandId.class, true), getJsonDeserializer(Command.class, false));
    KafkaProducer<CommandId, Command> commandProducer = new KafkaProducer<>(restConfig.getCommandProducerProperties(), getJsonSerializer(true), getJsonSerializer(false));
    CommandStore commandStore = new CommandStore(commandTopic, commandConsumer, commandProducer, new CommandIdAssigner(ksqlEngine.getMetaStore()));
    StatementParser statementParser = new StatementParser(ksqlEngine);
    StatementExecutor statementExecutor = new StatementExecutor(ksqlEngine, statementParser);
    CommandRunner commandRunner = new CommandRunner(statementExecutor, commandStore);
    RootDocument rootDocument = new RootDocument(isUiEnabled, restConfig.getList(RestConfig.LISTENERS_CONFIG).get(0));
    StatusResource statusResource = new StatusResource(statementExecutor);
    StreamedQueryResource streamedQueryResource = new StreamedQueryResource(ksqlEngine, statementParser, restConfig.getLong(KsqlRestConfig.STREAMED_QUERY_DISCONNECT_CHECK_MS_CONFIG));
    KsqlResource ksqlResource = new KsqlResource(ksqlEngine, commandStore, statementExecutor, restConfig.getLong(KsqlRestConfig.DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG));
    commandRunner.processPriorCommands();
    return new KsqlRestApplication(ksqlEngine, restConfig, commandRunner, rootDocument, statusResource, streamedQueryResource, ksqlResource, isUiEnabled, versionCheckerAgent, new ServerInfo(Version.getVersion(), kafkaClusterId));
}
Also used : KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) KsqlEngine(io.confluent.ksql.KsqlEngine) HashMap(java.util.HashMap) RegisterTopic(io.confluent.ksql.parser.tree.RegisterTopic) KsqlResource(io.confluent.ksql.rest.server.resources.KsqlResource) ServerInfo(io.confluent.ksql.rest.entity.ServerInfo) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) KsqlException(io.confluent.ksql.util.KsqlException) StatementExecutor(io.confluent.ksql.rest.server.computation.StatementExecutor) StatusResource(io.confluent.ksql.rest.server.resources.StatusResource) TableElement(io.confluent.ksql.parser.tree.TableElement) StreamedQueryResource(io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource) CreateStreamCommand(io.confluent.ksql.ddl.commands.CreateStreamCommand) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl) RootDocument(io.confluent.ksql.rest.server.resources.RootDocument) KsqlConfig(io.confluent.ksql.util.KsqlConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) RegisterTopicCommand(io.confluent.ksql.ddl.commands.RegisterTopicCommand) KafkaTopicClient(io.confluent.ksql.util.KafkaTopicClient) StringLiteral(io.confluent.ksql.parser.tree.StringLiteral) Expression(io.confluent.ksql.parser.tree.Expression) Command(io.confluent.ksql.rest.server.computation.Command) RegisterTopicCommand(io.confluent.ksql.ddl.commands.RegisterTopicCommand) CreateStreamCommand(io.confluent.ksql.ddl.commands.CreateStreamCommand) CommandIdAssigner(io.confluent.ksql.rest.server.computation.CommandIdAssigner) CommandId(io.confluent.ksql.rest.server.computation.CommandId) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException)

Example 3 with KsqlEngine

use of io.confluent.ksql.KsqlEngine in project ksql by confluentinc.

the class StatementExecutorTest method setUp.

@Before
public void setUp() {
    Map<String, Object> props = new HashMap<>();
    props.put("application.id", "ksqlStatementExecutorTest");
    props.put("bootstrap.servers", CLUSTER.bootstrapServers());
    ksqlEngine = new KsqlEngine(new KsqlConfig(props), new MockKafkaTopicClient());
    StatementParser statementParser = new StatementParser(ksqlEngine);
    statementExecutor = new StatementExecutor(ksqlEngine, statementParser);
}
Also used : KsqlEngine(io.confluent.ksql.KsqlEngine) StatementParser(io.confluent.ksql.rest.server.StatementParser) HashMap(java.util.HashMap) MockKafkaTopicClient(io.confluent.ksql.rest.server.mock.MockKafkaTopicClient) KsqlConfig(io.confluent.ksql.util.KsqlConfig) Before(org.junit.Before)

Example 4 with KsqlEngine

use of io.confluent.ksql.KsqlEngine in project ksql by confluentinc.

the class EndToEndIntegrationTest method before.

@Before
public void before() throws Exception {
    testHarness = new IntegrationTestHarness();
    testHarness.start();
    Map<String, Object> streamsConfig = testHarness.ksqlConfig.getKsqlStreamConfigProps();
    streamsConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    KsqlConfig ksqlconfig = new KsqlConfig(streamsConfig);
    AdminClient adminClient = AdminClient.create(ksqlconfig.getKsqlAdminClientConfigProps());
    KafkaTopicClient topicClient = new KafkaTopicClientImpl(adminClient);
    ksqlEngine = new KsqlEngine(ksqlconfig, topicClient);
    testHarness.createTopic(pageViewTopic);
    testHarness.createTopic(usersTopic);
    pageViewDataProvider = new PageViewDataProvider();
    testHarness.publishTestData(usersTopic, new UserDataProvider(), System.currentTimeMillis() - 10000);
    testHarness.publishTestData(pageViewTopic, pageViewDataProvider, System.currentTimeMillis());
    ksqlEngine.buildMultipleQueries(format("CREATE TABLE %s (registertime bigint, gender varchar, regionid varchar, " + "userid varchar) WITH (kafka_topic='%s', value_format='JSON', key = 'userid');", userTable, usersTopic), Collections.emptyMap());
    ksqlEngine.buildMultipleQueries(format("CREATE STREAM %s (viewtime bigint, userid varchar, pageid varchar) " + "WITH (kafka_topic='%s', value_format='JSON');", pageViewStream, pageViewTopic), Collections.emptyMap());
}
Also used : KsqlEngine(io.confluent.ksql.KsqlEngine) UserDataProvider(io.confluent.ksql.util.UserDataProvider) KafkaTopicClient(io.confluent.ksql.util.KafkaTopicClient) PageViewDataProvider(io.confluent.ksql.util.PageViewDataProvider) KsqlConfig(io.confluent.ksql.util.KsqlConfig) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl) AdminClient(org.apache.kafka.clients.admin.AdminClient) Before(org.junit.Before)

Example 5 with KsqlEngine

use of io.confluent.ksql.KsqlEngine in project ksql by confluentinc.

the class SecureIntegrationTest method givenTestSetupWithConfig.

private void givenTestSetupWithConfig(final Map<String, Object> ksqlConfigs) throws Exception {
    final KsqlConfig ksqlConfig = new KsqlConfig(ksqlConfigs);
    ksqlEngine = new KsqlEngine(ksqlConfig, new KafkaTopicClientImpl(AdminClient.create(ksqlConfig.getKsqlAdminClientConfigProps())));
    execInitCreateStreamQueries();
}
Also used : KsqlEngine(io.confluent.ksql.KsqlEngine) KsqlConfig(io.confluent.ksql.util.KsqlConfig) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl)

Aggregations

KsqlEngine (io.confluent.ksql.KsqlEngine)8 KsqlConfig (io.confluent.ksql.util.KsqlConfig)7 KafkaTopicClientImpl (io.confluent.ksql.util.KafkaTopicClientImpl)6 Before (org.junit.Before)4 KafkaTopicClient (io.confluent.ksql.util.KafkaTopicClient)3 HashMap (java.util.HashMap)3 StatementParser (io.confluent.ksql.rest.server.StatementParser)2 MockKafkaTopicClient (io.confluent.ksql.rest.server.mock.MockKafkaTopicClient)2 StreamedQueryResource (io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 MockSchemaRegistryClient (io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient)1 SchemaRegistryClient (io.confluent.kafka.schemaregistry.client.SchemaRegistryClient)1 GenericRow (io.confluent.ksql.GenericRow)1 CreateStreamCommand (io.confluent.ksql.ddl.commands.CreateStreamCommand)1 RegisterTopicCommand (io.confluent.ksql.ddl.commands.RegisterTopicCommand)1 MetaStoreImpl (io.confluent.ksql.metastore.MetaStoreImpl)1 CreateStream (io.confluent.ksql.parser.tree.CreateStream)1 Expression (io.confluent.ksql.parser.tree.Expression)1 Query (io.confluent.ksql.parser.tree.Query)1 RegisterTopic (io.confluent.ksql.parser.tree.RegisterTopic)1