Search in sources :

Example 1 with StreamedQueryResource

use of io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource in project ksql by confluentinc.

the class KsqlRestApplication method buildApplication.

public static KsqlRestApplication buildApplication(KsqlRestConfig restConfig, boolean isUiEnabled, VersionCheckerAgent versionCheckerAgent) throws Exception {
    Map<String, Object> ksqlConfProperties = new HashMap<>();
    ksqlConfProperties.putAll(restConfig.getKsqlConfigProperties());
    KsqlConfig ksqlConfig = new KsqlConfig(ksqlConfProperties);
    adminClient = AdminClient.create(ksqlConfig.getKsqlAdminClientConfigProps());
    KsqlEngine ksqlEngine = new KsqlEngine(ksqlConfig, new KafkaTopicClientImpl(adminClient));
    KafkaTopicClient topicClient = ksqlEngine.getTopicClient();
    final String kafkaClusterId;
    try {
        kafkaClusterId = adminClient.describeCluster().clusterId().get();
    } catch (final UnsupportedVersionException e) {
        throw new KsqlException("The kafka brokers are incompatible with. " + "KSQL requires broker versions >= 0.10.1.x");
    }
    String commandTopic = restConfig.getCommandTopic(ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG));
    ensureCommandTopic(restConfig, topicClient, commandTopic);
    Map<String, Expression> commandTopicProperties = new HashMap<>();
    commandTopicProperties.put(DdlConfig.VALUE_FORMAT_PROPERTY, new StringLiteral("json"));
    commandTopicProperties.put(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY, new StringLiteral(commandTopic));
    ksqlEngine.getDdlCommandExec().execute(new RegisterTopicCommand(new RegisterTopic(QualifiedName.of(COMMANDS_KSQL_TOPIC_NAME), false, commandTopicProperties)));
    ksqlEngine.getDdlCommandExec().execute(new CreateStreamCommand("statementText", new CreateStream(QualifiedName.of(COMMANDS_STREAM_NAME), Collections.singletonList(new TableElement("STATEMENT", "STRING")), false, Collections.singletonMap(DdlConfig.TOPIC_NAME_PROPERTY, new StringLiteral(COMMANDS_KSQL_TOPIC_NAME))), Collections.emptyMap(), ksqlEngine.getTopicClient(), true));
    Map<String, Object> commandConsumerProperties = restConfig.getCommandConsumerProperties();
    KafkaConsumer<CommandId, Command> commandConsumer = new KafkaConsumer<>(commandConsumerProperties, getJsonDeserializer(CommandId.class, true), getJsonDeserializer(Command.class, false));
    KafkaProducer<CommandId, Command> commandProducer = new KafkaProducer<>(restConfig.getCommandProducerProperties(), getJsonSerializer(true), getJsonSerializer(false));
    CommandStore commandStore = new CommandStore(commandTopic, commandConsumer, commandProducer, new CommandIdAssigner(ksqlEngine.getMetaStore()));
    StatementParser statementParser = new StatementParser(ksqlEngine);
    StatementExecutor statementExecutor = new StatementExecutor(ksqlEngine, statementParser);
    CommandRunner commandRunner = new CommandRunner(statementExecutor, commandStore);
    RootDocument rootDocument = new RootDocument(isUiEnabled, restConfig.getList(RestConfig.LISTENERS_CONFIG).get(0));
    StatusResource statusResource = new StatusResource(statementExecutor);
    StreamedQueryResource streamedQueryResource = new StreamedQueryResource(ksqlEngine, statementParser, restConfig.getLong(KsqlRestConfig.STREAMED_QUERY_DISCONNECT_CHECK_MS_CONFIG));
    KsqlResource ksqlResource = new KsqlResource(ksqlEngine, commandStore, statementExecutor, restConfig.getLong(KsqlRestConfig.DISTRIBUTED_COMMAND_RESPONSE_TIMEOUT_MS_CONFIG));
    commandRunner.processPriorCommands();
    return new KsqlRestApplication(ksqlEngine, restConfig, commandRunner, rootDocument, statusResource, streamedQueryResource, ksqlResource, isUiEnabled, versionCheckerAgent, new ServerInfo(Version.getVersion(), kafkaClusterId));
}
Also used : KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) KsqlEngine(io.confluent.ksql.KsqlEngine) HashMap(java.util.HashMap) RegisterTopic(io.confluent.ksql.parser.tree.RegisterTopic) KsqlResource(io.confluent.ksql.rest.server.resources.KsqlResource) ServerInfo(io.confluent.ksql.rest.entity.ServerInfo) CommandStore(io.confluent.ksql.rest.server.computation.CommandStore) KsqlException(io.confluent.ksql.util.KsqlException) StatementExecutor(io.confluent.ksql.rest.server.computation.StatementExecutor) StatusResource(io.confluent.ksql.rest.server.resources.StatusResource) TableElement(io.confluent.ksql.parser.tree.TableElement) StreamedQueryResource(io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource) CreateStreamCommand(io.confluent.ksql.ddl.commands.CreateStreamCommand) CommandRunner(io.confluent.ksql.rest.server.computation.CommandRunner) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl) RootDocument(io.confluent.ksql.rest.server.resources.RootDocument) KsqlConfig(io.confluent.ksql.util.KsqlConfig) CreateStream(io.confluent.ksql.parser.tree.CreateStream) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) RegisterTopicCommand(io.confluent.ksql.ddl.commands.RegisterTopicCommand) KafkaTopicClient(io.confluent.ksql.util.KafkaTopicClient) StringLiteral(io.confluent.ksql.parser.tree.StringLiteral) Expression(io.confluent.ksql.parser.tree.Expression) Command(io.confluent.ksql.rest.server.computation.Command) RegisterTopicCommand(io.confluent.ksql.ddl.commands.RegisterTopicCommand) CreateStreamCommand(io.confluent.ksql.ddl.commands.CreateStreamCommand) CommandIdAssigner(io.confluent.ksql.rest.server.computation.CommandIdAssigner) CommandId(io.confluent.ksql.rest.server.computation.CommandId) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException)

Example 2 with StreamedQueryResource

use of io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource in project ksql by confluentinc.

the class StreamedQueryResourceTest method testStreamQuery.

@Test
public void testStreamQuery() throws Throwable {
    final AtomicReference<Throwable> threadException = new AtomicReference<>(null);
    final Thread.UncaughtExceptionHandler threadExceptionHandler = (thread, exception) -> threadException.compareAndSet(null, exception);
    final String queryString = "SELECT * FROM test_stream;";
    final SynchronousQueue<KeyValue<String, GenericRow>> rowQueue = new SynchronousQueue<>();
    final LinkedList<GenericRow> writtenRows = new LinkedList<>();
    final Thread rowQueuePopulatorThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                for (int i = 0; ; i++) {
                    String key = Integer.toString(i);
                    GenericRow value = new GenericRow(Collections.singletonList(i));
                    synchronized (writtenRows) {
                        writtenRows.add(value);
                    }
                    rowQueue.put(new KeyValue<>(key, value));
                }
            } catch (InterruptedException exception) {
            // This should happen during the test, so it's fine
            }
        }
    }, "Row Queue Populator");
    rowQueuePopulatorThread.setUncaughtExceptionHandler(threadExceptionHandler);
    rowQueuePopulatorThread.start();
    final KafkaStreams mockKafkaStreams = mock(KafkaStreams.class);
    mockKafkaStreams.start();
    expectLastCall();
    mockKafkaStreams.setUncaughtExceptionHandler(anyObject(Thread.UncaughtExceptionHandler.class));
    expectLastCall();
    expect(mockKafkaStreams.state()).andReturn(KafkaStreams.State.NOT_RUNNING);
    mockKafkaStreams.close();
    expectLastCall();
    mockKafkaStreams.cleanUp();
    expectLastCall();
    final OutputNode mockOutputNode = mock(OutputNode.class);
    expect(mockOutputNode.getSchema()).andReturn(SchemaBuilder.struct().field("f1", SchemaBuilder.INT32_SCHEMA));
    final Map<String, Object> requestStreamsProperties = Collections.emptyMap();
    KsqlEngine mockKsqlEngine = mock(KsqlEngine.class);
    KafkaTopicClient mockKafkaTopicClient = mock(KafkaTopicClientImpl.class);
    expect(mockKsqlEngine.getTopicClient()).andReturn(mockKafkaTopicClient);
    final QueuedQueryMetadata queuedQueryMetadata = new QueuedQueryMetadata(queryString, mockKafkaStreams, mockOutputNode, "", rowQueue, DataSource.DataSourceType.KSTREAM, "", mockKafkaTopicClient, null);
    expect(mockKsqlEngine.buildMultipleQueries(queryString, requestStreamsProperties)).andReturn(Collections.singletonList(queuedQueryMetadata));
    mockKsqlEngine.removeTemporaryQuery(queuedQueryMetadata);
    expectLastCall();
    StatementParser mockStatementParser = mock(StatementParser.class);
    expect(mockStatementParser.parseSingleStatement(queryString)).andReturn(mock(Query.class));
    replay(mockKsqlEngine, mockStatementParser, mockKafkaStreams, mockOutputNode);
    StreamedQueryResource testResource = new StreamedQueryResource(mockKsqlEngine, mockStatementParser, 1000);
    Response response = testResource.streamQuery(new KsqlRequest(queryString, requestStreamsProperties));
    PipedOutputStream responseOutputStream = new EOFPipedOutputStream();
    PipedInputStream responseInputStream = new PipedInputStream(responseOutputStream, 1);
    StreamingOutput responseStream = (StreamingOutput) response.getEntity();
    final Thread queryWriterThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                responseStream.write(responseOutputStream);
            } catch (EOFException exception) {
            // It's fine
            } catch (IOException exception) {
                throw new RuntimeException(exception);
            }
        }
    }, "Query Writer");
    queryWriterThread.setUncaughtExceptionHandler(threadExceptionHandler);
    queryWriterThread.start();
    Scanner responseScanner = new Scanner(responseInputStream);
    ObjectMapper objectMapper = new ObjectMapper();
    for (int i = 0; i < 5; i++) {
        if (!responseScanner.hasNextLine()) {
            throw new Exception("Response input stream failed to have expected line available");
        }
        String responseLine = responseScanner.nextLine();
        if (responseLine.trim().isEmpty()) {
            i--;
        } else {
            GenericRow expectedRow;
            synchronized (writtenRows) {
                expectedRow = writtenRows.poll();
            }
            GenericRow testRow = objectMapper.readValue(responseLine, StreamedRow.class).getRow();
            assertEquals(expectedRow, testRow);
        }
    }
    responseOutputStream.close();
    queryWriterThread.join();
    rowQueuePopulatorThread.interrupt();
    rowQueuePopulatorThread.join();
    // Definitely want to make sure that the Kafka Streams instance has been closed and cleaned up
    verify(mockKafkaStreams);
    // If one of the other threads has somehow managed to throw an exception without breaking things up until this
    // point, we throw that exception now in the main thread and cause the test to fail
    Throwable exception = threadException.get();
    if (exception != null) {
        throw exception;
    }
}
Also used : Query(io.confluent.ksql.parser.tree.Query) StreamedQueryResource(io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) KafkaTopicClientImpl(io.confluent.ksql.util.KafkaTopicClientImpl) Scanner(java.util.Scanner) EasyMock.mock(org.easymock.EasyMock.mock) KsqlEngine(io.confluent.ksql.KsqlEngine) KafkaTopicClient(io.confluent.ksql.util.KafkaTopicClient) AtomicReference(java.util.concurrent.atomic.AtomicReference) PipedInputStream(java.io.PipedInputStream) Map(java.util.Map) EasyMock.replay(org.easymock.EasyMock.replay) LinkedList(java.util.LinkedList) EasyMock.anyObject(org.easymock.EasyMock.anyObject) OutputNode(io.confluent.ksql.planner.plan.OutputNode) SynchronousQueue(java.util.concurrent.SynchronousQueue) StatementParser(io.confluent.ksql.rest.server.StatementParser) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) KeyValue(org.apache.kafka.streams.KeyValue) Test(org.junit.Test) StreamingOutput(javax.ws.rs.core.StreamingOutput) IOException(java.io.IOException) PipedOutputStream(java.io.PipedOutputStream) EasyMock.expect(org.easymock.EasyMock.expect) QueuedQueryMetadata(io.confluent.ksql.util.QueuedQueryMetadata) EOFException(java.io.EOFException) DataSource(io.confluent.ksql.serde.DataSource) EasyMock.expectLastCall(org.easymock.EasyMock.expectLastCall) Response(javax.ws.rs.core.Response) GenericRow(io.confluent.ksql.GenericRow) SchemaBuilder(org.apache.kafka.connect.data.SchemaBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) EasyMock.verify(org.easymock.EasyMock.verify) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) KsqlEngine(io.confluent.ksql.KsqlEngine) Scanner(java.util.Scanner) KeyValue(org.apache.kafka.streams.KeyValue) Query(io.confluent.ksql.parser.tree.Query) PipedOutputStream(java.io.PipedOutputStream) StreamingOutput(javax.ws.rs.core.StreamingOutput) StreamedQueryResource(io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource) GenericRow(io.confluent.ksql.GenericRow) KsqlRequest(io.confluent.ksql.rest.entity.KsqlRequest) SynchronousQueue(java.util.concurrent.SynchronousQueue) EOFException(java.io.EOFException) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) KafkaStreams(org.apache.kafka.streams.KafkaStreams) OutputNode(io.confluent.ksql.planner.plan.OutputNode) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) StatementParser(io.confluent.ksql.rest.server.StatementParser) QueuedQueryMetadata(io.confluent.ksql.util.QueuedQueryMetadata) AtomicReference(java.util.concurrent.atomic.AtomicReference) PipedInputStream(java.io.PipedInputStream) IOException(java.io.IOException) LinkedList(java.util.LinkedList) IOException(java.io.IOException) EOFException(java.io.EOFException) Response(javax.ws.rs.core.Response) KafkaTopicClient(io.confluent.ksql.util.KafkaTopicClient) EasyMock.anyObject(org.easymock.EasyMock.anyObject) Test(org.junit.Test)

Aggregations

KsqlEngine (io.confluent.ksql.KsqlEngine)2 StreamedQueryResource (io.confluent.ksql.rest.server.resources.streaming.StreamedQueryResource)2 KafkaTopicClient (io.confluent.ksql.util.KafkaTopicClient)2 KafkaTopicClientImpl (io.confluent.ksql.util.KafkaTopicClientImpl)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 GenericRow (io.confluent.ksql.GenericRow)1 CreateStreamCommand (io.confluent.ksql.ddl.commands.CreateStreamCommand)1 RegisterTopicCommand (io.confluent.ksql.ddl.commands.RegisterTopicCommand)1 CreateStream (io.confluent.ksql.parser.tree.CreateStream)1 Expression (io.confluent.ksql.parser.tree.Expression)1 Query (io.confluent.ksql.parser.tree.Query)1 RegisterTopic (io.confluent.ksql.parser.tree.RegisterTopic)1 StringLiteral (io.confluent.ksql.parser.tree.StringLiteral)1 TableElement (io.confluent.ksql.parser.tree.TableElement)1 OutputNode (io.confluent.ksql.planner.plan.OutputNode)1 KsqlRequest (io.confluent.ksql.rest.entity.KsqlRequest)1 ServerInfo (io.confluent.ksql.rest.entity.ServerInfo)1 StreamedRow (io.confluent.ksql.rest.entity.StreamedRow)1 StatementParser (io.confluent.ksql.rest.server.StatementParser)1 Command (io.confluent.ksql.rest.server.computation.Command)1