Examples with KsqlEngine io.confluent.ksql.engine.KsqlEngine used on opensource projects

Example 11 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class LocalCommandsTest method shouldWriteAppIdToCommandFile.

@Test
public void shouldWriteAppIdToCommandFile() throws IOException {
    // Given
    final File dir = commandsDir.newFolder();
    LocalCommands localCommands = LocalCommands.open(ksqlEngine, dir);
    File processedFile = localCommands.getCurrentLocalCommandsFile();
    // When
    localCommands.write(metadata1);
    localCommands.write(metadata2);
    // Then
    // Need to create a new local commands in order not to skip the "current" file we just wrote.
    localCommands = LocalCommands.open(ksqlEngine, dir);
    localCommands.write(metadata3);
    localCommands.processLocalCommandFiles(serviceContext);
    verify(ksqlEngine).cleanupOrphanedInternalTopics(any(), eq(ImmutableSet.of(QUERY_APP_ID1, QUERY_APP_ID2)));
    List<Path> paths = Files.list(dir.toPath()).collect(Collectors.toList());
    String expectedProcessedFileName = processedFile.getAbsolutePath() + LOCAL_COMMANDS_PROCESSED_SUFFIX;
    assertThat(paths.size(), is(2));
    assertThat(paths.stream().anyMatch(path -> path.toFile().getAbsolutePath().equals(expectedProcessedFileName)), is(true));
}

Example 12 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class JsonFormatTest method before.

@Before
public void before() {
    streamName = "STREAM_" + COUNTER.getAndIncrement();
    ksqlConfig = KsqlConfigTestUtil.create(TEST_HARNESS.kafkaBootstrapServers());
    serviceContext = ServiceContextFactory.create(ksqlConfig, DisabledKsqlClient::instance);
    functionRegistry = new InternalFunctionRegistry();
    UserFunctionLoader.newInstance(ksqlConfig, functionRegistry, ".", new Metrics()).load();
    ksqlEngine = new KsqlEngine(serviceContext, ProcessingLogContext.create(), functionRegistry, ServiceInfo.create(ksqlConfig), new SequentialQueryIdGenerator(), ksqlConfig, Collections.emptyList(), new MetricCollectors());
    topicClient = serviceContext.getTopicClient();
    metaStore = ksqlEngine.getMetaStore();
    createInitTopics();
    produceInitData();
    execInitCreateStreamQueries();
}

Example 13 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class ApiIntegrationTest method shouldExecutePushQueryNoLimit.

@Test
public void shouldExecutePushQueryNoLimit() throws Exception {
    KsqlEngine engine = (KsqlEngine) REST_APP.getEngine();
    // One persistent query for the agg table
    assertThatEventually(engine::numberOfLiveQueries, is(1));
    // Given:
    String sql = "SELECT * from " + TEST_STREAM + " EMIT CHANGES;";
    // Create a write stream to capture the incomplete response
    ReceiveStream writeStream = new ReceiveStream(vertx);
    // Make the request to stream a query
    JsonObject properties = new JsonObject();
    JsonObject requestBody = new JsonObject().put("sql", sql).put("properties", properties);
    VertxCompletableFuture<HttpResponse<Void>> responseFuture = new VertxCompletableFuture<>();
    client.post("/query-stream").as(BodyCodec.pipe(writeStream)).sendJsonObject(requestBody, responseFuture);
    assertThatEventually(engine::numberOfLiveQueries, is(2));
    // Wait for all rows in the response to arrive
    assertThatEventually(() -> {
        try {
            Buffer buff = writeStream.getBody();
            QueryResponse queryResponse = new QueryResponse(buff.toString());
            return queryResponse.rows.size();
        } catch (Throwable t) {
            return -1;
        }
    }, greaterThanOrEqualTo(6));
    // The response shouldn't have ended yet
    assertThat(writeStream.isEnded(), is(false));
    QueryResponse queryResponse = new QueryResponse(writeStream.getBody().toString());
    String queryId = queryResponse.responseObject.getString("queryId");
    // Now send another request to close the query
    JsonObject closeQueryRequestBody = new JsonObject().put("queryId", queryId);
    HttpResponse<Buffer> closeQueryResponse = sendRequest(client, "/close-query", closeQueryRequestBody.toBuffer());
    assertThat(closeQueryResponse.statusCode(), is(200));
    // The response should now be ended
    assertThatEventually(writeStream::isEnded, is(true));
    HttpResponse<Void> response = responseFuture.get();
    assertThat(response.statusCode(), is(200));
    // Make sure it's cleaned up on the server
    assertThatEventually(engine::numberOfLiveQueries, is(1));
}

Example 14 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class RecoveryTest method shouldRecover.

private void shouldRecover(final List<QueuedCommand> commands) {
    // Given:
    final KsqlServer executeServer = new KsqlServer(commands);
    executeServer.executeCommands();
    final KsqlEngine engine = executeServer.ksqlEngine;
    // When:
    final KsqlServer recoverServer = new KsqlServer(commands);
    recoverServer.recover();
    final KsqlEngine recovered = recoverServer.ksqlEngine;
    // Then:
    assertThat(recovered.getMetaStore(), sameStore(engine.getMetaStore()));
    final Map<QueryId, PersistentQueryMetadata> queries = queriesById(engine.getPersistentQueries());
    final Map<QueryId, PersistentQueryMetadata> recoveredQueries = queriesById(recovered.getPersistentQueries());
    assertThat(queries.keySet(), equalTo(recoveredQueries.keySet()));
    queries.forEach((queryId, query) -> assertThat(query, sameQuery(recoveredQueries.get(queryId))));
}

Example 15 with KsqlEngine

use of io.confluent.ksql.engine.KsqlEngine in project ksql by confluentinc.

the class ListQueriesExecutorTest method shouldIncludeUnresponsiveIfShowQueriesErrorResponse.

@Test
public void shouldIncludeUnresponsiveIfShowQueriesErrorResponse() {
    // Given
    when(sessionProperties.getInternalRequest()).thenReturn(false);
    final ConfiguredStatement<ListQueries> showQueries = (ConfiguredStatement<ListQueries>) engine.configure("SHOW QUERIES;");
    final PersistentQueryMetadata metadata = givenPersistentQuery("id", RUNNING_QUERY_STATE);
    final KsqlEngine engine = mock(KsqlEngine.class);
    when(engine.getAllLiveQueries()).thenReturn(ImmutableList.of(metadata));
    when(engine.getPersistentQueries()).thenReturn(ImmutableList.of(metadata));
    when(response.isErroneous()).thenReturn(true);
    when(response.getErrorMessage()).thenReturn(new KsqlErrorMessage(10000, "error"));
    queryStatusCount.updateStatusCount(RUNNING_QUERY_STATE, 1);
    queryStatusCount.updateStatusCount(KsqlQueryStatus.UNRESPONSIVE, 1);
    // When
    final Queries queries = (Queries) CUSTOM_EXECUTORS.listQueries().execute(showQueries, sessionProperties, engine, serviceContext).getEntity().orElseThrow(IllegalStateException::new);
    // Then
    assertThat(queries.getQueries(), containsInAnyOrder(persistentQueryMetadataToRunningQuery(metadata, queryStatusCount)));
}