Examples with PersistentQueryMetadata io.confluent.ksql.util.PersistentQueryMetadata used on opensource projects

Example 11 with PersistentQueryMetadata

use of io.confluent.ksql.util.PersistentQueryMetadata in project ksql by confluentinc.

the class KsqlEngine method planQueries.

private List<QueryMetadata> planQueries(final List<Pair<String, Statement>> statementList, final Map<String, Object> overriddenProperties, final MetaStore tempMetaStore) throws Exception {
    // Logical plan creation from the ASTs
    List<Pair<String, PlanNode>> logicalPlans = queryEngine.buildLogicalPlans(tempMetaStore, statementList);
    // Physical plan creation from logical plans.
    List<QueryMetadata> runningQueries = queryEngine.buildPhysicalPlans(logicalPlans, statementList, overriddenProperties, true);
    for (QueryMetadata queryMetadata : runningQueries) {
        if (queryMetadata instanceof PersistentQueryMetadata) {
            livePersistentQueries.add(queryMetadata);
            PersistentQueryMetadata persistentQueryMetadata = (PersistentQueryMetadata) queryMetadata;
            persistentQueries.put(persistentQueryMetadata.getId(), persistentQueryMetadata);
        }
        allLiveQueries.add(queryMetadata);
    }
    return runningQueries;
}

Example 12 with PersistentQueryMetadata

use of io.confluent.ksql.util.PersistentQueryMetadata in project ksql by confluentinc.

the class KsqlResource method getStatementExecutionPlan.

private SourceDescription getStatementExecutionPlan(String queryId, Statement statement, String statementText, Map<String, Object> properties) throws KsqlException {
    if (queryId != null) {
        PersistentQueryMetadata metadata = ksqlEngine.getPersistentQueries().get(new QueryId(queryId));
        if (metadata == null) {
            throw new KsqlException(("Query with id:" + queryId + " does not exist, use SHOW QUERIES to view the full set of " + "queries."));
        }
        KsqlStructuredDataOutputNode outputNode = (KsqlStructuredDataOutputNode) metadata.getOutputNode();
        return new SourceDescription(outputNode, metadata.getStatementString(), metadata.getStatementString(), metadata.getTopologyDescription(), metadata.getExecutionPlan(), ksqlEngine.getTopicClient());
    }
    DdlCommandTask ddlCommandTask = ddlCommandTasks.get(statement.getClass());
    if (ddlCommandTask != null) {
        try {
            String executionPlan = ddlCommandTask.execute(statement, statementText, properties);
            return new SourceDescription("", "User-Evaluation", Collections.EMPTY_LIST, Collections.EMPTY_LIST, Collections.EMPTY_LIST, "QUERY", "", "", "", "", true, "", "", "", executionPlan, 0, 0);
        } catch (KsqlException ksqlException) {
            throw ksqlException;
        } catch (Throwable t) {
            throw new KsqlException("Cannot RUN execution plan for this statement, " + statement, t);
        }
    }
    throw new KsqlException("Cannot FIND execution plan for this statement:" + statement);
}

Example 13 with PersistentQueryMetadata

use of io.confluent.ksql.util.PersistentQueryMetadata in project ksql by confluentinc.

the class PhysicalPlanBuilder method buildPlanForStructuredOutputNode.

private QueryMetadata buildPlanForStructuredOutputNode(String sqlExpression, final SchemaKStream schemaKStream, final KsqlStructuredDataOutputNode outputNode, final String serviceId, final String persistanceQueryPrefix, final String statement) {
    if (metaStore.getTopic(outputNode.getKafkaTopicName()) == null) {
        metaStore.putTopic(outputNode.getKsqlTopic());
    }
    StructuredDataSource sinkDataSource;
    if (schemaKStream instanceof SchemaKTable) {
        SchemaKTable schemaKTable = (SchemaKTable) schemaKStream;
        sinkDataSource = new KsqlTable(sqlExpression, outputNode.getId().toString(), outputNode.getSchema(), schemaKStream.getKeyField(), outputNode.getTimestampField(), outputNode.getKsqlTopic(), outputNode.getId().toString() + ksqlConfig.get(KsqlConfig.KSQL_TABLE_STATESTORE_NAME_SUFFIX_CONFIG), schemaKTable.isWindowed());
    } else {
        sinkDataSource = new KsqlStream(sqlExpression, outputNode.getId().toString(), outputNode.getSchema(), schemaKStream.getKeyField(), outputNode.getTimestampField(), outputNode.getKsqlTopic());
    }
    if (updateMetastore) {
        metaStore.putSource(sinkDataSource.cloneWithTimeKeyColumns());
    }
    final QueryId queryId = sinkDataSource.getPersistentQueryId();
    final String applicationId = serviceId + persistanceQueryPrefix + queryId;
    KafkaStreams streams = buildStreams(builder, applicationId, ksqlConfig, overriddenStreamsProperties);
    Topology topology = builder.build();
    return new PersistentQueryMetadata(statement, streams, outputNode, schemaKStream.getExecutionPlan(""), queryId, (schemaKStream instanceof SchemaKTable) ? DataSource.DataSourceType.KTABLE : DataSource.DataSourceType.KSTREAM, applicationId, kafkaTopicClient, outputNode.getSchema(), sinkDataSource.getKsqlTopic(), topology);
}

Example 14 with PersistentQueryMetadata

use of io.confluent.ksql.util.PersistentQueryMetadata in project ksql by confluentinc.

the class KsqlResource method registerDdlCommandTasks.

private void registerDdlCommandTasks() {
    ddlCommandTasks.put(Query.class, (statement, statementText, properties) -> ksqlEngine.getQueryExecutionPlan((Query) statement).getExecutionPlan());
    ddlCommandTasks.put(CreateStreamAsSelect.class, (statement, statementText, properties) -> {
        QueryMetadata queryMetadata = ksqlEngine.getQueryExecutionPlan(((CreateStreamAsSelect) statement).getQuery());
        if (queryMetadata.getDataSourceType() == DataSource.DataSourceType.KTABLE) {
            throw new KsqlException("Invalid result type. Your SELECT query produces a TABLE. Please " + "use CREATE TABLE AS SELECT statement instead.");
        }
        if (queryMetadata instanceof PersistentQueryMetadata) {
            new AvroUtil().validatePersistentQueryResults((PersistentQueryMetadata) queryMetadata, ksqlEngine.getSchemaRegistryClient());
        }
        queryMetadata.close();
        return queryMetadata.getExecutionPlan();
    });
    ddlCommandTasks.put(CreateTableAsSelect.class, (statement, statementText, properties) -> {
        QueryMetadata queryMetadata = ksqlEngine.getQueryExecutionPlan(((CreateTableAsSelect) statement).getQuery());
        if (queryMetadata.getDataSourceType() != DataSource.DataSourceType.KTABLE) {
            throw new KsqlException("Invalid result type. Your SELECT query produces a STREAM. Please " + "use CREATE STREAM AS SELECT statement instead.");
        }
        if (queryMetadata instanceof PersistentQueryMetadata) {
            new AvroUtil().validatePersistentQueryResults((PersistentQueryMetadata) queryMetadata, ksqlEngine.getSchemaRegistryClient());
        }
        queryMetadata.close();
        return queryMetadata.getExecutionPlan();
    });
    ddlCommandTasks.put(RegisterTopic.class, (statement, statementText, properties) -> {
        RegisterTopicCommand registerTopicCommand = new RegisterTopicCommand((RegisterTopic) statement);
        new DdlCommandExec(ksqlEngine.getMetaStore().clone()).execute(registerTopicCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(CreateStream.class, (statement, statementText, properties) -> {
        CreateStreamCommand createStreamCommand = new CreateStreamCommand(statementText, (CreateStream) statement, properties, ksqlEngine.getTopicClient(), true);
        executeDdlCommand(createStreamCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(CreateTable.class, (statement, statementText, properties) -> {
        CreateTableCommand createTableCommand = new CreateTableCommand(statementText, (CreateTable) statement, properties, ksqlEngine.getTopicClient(), true);
        executeDdlCommand(createTableCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(DropTopic.class, (statement, statementText, properties) -> {
        DropTopicCommand dropTopicCommand = new DropTopicCommand((DropTopic) statement);
        new DdlCommandExec(ksqlEngine.getMetaStore().clone()).execute(dropTopicCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(DropStream.class, (statement, statementText, properties) -> {
        DropSourceCommand dropSourceCommand = new DropSourceCommand((DropStream) statement, DataSource.DataSourceType.KSTREAM, ksqlEngine);
        executeDdlCommand(dropSourceCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(DropTable.class, (statement, statementText, properties) -> {
        DropSourceCommand dropSourceCommand = new DropSourceCommand((DropTable) statement, DataSource.DataSourceType.KTABLE, ksqlEngine);
        executeDdlCommand(dropSourceCommand);
        return statement.toString();
    });
    ddlCommandTasks.put(TerminateQuery.class, (statement, statementText, properties) -> statement.toString());
}

Example 15 with PersistentQueryMetadata

use of io.confluent.ksql.util.PersistentQueryMetadata in project ksql by confluentinc.

the class KsqlResource method showQueries.

// Only shows queries running on the current machine, not across the entire cluster
private Queries showQueries(String statementText) {
    List<Queries.RunningQuery> runningQueries = new ArrayList<>();
    for (PersistentQueryMetadata persistentQueryMetadata : ksqlEngine.getPersistentQueries().values()) {
        KsqlStructuredDataOutputNode ksqlStructuredDataOutputNode = (KsqlStructuredDataOutputNode) persistentQueryMetadata.getOutputNode();
        runningQueries.add(new Queries.RunningQuery(persistentQueryMetadata.getStatementString(), ksqlStructuredDataOutputNode.getKafkaTopicName(), persistentQueryMetadata.getId()));
    }
    return new Queries(statementText, runningQueries);
}