Examples with ConfiguredStatement io.confluent.ksql.statement.ConfiguredStatement used on opensource projects

Example 11 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class TerminateQueryExecutorTest method shouldTerminateTransientQuery.

@Test
public void shouldTerminateTransientQuery() {
    // Given:
    final ConfiguredStatement<TerminateQuery> terminateTransient = (ConfiguredStatement<TerminateQuery>) engine.configure("TERMINATE TRANSIENT_QUERY;");
    final TransientQueryMetadata transientQueryMetadata = givenTransientQuery("TRANSIENT_QUERY", RUNNING_QUERY_STATE);
    final QueryId transientQueryId = transientQueryMetadata.getQueryId();
    final KsqlEngine engine = mock(KsqlEngine.class);
    when(engine.getQuery(transientQueryId)).thenReturn(Optional.of(transientQueryMetadata));
    // When:
    final Optional<KsqlEntity> ksqlEntity = CUSTOM_EXECUTORS.terminateQuery().execute(terminateTransient, mock(SessionProperties.class), engine, this.engine.getServiceContext()).getEntity();
    // Then:
    assertThat(ksqlEntity, is(Optional.of(new TerminateQueryEntity(terminateTransient.getStatementText(), transientQueryId.toString(), true))));
}

Example 12 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class TerminateQueryExecutorTest method shouldDefaultToDistributorForPersistentQuery.

@Test
public void shouldDefaultToDistributorForPersistentQuery() {
    // Given:
    final ConfiguredStatement<TerminateQuery> terminatePersistent = (ConfiguredStatement<TerminateQuery>) engine.configure("TERMINATE PERSISTENT_QUERY;");
    final PersistentQueryMetadata persistentQueryMetadata = givenPersistentQuery("PERSISTENT_QUERY", RUNNING_QUERY_STATE);
    final QueryId persistentQueryId = persistentQueryMetadata.getQueryId();
    final KsqlEngine engine = mock(KsqlEngine.class);
    when(engine.getPersistentQuery(persistentQueryId)).thenReturn(Optional.of(persistentQueryMetadata));
    // When:
    final Optional<KsqlEntity> ksqlEntity = CUSTOM_EXECUTORS.terminateQuery().execute(terminatePersistent, mock(SessionProperties.class), engine, this.engine.getServiceContext()).getEntity();
    // Then:
    assertThat(ksqlEntity, is(Optional.empty()));
}

Example 13 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class EngineExecutor method maybeCreateSinkDdl.

private Optional<DdlCommand> maybeCreateSinkDdl(final ConfiguredStatement<?> cfgStatement, final KsqlStructuredDataOutputNode outputNode) {
    if (!outputNode.createInto()) {
        validateExistingSink(outputNode);
        return Optional.empty();
    }
    final Statement statement = cfgStatement.getStatement();
    final SourceName intoSource = outputNode.getSinkName().get();
    final boolean orReplace = statement instanceof CreateAsSelect && ((CreateAsSelect) statement).isOrReplace();
    final boolean ifNotExists = statement instanceof CreateAsSelect && ((CreateAsSelect) statement).isNotExists();
    final DataSource dataSource = engineContext.getMetaStore().getSource(intoSource);
    if (dataSource != null && !ifNotExists && !orReplace) {
        final String failedSourceType = outputNode.getNodeOutputType().getKsqlType();
        final String foundSourceType = dataSource.getDataSourceType().getKsqlType();
        throw new KsqlException(String.format("Cannot add %s '%s': A %s with the same name already exists", failedSourceType.toLowerCase(), intoSource.text(), foundSourceType.toLowerCase()));
    }
    return Optional.of(engineContext.createDdlCommand(outputNode));
}

Example 14 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}

Example 15 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class EngineExecutor method executeScalablePushQuery.

ScalablePushQueryMetadata executeScalablePushQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final PushRouting pushRouting, final PushRoutingOptions pushRoutingOptions, final QueryPlannerOptions queryPlannerOptions, final Context context, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics) {
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = pushRoutingOptions.getHasBeenForwarded() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PushPhysicalPlan plan = null;
    try {
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, true);
        final PushPhysicalPlanCreator pushPhysicalPlanCreator = (offsetRange, catchupConsumerGroup) -> buildScalablePushPhysicalPlan(logicalPlan, analysis, context, offsetRange, catchupConsumerGroup);
        final Optional<PushOffsetRange> offsetRange = pushRoutingOptions.getContinuationToken().map(PushOffsetRange::deserialize);
        final Optional<String> catchupConsumerGroup = pushRoutingOptions.getCatchupConsumerGroup();
        final PushPhysicalPlanManager physicalPlanManager = new PushPhysicalPlanManager(pushPhysicalPlanCreator, catchupConsumerGroup, offsetRange);
        final PushPhysicalPlan physicalPlan = physicalPlanManager.getPhysicalPlan();
        plan = physicalPlan;
        final TransientQueryQueue transientQueryQueue = new TransientQueryQueue(analysis.getLimitClause());
        final PushQueryMetadata.ResultType resultType = physicalPlan.getScalablePushRegistry().isTable() ? physicalPlan.getScalablePushRegistry().isWindowed() ? ResultType.WINDOWED_TABLE : ResultType.TABLE : ResultType.STREAM;
        final PushQueryQueuePopulator populator = () -> pushRouting.handlePushQuery(serviceContext, physicalPlanManager, statement, pushRoutingOptions, physicalPlan.getOutputSchema(), transientQueryQueue, scalablePushQueryMetrics, offsetRange);
        final PushQueryPreparer preparer = () -> pushRouting.preparePushQuery(physicalPlanManager, statement, pushRoutingOptions);
        final ScalablePushQueryMetadata metadata = new ScalablePushQueryMetadata(physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), transientQueryQueue, scalablePushQueryMetrics, resultType, populator, preparer, physicalPlan.getSourceType(), routingNodeType, physicalPlan::getRowsReadFromDataSource);
        return metadata;
    } catch (final Exception e) {
        if (plan == null) {
            scalablePushQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PushPhysicalPlan pushPhysicalPlan = plan;
            scalablePushQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, pushPhysicalPlan.getSourceType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute push query V2 {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information." + " If you see this LOG message, please submit a GitHub ticket and" + " we will scrub the statement text from the error at {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute push query V2. {} {}", pushRoutingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed push query V2 text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}