Examples with ConfiguredStatement io.confluent.ksql.statement.ConfiguredStatement used on opensource projects

Example 46 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class TestCaseBuilderUtil method createTopicFromStatement.

private static Topic createTopicFromStatement(final String sql, final MutableMetaStore metaStore, final KsqlConfig ksqlConfig) {
    final KsqlParser parser = new DefaultKsqlParser();
    final Function<ConfiguredStatement<?>, Topic> extractTopic = (ConfiguredStatement<?> stmt) -> {
        final CreateSource statement = (CreateSource) stmt.getStatement();
        final CreateSourceProperties props = statement.getProperties();
        final LogicalSchema logicalSchema = statement.getElements().toLogicalSchema();
        final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(props, statement.getName());
        final Format keyFormat = FormatFactory.fromName(keyFormatInfo.getFormat());
        final SerdeFeatures keySerdeFeats = buildKeyFeatures(keyFormat, logicalSchema);
        final Optional<ParsedSchema> keySchema = keyFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.key(), keyFormatInfo, keyFormat, keySerdeFeats) : Optional.empty();
        final FormatInfo valFormatInfo = SourcePropertiesUtil.getValueFormat(props);
        final Format valFormat = FormatFactory.fromName(valFormatInfo.getFormat());
        final SerdeFeatures valSerdeFeats = buildValueFeatures(ksqlConfig, props, valFormat, logicalSchema);
        final Optional<ParsedSchema> valueSchema = valFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.value(), valFormatInfo, valFormat, valSerdeFeats) : Optional.empty();
        final int partitions = props.getPartitions().orElse(Topic.DEFAULT_PARTITIONS);
        final short rf = props.getReplicas().orElse(Topic.DEFAULT_RF);
        return new Topic(props.getKafkaTopic(), partitions, rf, keySchema, valueSchema, keySerdeFeats, valSerdeFeats);
    };
    try {
        final List<ParsedStatement> parsed = parser.parse(sql);
        if (parsed.size() > 1) {
            throw new IllegalArgumentException("SQL contains more than one statement: " + sql);
        }
        final List<Topic> topics = new ArrayList<>();
        for (ParsedStatement stmt : parsed) {
            // in order to extract the topics, we may need to also register type statements
            if (stmt.getStatement().statement() instanceof SqlBaseParser.RegisterTypeContext) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                registerType(prepare, metaStore);
            }
            if (isCsOrCT(stmt)) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                final ConfiguredStatement<?> configured = ConfiguredStatement.of(prepare, SessionConfig.of(ksqlConfig, Collections.emptyMap()));
                final ConfiguredStatement<?> withFormats = new DefaultFormatInjector().inject(configured);
                topics.add(extractTopic.apply(withFormats));
            }
        }
        return topics.isEmpty() ? null : topics.get(0);
    } catch (final Exception e) {
        // Statement won't parse: this will be detected/handled later.
        System.out.println("Error parsing statement (which may be expected): " + sql);
        e.printStackTrace(System.out);
        return null;
    }
}

Example 47 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class KsqlTesterTest method execute.

@SuppressWarnings("unchecked")
private void execute(final ParsedStatement parsedStatement) {
    final PreparedStatement<?> engineStatement = engine.prepare(parsedStatement);
    final ConfiguredStatement<?> configured = ConfiguredStatement.of(engineStatement, SessionConfig.of(config, overrides));
    createTopics(engineStatement);
    if (engineStatement.getStatement() instanceof InsertValues) {
        pipeInput((ConfiguredStatement<InsertValues>) configured);
        return;
    } else if (engineStatement.getStatement() instanceof SetProperty) {
        PropertyOverrider.set((ConfiguredStatement<SetProperty>) configured, overrides);
        return;
    } else if (engineStatement.getStatement() instanceof UnsetProperty) {
        PropertyOverrider.unset((ConfiguredStatement<UnsetProperty>) configured, overrides);
        return;
    }
    final ConfiguredStatement<?> injected = formatInjector.inject(configured);
    final ExecuteResult result = engine.execute(serviceContext, injected);
    // is DDL statement
    if (!result.getQuery().isPresent()) {
        return;
    }
    final PersistentQueryMetadata query = (PersistentQueryMetadata) result.getQuery().get();
    final Topology topology = query.getTopology();
    final Properties properties = new Properties();
    properties.putAll(query.getStreamsProperties());
    properties.put(StreamsConfig.STATE_DIR_CONFIG, tmpFolder.getRoot().getAbsolutePath());
    final TopologyTestDriver driver = new TopologyTestDriver(topology, properties);
    final List<TopicInfo> inputTopics = query.getSourceNames().stream().map(sn -> engine.getMetaStore().getSource(sn)).map(ds -> new TopicInfo(ds.getKafkaTopicName(), keySerde(ds), valueSerde(ds))).collect(Collectors.toList());
    // Sink may be Optional for source tables. Once source table query execution is supported, then
    // we would need have a condition to not create an output topic info
    final DataSource output = engine.getMetaStore().getSource(query.getSinkName().get());
    final TopicInfo outputInfo = new TopicInfo(output.getKafkaTopicName(), keySerde(output), valueSerde(output));
    driverPipeline.addDriver(driver, inputTopics, outputInfo);
    drivers.put(query.getQueryId(), new DriverAndProperties(driver, properties));
}

Example 48 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class EngineExecutor method executeTablePullQuery.

/**
 * Evaluates a pull query by first analyzing it, then building the logical plan and finally
 * the physical plan. The execution is then done using the physical plan in a pipelined manner.
 * @param statement The pull query
 * @param routingOptions Configuration parameters used for HA routing
 * @param pullQueryMetrics JMX metrics
 * @return the rows that are the result of evaluating the pull query
 */
PullQueryResult executeTablePullQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final HARouting routing, final RoutingOptions routingOptions, final QueryPlannerOptions queryPlannerOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final boolean startImmediately, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (!statement.getStatement().isPullQuery()) {
        throw new IllegalArgumentException("Executor can only handle pull queries");
    }
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = routingOptions.getIsSkipForwardRequest() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PullPhysicalPlan plan = null;
    try {
        // Do not set sessionConfig.getConfig to true! The copying is inefficient and slows down pull
        // query performance significantly.  Instead use QueryPlannerOptions which check overrides
        // deliberately.
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, false);
        // This is a cancel signal that is used to stop both local operations and requests
        final CompletableFuture<Void> shouldCancelRequests = new CompletableFuture<>();
        plan = buildPullPhysicalPlan(logicalPlan, analysis, queryPlannerOptions, shouldCancelRequests, consistencyOffsetVector);
        final PullPhysicalPlan physicalPlan = plan;
        final PullQueryQueue pullQueryQueue = new PullQueryQueue(analysis.getLimitClause());
        final PullQueryQueuePopulator populator = () -> routing.handlePullQuery(serviceContext, physicalPlan, statement, routingOptions, physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
        final PullQueryResult result = new PullQueryResult(physicalPlan.getOutputSchema(), populator, physicalPlan.getQueryId(), pullQueryQueue, pullQueryMetrics, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType, physicalPlan::getRowsReadFromDataSource, shouldCancelRequests, consistencyOffsetVector);
        if (startImmediately) {
            result.start();
        }
        return result;
    } catch (final Exception e) {
        if (plan == null) {
            pullQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PullPhysicalPlan physicalPlan = plan;
            pullQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute pull query {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information. If you " + "see this LOG message, please submit a GitHub ticket and we will scrub " + "the statement text from the error at {}", routingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute pull query. {} {}", routingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed pull query text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 49 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class SqlFormatInjector method inject.

@SuppressWarnings("unchecked")
@Override
public <T extends Statement> ConfiguredStatement<T> inject(final ConfiguredStatement<T> statement) {
    try {
        final Statement node = statement.getStatement();
        final String sql = SqlFormatter.formatSql(node);
        final String sqlWithSemiColon = sql.endsWith(";") ? sql : sql + ";";
        final PreparedStatement<?> prepare = executionContext.prepare(executionContext.parse(sqlWithSemiColon).get(0));
        return statement.withStatement(sql, (T) prepare.getStatement());
    } catch (final Exception e) {
        throw new KsqlException("Unable to format statement! This is bad because " + "it means we cannot persist it onto the command topic: " + statement, e);
    }
}

Example 50 with ConfiguredStatement

use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.

the class TopicCreateInjectorTest method shouldBuildWithClauseWithTopicProperties.

@SuppressWarnings("unchecked")
@Test
public void shouldBuildWithClauseWithTopicProperties() {
    // Given:
    givenStatement("CREATE STREAM x WITH (kafka_topic='topic') AS SELECT * FROM SOURCE;");
    when(builder.build()).thenReturn(new TopicProperties("expectedName", 10, (short) 10));
    // When:
    final ConfiguredStatement<CreateAsSelect> result = (ConfiguredStatement<CreateAsSelect>) injector.inject(statement, builder);
    // Then:
    final CreateSourceAsProperties props = result.getStatement().getProperties();
    assertThat(props.getKafkaTopic(), is(Optional.of("expectedName")));
    assertThat(props.getPartitions(), is(Optional.of(10)));
    assertThat(props.getReplicas(), is(Optional.of((short) 10)));
}