Examples with CreateSourceAsProperties io.confluent.ksql.parser.properties.with.CreateSourceAsProperties used on opensource projects

Example 1 with CreateSourceAsProperties

use of io.confluent.ksql.parser.properties.with.CreateSourceAsProperties in project ksql by confluentinc.

the class KsqlAuthorizationValidatorImpl method getCreateAsSelectSinkTopic.

private KsqlTopic getCreateAsSelectSinkTopic(final MetaStore metaStore, final CreateAsSelect createAsSelect) {
    final CreateSourceAsProperties properties = createAsSelect.getProperties();
    final String sinkTopicName;
    final KeyFormat sinkKeyFormat;
    final ValueFormat sinkValueFormat;
    if (!properties.getKafkaTopic().isPresent()) {
        final DataSource dataSource = metaStore.getSource(createAsSelect.getName());
        if (dataSource != null) {
            sinkTopicName = dataSource.getKafkaTopicName();
            sinkKeyFormat = dataSource.getKsqlTopic().getKeyFormat();
            sinkValueFormat = dataSource.getKsqlTopic().getValueFormat();
        } else {
            throw new KsqlException("Cannot validate for topic access from an unknown stream/table: " + createAsSelect.getName());
        }
    } else {
        sinkTopicName = properties.getKafkaTopic().get();
        // If no format is specified for the sink topic, then use the format from the primary
        // source topic.
        final SourceTopicsExtractor extractor = new SourceTopicsExtractor(metaStore);
        extractor.process(createAsSelect.getQuery(), null);
        final KsqlTopic primaryKsqlTopic = extractor.getPrimarySourceTopic();
        final Optional<Format> keyFormat = properties.getKeyFormat().map(formatName -> FormatFactory.fromName(formatName));
        final Optional<Format> valueFormat = properties.getValueFormat().map(formatName -> FormatFactory.fromName(formatName));
        sinkKeyFormat = keyFormat.map(format -> KeyFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of(), Optional.empty())).orElse(primaryKsqlTopic.getKeyFormat());
        sinkValueFormat = valueFormat.map(format -> ValueFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of())).orElse(primaryKsqlTopic.getValueFormat());
    }
    return new KsqlTopic(sinkTopicName, sinkKeyFormat, sinkValueFormat);
}

Example 2 with CreateSourceAsProperties

use of io.confluent.ksql.parser.properties.with.CreateSourceAsProperties in project ksql by confluentinc.

the class TopicCreateInjectorTest method shouldPassThroughWithClauseToBuilderForCreateAs.

@Test
public void shouldPassThroughWithClauseToBuilderForCreateAs() {
    // Given:
    givenStatement("CREATE STREAM x WITH (kafka_topic='topic') AS SELECT * FROM SOURCE;");
    final CreateSourceAsProperties props = ((CreateAsSelect) statement.getStatement()).getProperties();
    // When:
    injector.inject(statement, builder);
    // Then:
    verify(builder).withWithClause(props.getKafkaTopic(), props.getPartitions(), props.getReplicas());
}

Example 3 with CreateSourceAsProperties

use of io.confluent.ksql.parser.properties.with.CreateSourceAsProperties in project ksql by confluentinc.

the class TopicCreateInjectorTest method shouldBuildWithClauseWithTopicProperties.

@SuppressWarnings("unchecked")
@Test
public void shouldBuildWithClauseWithTopicProperties() {
    // Given:
    givenStatement("CREATE STREAM x WITH (kafka_topic='topic') AS SELECT * FROM SOURCE;");
    when(builder.build()).thenReturn(new TopicProperties("expectedName", 10, (short) 10));
    // When:
    final ConfiguredStatement<CreateAsSelect> result = (ConfiguredStatement<CreateAsSelect>) injector.inject(statement, builder);
    // Then:
    final CreateSourceAsProperties props = result.getStatement().getProperties();
    assertThat(props.getKafkaTopic(), is(Optional.of("expectedName")));
    assertThat(props.getPartitions(), is(Optional.of(10)));
    assertThat(props.getReplicas(), is(Optional.of((short) 10)));
}

Example 4 with CreateSourceAsProperties

use of io.confluent.ksql.parser.properties.with.CreateSourceAsProperties in project ksql by confluentinc.

the class TopicCreateInjector method injectForCreateAsSelect.

@SuppressWarnings("unchecked")
private <T extends CreateAsSelect> ConfiguredStatement<?> injectForCreateAsSelect(final ConfiguredStatement<T> statement, final TopicProperties.Builder topicPropertiesBuilder) {
    final String prefix = statement.getSessionConfig().getConfig(true).getString(KsqlConfig.KSQL_OUTPUT_TOPIC_NAME_PREFIX_CONFIG);
    final T createAsSelect = statement.getStatement();
    final CreateSourceAsProperties properties = createAsSelect.getProperties();
    final SourceTopicsExtractor extractor = new SourceTopicsExtractor(metaStore);
    extractor.process(statement.getStatement().getQuery(), null);
    final String sourceTopicName = extractor.getPrimarySourceTopic().getKafkaTopicName();
    topicPropertiesBuilder.withName(prefix + createAsSelect.getName().text()).withSource(() -> topicClient.describeTopic(sourceTopicName)).withWithClause(properties.getKafkaTopic(), properties.getPartitions(), properties.getReplicas());
    final String topicCleanUpPolicy;
    final Map<String, Object> additionalTopicConfigs = new HashMap<>();
    if (createAsSelect instanceof CreateStreamAsSelect) {
        topicCleanUpPolicy = TopicConfig.CLEANUP_POLICY_DELETE;
    } else {
        if (createAsSelect.getQuery().getWindow().isPresent()) {
            topicCleanUpPolicy = TopicConfig.CLEANUP_POLICY_COMPACT + "," + TopicConfig.CLEANUP_POLICY_DELETE;
            createAsSelect.getQuery().getWindow().get().getKsqlWindowExpression().getRetention().ifPresent(retention -> additionalTopicConfigs.put(TopicConfig.RETENTION_MS_CONFIG, retention.toDuration().toMillis()));
        } else {
            topicCleanUpPolicy = TopicConfig.CLEANUP_POLICY_COMPACT;
        }
    }
    final TopicProperties info = createTopic(topicPropertiesBuilder, topicCleanUpPolicy, additionalTopicConfigs);
    final T withTopic = (T) createAsSelect.copyWith(properties.withTopic(info.getTopicName(), info.getPartitions(), info.getReplicas()));
    final String withTopicText = SqlFormatter.formatSql(withTopic) + ";";
    return statement.withStatement(withTopicText, withTopic);
}

Example 5 with CreateSourceAsProperties

use of io.confluent.ksql.parser.properties.with.CreateSourceAsProperties in project ksql by confluentinc.

the class DefaultSchemaInjector method addSchemaFieldsCas.

private static CreateAsSelect addSchemaFieldsCas(final ConfiguredStatement<CreateAsSelect> preparedStatement, final Optional<SchemaAndId> keySchema, final Optional<SchemaAndId> valueSchema) {
    final CreateAsSelect statement = preparedStatement.getStatement();
    final CreateSourceAsProperties properties = statement.getProperties();
    final Optional<String> keySchemaName;
    final Optional<String> valueSchemaName;
    // Only populate key and value schema names when schema ids are explicitly provided
    if (properties.getKeySchemaId().isPresent() && keySchema.isPresent()) {
        keySchemaName = Optional.ofNullable(keySchema.get().rawSchema.name());
    } else {
        keySchemaName = Optional.empty();
    }
    if (properties.getValueSchemaId().isPresent() && valueSchema.isPresent()) {
        valueSchemaName = Optional.ofNullable(valueSchema.get().rawSchema.name());
    } else {
        valueSchemaName = Optional.empty();
    }
    final CreateSourceAsProperties newProperties = statement.getProperties().withKeyValueSchemaName(keySchemaName, valueSchemaName);
    return statement.copyWith(newProperties);
}