Search in sources :

Example 1 with SerdeFeatures

use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.

the class CreateSourceFactory method buildFormats.

private Formats buildFormats(final SourceName name, final LogicalSchema schema, final CreateSourceProperties props, final KsqlConfig ksqlConfig) {
    final FormatInfo keyFormat = SourcePropertiesUtil.getKeyFormat(props, name);
    final FormatInfo valueFormat = SourcePropertiesUtil.getValueFormat(props);
    final SerdeFeatures keyFeatures = keySerdeFeaturesSupplier.build(schema, FormatFactory.of(keyFormat), SerdeFeatures.of(), ksqlConfig);
    final SerdeFeatures valFeatures = valueSerdeFeaturesSupplier.build(schema, FormatFactory.of(valueFormat), props.getValueSerdeFeatures(), ksqlConfig);
    final Formats formats = Formats.of(keyFormat, valueFormat, keyFeatures, valFeatures);
    validateSerdesCanHandleSchemas(ksqlConfig, schema, formats);
    return formats;
}
Also used : Formats(io.confluent.ksql.execution.plan.Formats) FormatInfo(io.confluent.ksql.serde.FormatInfo) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures)

Example 2 with SerdeFeatures

use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.

the class DataGenProducer method getKeySerializer.

private Serializer<GenericKey> getKeySerializer(final LogicalSchema schema) {
    final Set<SerdeFeature> supported = keySerializerFactory.format().supportedFeatures();
    final SerdeFeatures features = supported.contains(SerdeFeature.UNWRAP_SINGLES) ? SerdeFeatures.of(SerdeFeature.UNWRAP_SINGLES) : SerdeFeatures.of();
    final PersistenceSchema persistenceSchema = PersistenceSchema.from(schema.key(), features);
    return keySerializerFactory.create(persistenceSchema);
}
Also used : SerdeFeature(io.confluent.ksql.serde.SerdeFeature) PersistenceSchema(io.confluent.ksql.schema.ksql.PersistenceSchema) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures)

Example 3 with SerdeFeatures

use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.

the class TestCaseBuilderUtil method createTopicFromStatement.

private static Topic createTopicFromStatement(final String sql, final MutableMetaStore metaStore, final KsqlConfig ksqlConfig) {
    final KsqlParser parser = new DefaultKsqlParser();
    final Function<ConfiguredStatement<?>, Topic> extractTopic = (ConfiguredStatement<?> stmt) -> {
        final CreateSource statement = (CreateSource) stmt.getStatement();
        final CreateSourceProperties props = statement.getProperties();
        final LogicalSchema logicalSchema = statement.getElements().toLogicalSchema();
        final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(props, statement.getName());
        final Format keyFormat = FormatFactory.fromName(keyFormatInfo.getFormat());
        final SerdeFeatures keySerdeFeats = buildKeyFeatures(keyFormat, logicalSchema);
        final Optional<ParsedSchema> keySchema = keyFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.key(), keyFormatInfo, keyFormat, keySerdeFeats) : Optional.empty();
        final FormatInfo valFormatInfo = SourcePropertiesUtil.getValueFormat(props);
        final Format valFormat = FormatFactory.fromName(valFormatInfo.getFormat());
        final SerdeFeatures valSerdeFeats = buildValueFeatures(ksqlConfig, props, valFormat, logicalSchema);
        final Optional<ParsedSchema> valueSchema = valFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.value(), valFormatInfo, valFormat, valSerdeFeats) : Optional.empty();
        final int partitions = props.getPartitions().orElse(Topic.DEFAULT_PARTITIONS);
        final short rf = props.getReplicas().orElse(Topic.DEFAULT_RF);
        return new Topic(props.getKafkaTopic(), partitions, rf, keySchema, valueSchema, keySerdeFeats, valSerdeFeats);
    };
    try {
        final List<ParsedStatement> parsed = parser.parse(sql);
        if (parsed.size() > 1) {
            throw new IllegalArgumentException("SQL contains more than one statement: " + sql);
        }
        final List<Topic> topics = new ArrayList<>();
        for (ParsedStatement stmt : parsed) {
            // in order to extract the topics, we may need to also register type statements
            if (stmt.getStatement().statement() instanceof SqlBaseParser.RegisterTypeContext) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                registerType(prepare, metaStore);
            }
            if (isCsOrCT(stmt)) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                final ConfiguredStatement<?> configured = ConfiguredStatement.of(prepare, SessionConfig.of(ksqlConfig, Collections.emptyMap()));
                final ConfiguredStatement<?> withFormats = new DefaultFormatInjector().inject(configured);
                topics.add(extractTopic.apply(withFormats));
            }
        }
        return topics.isEmpty() ? null : topics.get(0);
    } catch (final Exception e) {
        // Statement won't parse: this will be detected/handled later.
        System.out.println("Error parsing statement (which may be expected): " + sql);
        e.printStackTrace(System.out);
        return null;
    }
}
Also used : Optional(java.util.Optional) CreateSource(io.confluent.ksql.parser.tree.CreateSource) ArrayList(java.util.ArrayList) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) ParsedStatement(io.confluent.ksql.parser.KsqlParser.ParsedStatement) DefaultFormatInjector(io.confluent.ksql.format.DefaultFormatInjector) DefaultKsqlParser(io.confluent.ksql.parser.DefaultKsqlParser) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) Format(io.confluent.ksql.serde.Format) KsqlParser(io.confluent.ksql.parser.KsqlParser) DefaultKsqlParser(io.confluent.ksql.parser.DefaultKsqlParser) FormatInfo(io.confluent.ksql.serde.FormatInfo) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) CreateSourceProperties(io.confluent.ksql.parser.properties.with.CreateSourceProperties)

Example 4 with SerdeFeatures

use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.

the class SchemaKStream method throwOnJoinKeyFormatsMismatch.

void throwOnJoinKeyFormatsMismatch(final SchemaKStream<?> right) {
    final FormatInfo leftFmt = this.keyFormat.getFormatInfo();
    final FormatInfo rightFmt = right.keyFormat.getFormatInfo();
    if (!leftFmt.equals(rightFmt)) {
        throw new IllegalArgumentException("Key format mismatch in join. " + "left: " + leftFmt + ", right: " + rightFmt);
    }
    final SerdeFeatures leftFeats = this.keyFormat.getFeatures();
    final SerdeFeatures rightFeats = right.keyFormat.getFeatures();
    if (!leftFeats.equals(rightFeats)) {
        throw new IllegalArgumentException("Key format features mismatch in join. " + "left: " + leftFeats + ", right: " + rightFeats);
    }
    final Optional<WindowType> leftWnd = this.keyFormat.getWindowInfo().map(WindowInfo::getType);
    final Optional<WindowType> rightWnd = right.keyFormat.getWindowInfo().map(WindowInfo::getType);
    if (leftWnd.isPresent() != rightWnd.isPresent()) {
        throw new IllegalArgumentException("Key format windowing mismatch in join. " + "left: " + leftWnd + ", right: " + rightWnd);
    }
    final boolean leftIsSession = leftWnd.map(type -> type == WindowType.SESSION).orElse(false);
    final boolean rightIsSession = rightWnd.map(type -> type == WindowType.SESSION).orElse(false);
    if (leftIsSession != rightIsSession) {
        throw new IllegalArgumentException("Key format window type mismatch in join. " + "left: " + (leftIsSession ? "Session Windowed" : "Non Session Windowed") + ", right: " + (rightIsSession ? "Session Windowed" : "Non Session Windowed"));
    }
}
Also used : StatementRewriteForMagicPseudoTimestamp(io.confluent.ksql.engine.rewrite.StatementRewriteForMagicPseudoTimestamp) ColumnName(io.confluent.ksql.name.ColumnName) WithinExpression(io.confluent.ksql.parser.tree.WithinExpression) StreamFilter(io.confluent.ksql.execution.plan.StreamFilter) KeyFormat(io.confluent.ksql.serde.KeyFormat) QueryContext(io.confluent.ksql.execution.context.QueryContext) JoinType(io.confluent.ksql.execution.plan.JoinType) SerdeFeaturesFactory(io.confluent.ksql.serde.SerdeFeaturesFactory) Formats(io.confluent.ksql.execution.plan.Formats) StreamGroupByKey(io.confluent.ksql.execution.plan.StreamGroupByKey) StepSchemaResolver(io.confluent.ksql.execution.streams.StepSchemaResolver) WindowInfo(io.confluent.ksql.serde.WindowInfo) NoneFormat(io.confluent.ksql.serde.none.NoneFormat) StreamSelect(io.confluent.ksql.execution.plan.StreamSelect) Objects.requireNonNull(java.util.Objects.requireNonNull) ExpressionTypeManager(io.confluent.ksql.execution.util.ExpressionTypeManager) WindowType(io.confluent.ksql.model.WindowType) Repartitioning(io.confluent.ksql.util.Repartitioning) SqlType(io.confluent.ksql.schema.ksql.types.SqlType) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) ExecutionStepFactory(io.confluent.ksql.execution.streams.ExecutionStepFactory) StreamTableJoin(io.confluent.ksql.execution.plan.StreamTableJoin) ExecutionStep(io.confluent.ksql.execution.plan.ExecutionStep) PlanBuildContext(io.confluent.ksql.planner.plan.PlanBuildContext) Expression(io.confluent.ksql.execution.expression.tree.Expression) FunctionRegistry(io.confluent.ksql.function.FunctionRegistry) KsqlConfig(io.confluent.ksql.util.KsqlConfig) InternalFormats(io.confluent.ksql.serde.InternalFormats) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) TimestampColumn(io.confluent.ksql.execution.timestamp.TimestampColumn) SelectExpression(io.confluent.ksql.execution.plan.SelectExpression) KStreamHolder(io.confluent.ksql.execution.plan.KStreamHolder) Objects(java.util.Objects) FunctionCall(io.confluent.ksql.execution.expression.tree.FunctionCall) List(java.util.List) StreamStreamJoin(io.confluent.ksql.execution.plan.StreamStreamJoin) StreamSink(io.confluent.ksql.execution.plan.StreamSink) Stacker(io.confluent.ksql.execution.context.QueryContext.Stacker) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) StreamFlatMap(io.confluent.ksql.execution.plan.StreamFlatMap) FormatInfo(io.confluent.ksql.serde.FormatInfo) StreamGroupBy(io.confluent.ksql.execution.plan.StreamGroupBy) FormatInfo(io.confluent.ksql.serde.FormatInfo) WindowType(io.confluent.ksql.model.WindowType) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) WindowInfo(io.confluent.ksql.serde.WindowInfo)

Example 5 with SerdeFeatures

use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.

the class LogicalPlanner method getSinkTopic.

private KsqlTopic getSinkTopic(final Into into, final LogicalSchema schema) {
    if (into.getExistingTopic().isPresent()) {
        return into.getExistingTopic().get();
    }
    final NewTopic newTopic = into.getNewTopic().orElseThrow(IllegalStateException::new);
    final FormatInfo keyFormat = getSinkKeyFormat(schema, newTopic);
    final SerdeFeatures keyFeatures = SerdeFeaturesFactory.buildKeyFeatures(schema, FormatFactory.of(keyFormat));
    final SerdeFeatures valFeatures = SerdeFeaturesFactory.buildValueFeatures(schema, FormatFactory.of(newTopic.getValueFormat()), analysis.getProperties().getValueSerdeFeatures(), ksqlConfig);
    return new KsqlTopic(newTopic.getTopicName(), KeyFormat.of(keyFormat, keyFeatures, newTopic.getWindowInfo()), ValueFormat.of(newTopic.getValueFormat(), valFeatures));
}
Also used : NewTopic(io.confluent.ksql.analyzer.Analysis.Into.NewTopic) FormatInfo(io.confluent.ksql.serde.FormatInfo) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic)

Aggregations

SerdeFeatures (io.confluent.ksql.serde.SerdeFeatures)7 FormatInfo (io.confluent.ksql.serde.FormatInfo)6 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)3 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)2 Formats (io.confluent.ksql.execution.plan.Formats)2 CreateSource (io.confluent.ksql.parser.tree.CreateSource)2 SchemaAndId (io.confluent.ksql.schema.ksql.inference.TopicSchemaSupplier.SchemaAndId)2 Format (io.confluent.ksql.serde.Format)2 Optional (java.util.Optional)2 NewTopic (io.confluent.ksql.analyzer.Analysis.Into.NewTopic)1 StatementRewriteForMagicPseudoTimestamp (io.confluent.ksql.engine.rewrite.StatementRewriteForMagicPseudoTimestamp)1 QueryContext (io.confluent.ksql.execution.context.QueryContext)1 Stacker (io.confluent.ksql.execution.context.QueryContext.Stacker)1 Expression (io.confluent.ksql.execution.expression.tree.Expression)1 FunctionCall (io.confluent.ksql.execution.expression.tree.FunctionCall)1 ExecutionStep (io.confluent.ksql.execution.plan.ExecutionStep)1 JoinType (io.confluent.ksql.execution.plan.JoinType)1 KStreamHolder (io.confluent.ksql.execution.plan.KStreamHolder)1 SelectExpression (io.confluent.ksql.execution.plan.SelectExpression)1 StreamFilter (io.confluent.ksql.execution.plan.StreamFilter)1