use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.
the class CreateSourceFactory method buildFormats.
private Formats buildFormats(final SourceName name, final LogicalSchema schema, final CreateSourceProperties props, final KsqlConfig ksqlConfig) {
final FormatInfo keyFormat = SourcePropertiesUtil.getKeyFormat(props, name);
final FormatInfo valueFormat = SourcePropertiesUtil.getValueFormat(props);
final SerdeFeatures keyFeatures = keySerdeFeaturesSupplier.build(schema, FormatFactory.of(keyFormat), SerdeFeatures.of(), ksqlConfig);
final SerdeFeatures valFeatures = valueSerdeFeaturesSupplier.build(schema, FormatFactory.of(valueFormat), props.getValueSerdeFeatures(), ksqlConfig);
final Formats formats = Formats.of(keyFormat, valueFormat, keyFeatures, valFeatures);
validateSerdesCanHandleSchemas(ksqlConfig, schema, formats);
return formats;
}
use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.
the class DataGenProducer method getKeySerializer.
private Serializer<GenericKey> getKeySerializer(final LogicalSchema schema) {
final Set<SerdeFeature> supported = keySerializerFactory.format().supportedFeatures();
final SerdeFeatures features = supported.contains(SerdeFeature.UNWRAP_SINGLES) ? SerdeFeatures.of(SerdeFeature.UNWRAP_SINGLES) : SerdeFeatures.of();
final PersistenceSchema persistenceSchema = PersistenceSchema.from(schema.key(), features);
return keySerializerFactory.create(persistenceSchema);
}
use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.
the class TestCaseBuilderUtil method createTopicFromStatement.
private static Topic createTopicFromStatement(final String sql, final MutableMetaStore metaStore, final KsqlConfig ksqlConfig) {
final KsqlParser parser = new DefaultKsqlParser();
final Function<ConfiguredStatement<?>, Topic> extractTopic = (ConfiguredStatement<?> stmt) -> {
final CreateSource statement = (CreateSource) stmt.getStatement();
final CreateSourceProperties props = statement.getProperties();
final LogicalSchema logicalSchema = statement.getElements().toLogicalSchema();
final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(props, statement.getName());
final Format keyFormat = FormatFactory.fromName(keyFormatInfo.getFormat());
final SerdeFeatures keySerdeFeats = buildKeyFeatures(keyFormat, logicalSchema);
final Optional<ParsedSchema> keySchema = keyFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.key(), keyFormatInfo, keyFormat, keySerdeFeats) : Optional.empty();
final FormatInfo valFormatInfo = SourcePropertiesUtil.getValueFormat(props);
final Format valFormat = FormatFactory.fromName(valFormatInfo.getFormat());
final SerdeFeatures valSerdeFeats = buildValueFeatures(ksqlConfig, props, valFormat, logicalSchema);
final Optional<ParsedSchema> valueSchema = valFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.value(), valFormatInfo, valFormat, valSerdeFeats) : Optional.empty();
final int partitions = props.getPartitions().orElse(Topic.DEFAULT_PARTITIONS);
final short rf = props.getReplicas().orElse(Topic.DEFAULT_RF);
return new Topic(props.getKafkaTopic(), partitions, rf, keySchema, valueSchema, keySerdeFeats, valSerdeFeats);
};
try {
final List<ParsedStatement> parsed = parser.parse(sql);
if (parsed.size() > 1) {
throw new IllegalArgumentException("SQL contains more than one statement: " + sql);
}
final List<Topic> topics = new ArrayList<>();
for (ParsedStatement stmt : parsed) {
// in order to extract the topics, we may need to also register type statements
if (stmt.getStatement().statement() instanceof SqlBaseParser.RegisterTypeContext) {
final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
registerType(prepare, metaStore);
}
if (isCsOrCT(stmt)) {
final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
final ConfiguredStatement<?> configured = ConfiguredStatement.of(prepare, SessionConfig.of(ksqlConfig, Collections.emptyMap()));
final ConfiguredStatement<?> withFormats = new DefaultFormatInjector().inject(configured);
topics.add(extractTopic.apply(withFormats));
}
}
return topics.isEmpty() ? null : topics.get(0);
} catch (final Exception e) {
// Statement won't parse: this will be detected/handled later.
System.out.println("Error parsing statement (which may be expected): " + sql);
e.printStackTrace(System.out);
return null;
}
}
use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.
the class SchemaKStream method throwOnJoinKeyFormatsMismatch.
void throwOnJoinKeyFormatsMismatch(final SchemaKStream<?> right) {
final FormatInfo leftFmt = this.keyFormat.getFormatInfo();
final FormatInfo rightFmt = right.keyFormat.getFormatInfo();
if (!leftFmt.equals(rightFmt)) {
throw new IllegalArgumentException("Key format mismatch in join. " + "left: " + leftFmt + ", right: " + rightFmt);
}
final SerdeFeatures leftFeats = this.keyFormat.getFeatures();
final SerdeFeatures rightFeats = right.keyFormat.getFeatures();
if (!leftFeats.equals(rightFeats)) {
throw new IllegalArgumentException("Key format features mismatch in join. " + "left: " + leftFeats + ", right: " + rightFeats);
}
final Optional<WindowType> leftWnd = this.keyFormat.getWindowInfo().map(WindowInfo::getType);
final Optional<WindowType> rightWnd = right.keyFormat.getWindowInfo().map(WindowInfo::getType);
if (leftWnd.isPresent() != rightWnd.isPresent()) {
throw new IllegalArgumentException("Key format windowing mismatch in join. " + "left: " + leftWnd + ", right: " + rightWnd);
}
final boolean leftIsSession = leftWnd.map(type -> type == WindowType.SESSION).orElse(false);
final boolean rightIsSession = rightWnd.map(type -> type == WindowType.SESSION).orElse(false);
if (leftIsSession != rightIsSession) {
throw new IllegalArgumentException("Key format window type mismatch in join. " + "left: " + (leftIsSession ? "Session Windowed" : "Non Session Windowed") + ", right: " + (rightIsSession ? "Session Windowed" : "Non Session Windowed"));
}
}
use of io.confluent.ksql.serde.SerdeFeatures in project ksql by confluentinc.
the class LogicalPlanner method getSinkTopic.
private KsqlTopic getSinkTopic(final Into into, final LogicalSchema schema) {
if (into.getExistingTopic().isPresent()) {
return into.getExistingTopic().get();
}
final NewTopic newTopic = into.getNewTopic().orElseThrow(IllegalStateException::new);
final FormatInfo keyFormat = getSinkKeyFormat(schema, newTopic);
final SerdeFeatures keyFeatures = SerdeFeaturesFactory.buildKeyFeatures(schema, FormatFactory.of(keyFormat));
final SerdeFeatures valFeatures = SerdeFeaturesFactory.buildValueFeatures(schema, FormatFactory.of(newTopic.getValueFormat()), analysis.getProperties().getValueSerdeFeatures(), ksqlConfig);
return new KsqlTopic(newTopic.getTopicName(), KeyFormat.of(keyFormat, keyFeatures, newTopic.getWindowInfo()), ValueFormat.of(newTopic.getValueFormat(), valFeatures));
}
Aggregations