use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.
the class KafkaConsumerFactory method create.
public static KafkaConsumer<Object, GenericRow> create(final KsqlTopic ksqlTopic, final LogicalSchema logicalSchema, final ServiceContext serviceContext, final Map<String, Object> consumerProperties, final KsqlConfig ksqlConfig, final String consumerGroupId) {
final PhysicalSchema physicalSchema = PhysicalSchema.from(logicalSchema, ksqlTopic.getKeyFormat().getFeatures(), ksqlTopic.getValueFormat().getFeatures());
final KeySerdeFactory keySerdeFactory = new GenericKeySerDe();
final Deserializer<Object> keyDeserializer;
if (ksqlTopic.getKeyFormat().getWindowInfo().isPresent()) {
final Serde<Windowed<GenericKey>> keySerde = keySerdeFactory.create(ksqlTopic.getKeyFormat().getFormatInfo(), ksqlTopic.getKeyFormat().getWindowInfo().get(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
keyDeserializer = getDeserializer(keySerde.deserializer());
} else {
final Serde<GenericKey> keySerde = keySerdeFactory.create(ksqlTopic.getKeyFormat().getFormatInfo(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
keyDeserializer = getDeserializer(keySerde.deserializer());
}
final ValueSerdeFactory valueSerdeFactory = new GenericRowSerDe();
final Serde<GenericRow> valueSerde = valueSerdeFactory.create(ksqlTopic.getValueFormat().getFormatInfo(), physicalSchema.valueSchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
return new KafkaConsumer<>(consumerConfig(consumerProperties, ksqlConfig, consumerGroupId), keyDeserializer, valueSerde.deserializer());
}
use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.
the class InsertValuesExecutor method serializeValue.
private byte[] serializeValue(final GenericRow row, final DataSource dataSource, final KsqlConfig config, final ServiceContext serviceContext) {
final PhysicalSchema physicalSchema = PhysicalSchema.from(dataSource.getSchema(), dataSource.getKsqlTopic().getKeyFormat().getFeatures(), dataSource.getKsqlTopic().getValueFormat().getFeatures());
final Serde<GenericRow> valueSerde = valueSerdeFactory.create(dataSource.getKsqlTopic().getValueFormat().getFormatInfo(), physicalSchema.valueSchema(), config, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
final String topicName = dataSource.getKafkaTopicName();
try {
return valueSerde.serializer().serialize(topicName, row);
} catch (final Exception e) {
maybeThrowSchemaRegistryAuthError(FormatFactory.fromName(dataSource.getKsqlTopic().getValueFormat().getFormat()), topicName, false, AclOperation.WRITE, e);
LOG.error("Could not serialize value.", e);
throw new KsqlException("Could not serialize value: " + row + ". " + e.getMessage(), e);
}
}
use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.
the class InsertValuesExecutor method serializeKey.
private byte[] serializeKey(final GenericKey keyValue, final DataSource dataSource, final KsqlConfig config, final ServiceContext serviceContext) {
final PhysicalSchema physicalSchema = PhysicalSchema.from(dataSource.getSchema(), dataSource.getKsqlTopic().getKeyFormat().getFeatures(), dataSource.getKsqlTopic().getValueFormat().getFeatures());
ensureKeySchemasMatch(physicalSchema.keySchema(), dataSource, serviceContext);
final Serde<GenericKey> keySerde = keySerdeFactory.create(dataSource.getKsqlTopic().getKeyFormat().getFormatInfo(), physicalSchema.keySchema(), config, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
final String topicName = dataSource.getKafkaTopicName();
try {
return keySerde.serializer().serialize(topicName, keyValue);
} catch (final Exception e) {
maybeThrowSchemaRegistryAuthError(FormatFactory.fromName(dataSource.getKsqlTopic().getKeyFormat().getFormat()), topicName, true, AclOperation.WRITE, e);
LOG.error("Could not serialize key.", e);
throw new KsqlException("Could not serialize key: " + keyValue, e);
}
}
use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.
the class CreateSourceFactory method validateSerdesCanHandleSchemas.
private void validateSerdesCanHandleSchemas(final KsqlConfig ksqlConfig, final LogicalSchema schema, final Formats formats) {
final PhysicalSchema physicalSchema = PhysicalSchema.from(schema, formats.getKeyFeatures(), formats.getValueFeatures());
keySerdeFactory.create(formats.getKeyFormat(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty()).close();
valueSerdeFactory.create(formats.getValueFormat(), physicalSchema.valueSchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty()).close();
}
use of io.confluent.ksql.schema.ksql.PhysicalSchema in project ksql by confluentinc.
the class TableGroupByBuilderBase method build.
public <K> KGroupedTableHolder build(final KTableHolder<K> table, final QueryContext queryContext, final Formats formats, final List<Expression> groupByExpressions) {
final LogicalSchema sourceSchema = table.getSchema();
final List<CompiledExpression> groupBy = CodeGenRunner.compileExpressions(groupByExpressions.stream(), "Group By", sourceSchema, buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
final GroupByParams params = paramsFactory.build(sourceSchema, groupBy, logger);
final PhysicalSchema physicalSchema = PhysicalSchema.from(params.getSchema(), formats.getKeyFeatures(), formats.getValueFeatures());
final Serde<GenericKey> keySerde = buildContext.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
final Serde<GenericRow> valSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
final Grouped<GenericKey, GenericRow> grouped = groupedFactory.create(StreamsUtil.buildOpName(queryContext), keySerde, valSerde);
final KGroupedTable<GenericKey, GenericRow> groupedTable = table.getTable().filter((k, v) -> v != null).groupBy(new TableKeyValueMapper<>(params.getMapper()), grouped);
return KGroupedTableHolder.of(groupedTable, params.getSchema());
}
Aggregations