Examples with FormatInfo io.confluent.ksql.serde.FormatInfo used on opensource projects

Example 11 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class SchemaKStream method throwOnJoinKeyFormatsMismatch.

void throwOnJoinKeyFormatsMismatch(final SchemaKStream<?> right) {
    final FormatInfo leftFmt = this.keyFormat.getFormatInfo();
    final FormatInfo rightFmt = right.keyFormat.getFormatInfo();
    if (!leftFmt.equals(rightFmt)) {
        throw new IllegalArgumentException("Key format mismatch in join. " + "left: " + leftFmt + ", right: " + rightFmt);
    }
    final SerdeFeatures leftFeats = this.keyFormat.getFeatures();
    final SerdeFeatures rightFeats = right.keyFormat.getFeatures();
    if (!leftFeats.equals(rightFeats)) {
        throw new IllegalArgumentException("Key format features mismatch in join. " + "left: " + leftFeats + ", right: " + rightFeats);
    }
    final Optional<WindowType> leftWnd = this.keyFormat.getWindowInfo().map(WindowInfo::getType);
    final Optional<WindowType> rightWnd = right.keyFormat.getWindowInfo().map(WindowInfo::getType);
    if (leftWnd.isPresent() != rightWnd.isPresent()) {
        throw new IllegalArgumentException("Key format windowing mismatch in join. " + "left: " + leftWnd + ", right: " + rightWnd);
    }
    final boolean leftIsSession = leftWnd.map(type -> type == WindowType.SESSION).orElse(false);
    final boolean rightIsSession = rightWnd.map(type -> type == WindowType.SESSION).orElse(false);
    if (leftIsSession != rightIsSession) {
        throw new IllegalArgumentException("Key format window type mismatch in join. " + "left: " + (leftIsSession ? "Session Windowed" : "Non Session Windowed") + ", right: " + (rightIsSession ? "Session Windowed" : "Non Session Windowed"));
    }
}

Example 12 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class DefaultSchemaInjector method getCreateAsKeySchema.

private Optional<SchemaAndId> getCreateAsKeySchema(final ConfiguredStatement<CreateAsSelect> statement, final CreateSourceCommand createSourceCommand) {
    final CreateAsSelect csStmt = statement.getStatement();
    final CreateSourceAsProperties props = csStmt.getProperties();
    final FormatInfo keyFormat = createSourceCommand.getFormats().getKeyFormat();
    if (!shouldInferSchema(props.getKeySchemaId(), statement, keyFormat, true)) {
        return Optional.empty();
    }
    // until we support user-configuration of single key wrapping/unwrapping, we choose
    // to have key schema inference always result in an unwrapped key
    final SerdeFeatures serdeFeatures = SerdeFeaturesFactory.buildKeyFeatures(FormatFactory.of(keyFormat), true);
    if (!shouldInferSchema(props.getKeySchemaId(), statement, keyFormat, true)) {
        return Optional.empty();
    }
    final SchemaAndId schemaAndId = getSchema(props.getKafkaTopic(), props.getKeySchemaId(), keyFormat, serdeFeatures, statement.getStatementText(), true);
    final List<Column> tableColumns = createSourceCommand.getSchema().key();
    checkColumnsCompatibility(props.getKeySchemaId(), tableColumns, schemaAndId.columns, true);
    return Optional.of(schemaAndId);
}

Example 13 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class DefaultSchemaInjector method getKeySchema.

private Optional<SchemaAndId> getKeySchema(final ConfiguredStatement<CreateSource> statement) {
    final CreateSource csStmt = statement.getStatement();
    final CreateSourceProperties props = csStmt.getProperties();
    final FormatInfo keyFormat = SourcePropertiesUtil.getKeyFormat(props, csStmt.getName());
    if (!shouldInferSchema(props.getKeySchemaId(), statement, keyFormat, true)) {
        return Optional.empty();
    }
    return Optional.of(getSchema(Optional.of(props.getKafkaTopic()), props.getKeySchemaId(), keyFormat, // to have key schema inference always result in an unwrapped key
    SerdeFeaturesFactory.buildKeyFeatures(FormatFactory.of(keyFormat), true), statement.getStatementText(), true));
}

Example 14 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class DefaultSchemaInjector method getCreateAsValueSchema.

private Optional<SchemaAndId> getCreateAsValueSchema(final ConfiguredStatement<CreateAsSelect> statement, final CreateSourceCommand createSourceCommand) {
    final CreateAsSelect csStmt = statement.getStatement();
    final CreateSourceAsProperties props = csStmt.getProperties();
    final FormatInfo valueFormat = createSourceCommand.getFormats().getValueFormat();
    if (!shouldInferSchema(props.getValueSchemaId(), statement, valueFormat, false)) {
        return Optional.empty();
    }
    final SchemaAndId schemaAndId = getSchema(props.getKafkaTopic(), props.getValueSchemaId(), valueFormat, createSourceCommand.getFormats().getValueFeatures(), statement.getStatementText(), false);
    final List<Column> tableColumns = createSourceCommand.getSchema().value();
    checkColumnsCompatibility(props.getValueSchemaId(), tableColumns, schemaAndId.columns, false);
    return Optional.of(schemaAndId);
}

Example 15 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class SchemaRegisterInjector method registerForCreateSource.

private void registerForCreateSource(final ConfiguredStatement<? extends CreateSource> cs) {
    // since this injector is chained after the TopicCreateInjector,
    // we can assume that the kafka topic is always present in the
    // statement properties
    final CreateSource statement = cs.getStatement();
    final LogicalSchema schema = statement.getElements().toLogicalSchema();
    final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(statement.getProperties(), statement.getName());
    final Format keyFormat = tryGetFormat(keyFormatInfo, true, cs.getStatementText());
    final SerdeFeatures keyFeatures = SerdeFeaturesFactory.buildKeyFeatures(schema, keyFormat);
    final FormatInfo valueFormatInfo = SourcePropertiesUtil.getValueFormat(statement.getProperties());
    final Format valueFormat = tryGetFormat(valueFormatInfo, false, cs.getStatementText());
    final SerdeFeatures valFeatures = SerdeFeaturesFactory.buildValueFeatures(schema, valueFormat, statement.getProperties().getValueSerdeFeatures(), cs.getSessionConfig().getConfig(false));
    final SchemaAndId rawKeySchema = (SchemaAndId) cs.getSessionConfig().getOverrides().get(CommonCreateConfigs.KEY_SCHEMA_ID);
    final SchemaAndId rawValueSchema = (SchemaAndId) cs.getSessionConfig().getOverrides().get(CommonCreateConfigs.VALUE_SCHEMA_ID);
    registerSchemas(schema, Pair.of(rawKeySchema, rawValueSchema), statement.getProperties().getKafkaTopic(), keyFormatInfo, keyFeatures, valueFormatInfo, valFeatures, cs.getSessionConfig().getConfig(false), cs.getStatementText(), false);
}