Examples with SchemaTranslator io.confluent.ksql.serde.SchemaTranslator used on opensource projects

Example 1 with SchemaTranslator

use of io.confluent.ksql.serde.SchemaTranslator in project ksql by confluentinc.

the class SchemaRegisterInjector method sanityCheck.

private static void sanityCheck(final SchemaAndId schemaAndId, final FormatInfo formatInfo, final String topic, final KsqlConfig config, final String statementText, final boolean isKey) {
    final String schemaIdPropStr = isKey ? CommonCreateConfigs.KEY_SCHEMA_ID : CommonCreateConfigs.VALUE_SCHEMA_ID;
    final Format format = FormatFactory.of(formatInfo);
    if (!canRegister(format, config, topic)) {
        throw new KsqlStatementException(schemaIdPropStr + " is provided but format " + format.name() + " doesn't support registering in Schema Registry", statementText);
    }
    final SchemaTranslator translator = format.getSchemaTranslator(formatInfo.getProperties());
    if (!translator.name().equals(schemaAndId.rawSchema.schemaType())) {
        throw new KsqlStatementException(String.format("Format and fetched schema type using %s %d are different. Format: [%s], " + "Fetched schema type: [%s].", schemaIdPropStr, schemaAndId.id, format.name(), schemaAndId.rawSchema.schemaType()), statementText);
    }
}

Example 2 with SchemaTranslator

use of io.confluent.ksql.serde.SchemaTranslator in project ksql by confluentinc.

the class SchemaRegisterInjector method registerSchema.

private void registerSchema(final List<? extends SimpleColumn> schema, final String topic, final FormatInfo formatInfo, final SerdeFeatures serdeFeatures, final KsqlConfig config, final String statementText, final boolean registerIfSchemaExists, final String subject, final boolean isKey) {
    final Format format = FormatFactory.of(formatInfo);
    if (!canRegister(format, config, topic)) {
        return;
    }
    final SchemaRegistryClient srClient = serviceContext.getSchemaRegistryClient();
    if (registerIfSchemaExists || !SchemaRegistryUtil.subjectExists(srClient, subject)) {
        try {
            final SchemaTranslator translator = format.getSchemaTranslator(formatInfo.getProperties());
            final ParsedSchema parsedSchema = translator.toParsedSchema(PersistenceSchema.from(schema, serdeFeatures));
            SchemaRegistryUtil.registerSchema(srClient, parsedSchema, topic, subject, isKey);
        } catch (KsqlException e) {
            throw new KsqlStatementException("Could not register schema for topic: " + e.getMessage(), statementText, e);
        }
    }
}

Example 3 with SchemaTranslator

use of io.confluent.ksql.serde.SchemaTranslator in project ksql by confluentinc.

the class KsqlResourceFunctionalTest method shouldInsertIntoValuesForAvroTopic.

@Test
public void shouldInsertIntoValuesForAvroTopic() throws Exception {
    // Given:
    final PhysicalSchema schema = PhysicalSchema.from(LogicalSchema.builder().keyColumn(ColumnName.of("AUTHOR"), SqlTypes.STRING).valueColumn(ColumnName.of("TITLE"), SqlTypes.STRING).build(), SerdeFeatures.of(SerdeFeature.UNWRAP_SINGLES), SerdeFeatures.of());
    final SchemaTranslator translator = new AvroFormat().getSchemaTranslator(ImmutableMap.of(ConnectProperties.FULL_SCHEMA_NAME, "books_value"));
    final ParsedSchema keySchema = translator.toParsedSchema(PersistenceSchema.from(schema.logicalSchema().key(), schema.keySchema().features()));
    TEST_HARNESS.getSchemaRegistryClient().register(KsqlConstants.getSRSubject("books", true), keySchema);
    final ParsedSchema valueSchema = translator.toParsedSchema(PersistenceSchema.from(schema.logicalSchema().value(), schema.valueSchema().features()));
    TEST_HARNESS.getSchemaRegistryClient().register(KsqlConstants.getSRSubject("books", false), valueSchema);
    // When:
    final List<KsqlEntity> results = makeKsqlRequest("" + "CREATE STREAM books (author VARCHAR KEY, title VARCHAR) " + "WITH (kafka_topic='books', format='avro', partitions=1);" + " " + "INSERT INTO BOOKS (ROWTIME, author, title) VALUES (123, 'Metamorphosis', 'Franz Kafka');");
    // Then:
    assertSuccessful(results);
    TEST_HARNESS.verifyAvailableRows("books", contains(matches(genericKey("Metamorphosis"), genericRow("Franz Kafka"), 0, 0L, 123L)), FormatFactory.AVRO, FormatFactory.AVRO, schema);
}

Example 4 with SchemaTranslator

use of io.confluent.ksql.serde.SchemaTranslator in project ksql by confluentinc.

the class IntegrationTestHarness method ensureSchema.

public void ensureSchema(final String topicName, final PhysicalSchema schema, final boolean keySchema) {
    final SchemaRegistryClient srClient = serviceContext.get().getSchemaRegistryClient();
    try {
        final Map<String, String> formatProps = ImmutableMap.of(ConnectProperties.FULL_SCHEMA_NAME, "test_" + topicName);
        final SchemaTranslator translator = new AvroFormat().getSchemaTranslator(formatProps);
        final ParsedSchema parsedSchema = translator.toParsedSchema(PersistenceSchema.from(keySchema ? schema.logicalSchema().key() : schema.logicalSchema().value(), keySchema ? schema.keySchema().features() : schema.valueSchema().features()));
        srClient.register(KsqlConstants.getSRSubject(topicName, keySchema), parsedSchema);
    } catch (final Exception e) {
        throw new AssertionError(e);
    }
}

Example 5 with SchemaTranslator

use of io.confluent.ksql.serde.SchemaTranslator in project ksql by confluentinc.

the class SchemaRegistryTopicSchemaSupplier method fromParsedSchema.

private SchemaResult fromParsedSchema(final Optional<String> topic, final int id, final Optional<Integer> schemaId, final ParsedSchema parsedSchema, final FormatInfo expectedFormat, final SerdeFeatures serdeFeatures, final boolean isKey) {
    final Format format = formatFactory.apply(expectedFormat);
    // Topic existence means schema id is not provided original so that default
    // policy should be used
    final SchemaTranslator translator = schemaId.isPresent() ? format.getSchemaTranslator(expectedFormat.getProperties(), SchemaTranslationPolicies.of(SchemaTranslationPolicy.ORIGINAL_FIELD_NAME)) : format.getSchemaTranslator(expectedFormat.getProperties());
    if (!parsedSchema.schemaType().equals(translator.name())) {
        return incorrectFormat(topic, schemaId, translator.name(), parsedSchema.schemaType(), isKey);
    }
    final List<SimpleColumn> columns;
    try {
        columns = translator.toColumns(parsedSchema, serdeFeatures, isKey);
    } catch (final Exception e) {
        return notCompatible(topic, schemaId, parsedSchema.canonicalString(), e, isKey);
    }
    if (isKey && columns.size() > 1) {
        return multiColumnKeysNotSupported(topic, schemaId, parsedSchema.canonicalString());
    }
    return SchemaResult.success(SchemaAndId.schemaAndId(columns, parsedSchema, id));
}