Examples with Format io.confluent.ksql.serde.Format used on opensource projects

Example 6 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegisterInjector method registerForCreateSource.

private void registerForCreateSource(final ConfiguredStatement<? extends CreateSource> cs) {
    // since this injector is chained after the TopicCreateInjector,
    // we can assume that the kafka topic is always present in the
    // statement properties
    final CreateSource statement = cs.getStatement();
    final LogicalSchema schema = statement.getElements().toLogicalSchema();
    final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(statement.getProperties(), statement.getName());
    final Format keyFormat = tryGetFormat(keyFormatInfo, true, cs.getStatementText());
    final SerdeFeatures keyFeatures = SerdeFeaturesFactory.buildKeyFeatures(schema, keyFormat);
    final FormatInfo valueFormatInfo = SourcePropertiesUtil.getValueFormat(statement.getProperties());
    final Format valueFormat = tryGetFormat(valueFormatInfo, false, cs.getStatementText());
    final SerdeFeatures valFeatures = SerdeFeaturesFactory.buildValueFeatures(schema, valueFormat, statement.getProperties().getValueSerdeFeatures(), cs.getSessionConfig().getConfig(false));
    final SchemaAndId rawKeySchema = (SchemaAndId) cs.getSessionConfig().getOverrides().get(CommonCreateConfigs.KEY_SCHEMA_ID);
    final SchemaAndId rawValueSchema = (SchemaAndId) cs.getSessionConfig().getOverrides().get(CommonCreateConfigs.VALUE_SCHEMA_ID);
    registerSchemas(schema, Pair.of(rawKeySchema, rawValueSchema), statement.getProperties().getKafkaTopic(), keyFormatInfo, keyFeatures, valueFormatInfo, valFeatures, cs.getSessionConfig().getConfig(false), cs.getStatementText(), false);
}

Example 7 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class DefaultSchemaInjector method shouldInferSchema.

@SuppressWarnings({ "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" })
private static boolean shouldInferSchema(final Optional<Integer> schemaId, final ConfiguredStatement<? extends Statement> statement, final FormatInfo formatInfo, final boolean isKey) {
    /*
     * Conditions for schema inference:
     *   1. key_schema_id or value_schema_id property exist or
     *   2. Table elements doesn't exist and format support schema inference for CS/CT
     *
     * Do validation when schemaId presents, so we need to infer schema. Conditions to meet:
     *  1. If schema id is provided, format must support schema inference.
     *  2. Table elements must be empty for CS/CT.
     */
    final String formatProp = isKey ? CommonCreateConfigs.KEY_FORMAT_PROPERTY : CommonCreateConfigs.VALUE_FORMAT_PROPERTY;
    final String schemaIdName = isKey ? CommonCreateConfigs.KEY_SCHEMA_ID : CommonCreateConfigs.VALUE_SCHEMA_ID;
    final String formatPropMsg = String.format("%s should support schema inference when %s is " + "provided. Current format is %s.", formatProp, schemaIdName, formatInfo.getFormat());
    final Format format;
    try {
        format = FormatFactory.of(formatInfo);
    } catch (KsqlException e) {
        if (e.getMessage().contains("does not support the following configs: [schemaId]")) {
            throw new KsqlException(formatPropMsg);
        }
        throw e;
    }
    final boolean cas = statement.getStatement() instanceof CreateAsSelect;
    final boolean hasTableElements;
    if (cas) {
        hasTableElements = false;
    } else {
        hasTableElements = isKey ? hasKeyElements((ConfiguredStatement<CreateSource>) statement) : hasValueElements((ConfiguredStatement<CreateSource>) statement);
    }
    if (schemaId.isPresent()) {
        if (!formatSupportsSchemaInference(format)) {
            throw new KsqlException(formatPropMsg);
        }
        if (hasTableElements) {
            final String msg = "Table elements and " + schemaIdName + " cannot both exist for create " + "statement.";
            throw new KsqlException(msg);
        }
        return true;
    }
    return !cas && !hasTableElements && formatSupportsSchemaInference(format);
}

Example 8 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegistryTopicSchemaSupplier method fromParsedSchema.

private SchemaResult fromParsedSchema(final Optional<String> topic, final int id, final Optional<Integer> schemaId, final ParsedSchema parsedSchema, final FormatInfo expectedFormat, final SerdeFeatures serdeFeatures, final boolean isKey) {
    final Format format = formatFactory.apply(expectedFormat);
    // Topic existence means schema id is not provided original so that default
    // policy should be used
    final SchemaTranslator translator = schemaId.isPresent() ? format.getSchemaTranslator(expectedFormat.getProperties(), SchemaTranslationPolicies.of(SchemaTranslationPolicy.ORIGINAL_FIELD_NAME)) : format.getSchemaTranslator(expectedFormat.getProperties());
    if (!parsedSchema.schemaType().equals(translator.name())) {
        return incorrectFormat(topic, schemaId, translator.name(), parsedSchema.schemaType(), isKey);
    }
    final List<SimpleColumn> columns;
    try {
        columns = translator.toColumns(parsedSchema, serdeFeatures, isKey);
    } catch (final Exception e) {
        return notCompatible(topic, schemaId, parsedSchema.canonicalString(), e, isKey);
    }
    if (isKey && columns.size() > 1) {
        return multiColumnKeysNotSupported(topic, schemaId, parsedSchema.canonicalString());
    }
    return SchemaResult.success(SchemaAndId.schemaAndId(columns, parsedSchema, id));
}

Example 9 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegistryTopicSchemaSupplierTest method verifyFailureMessage.

private void verifyFailureMessage(final SchemaResult result, final boolean isKey, Optional<Integer> id) {
    final String keyOrValue = isKey ? "keys" : "values";
    final String keyOrValueSuffix = isKey ? "key" : "value";
    final String schemaIdMsg = id.map(integer -> "Schema Id: " + integer + System.lineSeparator()).orElse("");
    assertThat(result.failureReason.get().getMessage(), is("Schema for message " + keyOrValue + " on topic '" + TOPIC_NAME + "' does not exist in the Schema Registry." + System.lineSeparator() + "Subject: " + TOPIC_NAME + "-" + keyOrValueSuffix + System.lineSeparator() + schemaIdMsg + "Possible causes include:" + System.lineSeparator() + "- The topic itself does not exist" + System.lineSeparator() + "\t-> Use SHOW TOPICS; to check" + System.lineSeparator() + "- Messages on the topic are not serialized using a format Schema Registry supports" + System.lineSeparator() + "\t-> Use PRINT '" + TOPIC_NAME + "' FROM BEGINNING; to verify" + System.lineSeparator() + "- Messages on the topic have not been serialized using a Confluent Schema Registry supported serializer" + System.lineSeparator() + "\t-> See https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html" + System.lineSeparator() + "- The schema is registered on a different instance of the Schema Registry" + System.lineSeparator() + "\t-> Use the REST API to list available subjects\thttps://docs.confluent.io/current/schema-registry/docs/api.html#get--subjects" + System.lineSeparator() + "- You do not have permissions to access the Schema Registry." + System.lineSeparator() + "\t-> See https://docs.confluent.io/current/schema-registry/docs/security.html"));
}

Example 10 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegistryTopicSchemaSupplierTest method setUp.

@Before
public void setUp() throws Exception {
    supplier = new SchemaRegistryTopicSchemaSupplier(srClient, f -> format);
    when(srClient.getLatestSchemaMetadata(any())).thenReturn(new SchemaMetadata(SCHEMA_ID, -1, AVRO_SCHEMA));
    when(srClient.getSchemaBySubjectAndId(any(), anyInt())).thenReturn(parsedSchema);
    when(parsedSchema.schemaType()).thenReturn(AvroSchema.TYPE);
    when(parsedSchema.canonicalString()).thenReturn(AVRO_SCHEMA);
    when(format.getSchemaTranslator(any())).thenReturn(schemaTranslator);
    when(format.getSchemaTranslator(any(), any())).thenReturn(schemaTranslator);
    when(schemaTranslator.toColumns(eq(parsedSchema), any(), anyBoolean())).thenReturn(ImmutableList.of(column1));
    when(schemaTranslator.name()).thenReturn("AVRO");
    when(expectedFormat.getProperties()).thenReturn(formatProperties);
}