Examples with Format io.confluent.ksql.serde.Format used on opensource projects

Example 1 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegistryTopicSchemaSupplier method notFound.

private static SchemaResult notFound(final Optional<String> topicName, final Optional<Integer> schemaId, final boolean isKey) {
    final String suffixMsg = "- Messages on the topic have not been serialized using a Confluent " + "Schema Registry supported serializer" + System.lineSeparator() + "\t-> See " + DocumentationLinks.SR_SERIALISER_DOC_URL + System.lineSeparator() + "- The schema is registered on a different instance of the Schema Registry" + System.lineSeparator() + "\t-> Use the REST API to list available subjects" + "\t" + DocumentationLinks.SR_REST_GETSUBJECTS_DOC_URL + System.lineSeparator() + "- You do not have permissions to access the Schema Registry." + System.lineSeparator() + "\t-> See " + DocumentationLinks.SCHEMA_REGISTRY_SECURITY_DOC_URL;
    final String schemaIdMsg = schemaId.map(integer -> "Schema Id: " + integer + System.lineSeparator()).orElse("");
    if (topicName.isPresent()) {
        final String subject = getSRSubject(topicName.get(), isKey);
        return SchemaResult.failure(new KsqlException("Schema for message " + (isKey ? "keys" : "values") + " on topic '" + topicName.get() + "'" + " does not exist in the Schema Registry." + System.lineSeparator() + "Subject: " + subject + System.lineSeparator() + schemaIdMsg + "Possible causes include:" + System.lineSeparator() + "- The topic itself does not exist" + System.lineSeparator() + "\t-> Use SHOW TOPICS; to check" + System.lineSeparator() + "- Messages on the topic are not serialized using a format Schema Registry supports" + System.lineSeparator() + "\t-> Use PRINT '" + topicName.get() + "' FROM BEGINNING; to verify" + System.lineSeparator() + suffixMsg));
    }
    return SchemaResult.failure(new KsqlException("Schema for message " + (isKey ? "keys" : "values") + " on schema id'" + schemaId.get() + " does not exist in the Schema Registry." + System.lineSeparator() + schemaIdMsg + "Possible causes include:" + System.lineSeparator() + "- Schema Id " + schemaId + System.lineSeparator() + suffixMsg));
}

Example 2 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class KsqlAuthorizationValidatorImpl method getCreateAsSelectSinkTopic.

private KsqlTopic getCreateAsSelectSinkTopic(final MetaStore metaStore, final CreateAsSelect createAsSelect) {
    final CreateSourceAsProperties properties = createAsSelect.getProperties();
    final String sinkTopicName;
    final KeyFormat sinkKeyFormat;
    final ValueFormat sinkValueFormat;
    if (!properties.getKafkaTopic().isPresent()) {
        final DataSource dataSource = metaStore.getSource(createAsSelect.getName());
        if (dataSource != null) {
            sinkTopicName = dataSource.getKafkaTopicName();
            sinkKeyFormat = dataSource.getKsqlTopic().getKeyFormat();
            sinkValueFormat = dataSource.getKsqlTopic().getValueFormat();
        } else {
            throw new KsqlException("Cannot validate for topic access from an unknown stream/table: " + createAsSelect.getName());
        }
    } else {
        sinkTopicName = properties.getKafkaTopic().get();
        // If no format is specified for the sink topic, then use the format from the primary
        // source topic.
        final SourceTopicsExtractor extractor = new SourceTopicsExtractor(metaStore);
        extractor.process(createAsSelect.getQuery(), null);
        final KsqlTopic primaryKsqlTopic = extractor.getPrimarySourceTopic();
        final Optional<Format> keyFormat = properties.getKeyFormat().map(formatName -> FormatFactory.fromName(formatName));
        final Optional<Format> valueFormat = properties.getValueFormat().map(formatName -> FormatFactory.fromName(formatName));
        sinkKeyFormat = keyFormat.map(format -> KeyFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of(), Optional.empty())).orElse(primaryKsqlTopic.getKeyFormat());
        sinkValueFormat = valueFormat.map(format -> ValueFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of())).orElse(primaryKsqlTopic.getValueFormat());
    }
    return new KsqlTopic(sinkTopicName, sinkKeyFormat, sinkValueFormat);
}

Example 3 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegisterInjector method sanityCheck.

private static void sanityCheck(final SchemaAndId schemaAndId, final FormatInfo formatInfo, final String topic, final KsqlConfig config, final String statementText, final boolean isKey) {
    final String schemaIdPropStr = isKey ? CommonCreateConfigs.KEY_SCHEMA_ID : CommonCreateConfigs.VALUE_SCHEMA_ID;
    final Format format = FormatFactory.of(formatInfo);
    if (!canRegister(format, config, topic)) {
        throw new KsqlStatementException(schemaIdPropStr + " is provided but format " + format.name() + " doesn't support registering in Schema Registry", statementText);
    }
    final SchemaTranslator translator = format.getSchemaTranslator(formatInfo.getProperties());
    if (!translator.name().equals(schemaAndId.rawSchema.schemaType())) {
        throw new KsqlStatementException(String.format("Format and fetched schema type using %s %d are different. Format: [%s], " + "Fetched schema type: [%s].", schemaIdPropStr, schemaAndId.id, format.name(), schemaAndId.rawSchema.schemaType()), statementText);
    }
}

Example 4 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class SchemaRegisterInjector method registerSchema.

private void registerSchema(final List<? extends SimpleColumn> schema, final String topic, final FormatInfo formatInfo, final SerdeFeatures serdeFeatures, final KsqlConfig config, final String statementText, final boolean registerIfSchemaExists, final String subject, final boolean isKey) {
    final Format format = FormatFactory.of(formatInfo);
    if (!canRegister(format, config, topic)) {
        return;
    }
    final SchemaRegistryClient srClient = serviceContext.getSchemaRegistryClient();
    if (registerIfSchemaExists || !SchemaRegistryUtil.subjectExists(srClient, subject)) {
        try {
            final SchemaTranslator translator = format.getSchemaTranslator(formatInfo.getProperties());
            final ParsedSchema parsedSchema = translator.toParsedSchema(PersistenceSchema.from(schema, serdeFeatures));
            SchemaRegistryUtil.registerSchema(srClient, parsedSchema, topic, subject, isKey);
        } catch (KsqlException e) {
            throw new KsqlStatementException("Could not register schema for topic: " + e.getMessage(), statementText, e);
        }
    }
}

Example 5 with Format

use of io.confluent.ksql.serde.Format in project ksql by confluentinc.

the class TestCaseBuilderUtil method createTopicFromStatement.

private static Topic createTopicFromStatement(final String sql, final MutableMetaStore metaStore, final KsqlConfig ksqlConfig) {
    final KsqlParser parser = new DefaultKsqlParser();
    final Function<ConfiguredStatement<?>, Topic> extractTopic = (ConfiguredStatement<?> stmt) -> {
        final CreateSource statement = (CreateSource) stmt.getStatement();
        final CreateSourceProperties props = statement.getProperties();
        final LogicalSchema logicalSchema = statement.getElements().toLogicalSchema();
        final FormatInfo keyFormatInfo = SourcePropertiesUtil.getKeyFormat(props, statement.getName());
        final Format keyFormat = FormatFactory.fromName(keyFormatInfo.getFormat());
        final SerdeFeatures keySerdeFeats = buildKeyFeatures(keyFormat, logicalSchema);
        final Optional<ParsedSchema> keySchema = keyFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.key(), keyFormatInfo, keyFormat, keySerdeFeats) : Optional.empty();
        final FormatInfo valFormatInfo = SourcePropertiesUtil.getValueFormat(props);
        final Format valFormat = FormatFactory.fromName(valFormatInfo.getFormat());
        final SerdeFeatures valSerdeFeats = buildValueFeatures(ksqlConfig, props, valFormat, logicalSchema);
        final Optional<ParsedSchema> valueSchema = valFormat.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? buildSchema(sql, logicalSchema.value(), valFormatInfo, valFormat, valSerdeFeats) : Optional.empty();
        final int partitions = props.getPartitions().orElse(Topic.DEFAULT_PARTITIONS);
        final short rf = props.getReplicas().orElse(Topic.DEFAULT_RF);
        return new Topic(props.getKafkaTopic(), partitions, rf, keySchema, valueSchema, keySerdeFeats, valSerdeFeats);
    };
    try {
        final List<ParsedStatement> parsed = parser.parse(sql);
        if (parsed.size() > 1) {
            throw new IllegalArgumentException("SQL contains more than one statement: " + sql);
        }
        final List<Topic> topics = new ArrayList<>();
        for (ParsedStatement stmt : parsed) {
            // in order to extract the topics, we may need to also register type statements
            if (stmt.getStatement().statement() instanceof SqlBaseParser.RegisterTypeContext) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                registerType(prepare, metaStore);
            }
            if (isCsOrCT(stmt)) {
                final PreparedStatement<?> prepare = parser.prepare(stmt, metaStore);
                final ConfiguredStatement<?> configured = ConfiguredStatement.of(prepare, SessionConfig.of(ksqlConfig, Collections.emptyMap()));
                final ConfiguredStatement<?> withFormats = new DefaultFormatInjector().inject(configured);
                topics.add(extractTopic.apply(withFormats));
            }
        }
        return topics.isEmpty() ? null : topics.get(0);
    } catch (final Exception e) {
        // Statement won't parse: this will be detected/handled later.
        System.out.println("Error parsing statement (which may be expected): " + sql);
        e.printStackTrace(System.out);
        return null;
    }
}