Examples with ValueFormat io.confluent.ksql.serde.ValueFormat used on opensource projects

Example 1 with ValueFormat

use of io.confluent.ksql.serde.ValueFormat in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInSharedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInSharedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final String applicationId, final MetricCollectors metricCollectors) {
    final SharedKafkaStreamsRuntime sharedKafkaStreamsRuntime = getKafkaStreamsInstance(applicationId, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), queryId, metricCollectors);
    final Map<String, Object> queryOverrides = sharedKafkaStreamsRuntime.getStreamProperties();
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final NamedTopologyBuilder namedTopologyBuilder = sharedKafkaStreamsRuntime.getKafkaStreams().newNamedTopologyBuilder(queryId.toString(), PropertiesUtil.asProperties(queryOverrides));
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, namedTopologyBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final NamedTopology topology = namedTopologyBuilder.build();
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, queryOverrides, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, queryOverrides, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    final BinPackedPersistentQueryMetadataImpl binPackedPersistentQueryMetadata = new BinPackedPersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), planSummary, applicationId, topology, sharedKafkaStreamsRuntime, runtimeBuildContext.getSchemas(), config.getOverrides(), queryId, materializationProviderBuilder, physicalPlan, getUncaughtExceptionProcessingLogger(queryId), sinkDataSource, listener, queryOverrides, scalablePushRegistry, (streamsRuntime) -> getNamedTopology(streamsRuntime, queryId, applicationId, queryOverrides, physicalPlan));
    if (real) {
        return binPackedPersistentQueryMetadata;
    } else {
        return SandboxedBinPackedPersistentQueryMetadataImpl.of(binPackedPersistentQueryMetadata, listener);
    }
}

Example 2 with ValueFormat

use of io.confluent.ksql.serde.ValueFormat in project ksql by confluentinc.

the class KsqlAuthorizationValidatorImpl method getCreateAsSelectSinkTopic.

private KsqlTopic getCreateAsSelectSinkTopic(final MetaStore metaStore, final CreateAsSelect createAsSelect) {
    final CreateSourceAsProperties properties = createAsSelect.getProperties();
    final String sinkTopicName;
    final KeyFormat sinkKeyFormat;
    final ValueFormat sinkValueFormat;
    if (!properties.getKafkaTopic().isPresent()) {
        final DataSource dataSource = metaStore.getSource(createAsSelect.getName());
        if (dataSource != null) {
            sinkTopicName = dataSource.getKafkaTopicName();
            sinkKeyFormat = dataSource.getKsqlTopic().getKeyFormat();
            sinkValueFormat = dataSource.getKsqlTopic().getValueFormat();
        } else {
            throw new KsqlException("Cannot validate for topic access from an unknown stream/table: " + createAsSelect.getName());
        }
    } else {
        sinkTopicName = properties.getKafkaTopic().get();
        // If no format is specified for the sink topic, then use the format from the primary
        // source topic.
        final SourceTopicsExtractor extractor = new SourceTopicsExtractor(metaStore);
        extractor.process(createAsSelect.getQuery(), null);
        final KsqlTopic primaryKsqlTopic = extractor.getPrimarySourceTopic();
        final Optional<Format> keyFormat = properties.getKeyFormat().map(formatName -> FormatFactory.fromName(formatName));
        final Optional<Format> valueFormat = properties.getValueFormat().map(formatName -> FormatFactory.fromName(formatName));
        sinkKeyFormat = keyFormat.map(format -> KeyFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of(), Optional.empty())).orElse(primaryKsqlTopic.getKeyFormat());
        sinkValueFormat = valueFormat.map(format -> ValueFormat.of(FormatInfo.of(format.name()), format.supportsFeature(SerdeFeature.SCHEMA_INFERENCE) ? SerdeFeatures.of(SerdeFeature.SCHEMA_INFERENCE) : SerdeFeatures.of())).orElse(primaryKsqlTopic.getValueFormat());
    }
    return new KsqlTopic(sinkTopicName, sinkKeyFormat, sinkValueFormat);
}

Example 3 with ValueFormat

use of io.confluent.ksql.serde.ValueFormat in project ksql by confluentinc.

the class TopicInfoCache method load.

private TopicInfo load(final String topicName) {
    try {
        final Optional<InternalTopic> internalTopic = INTERNAL_TOPIC_PATTERNS.stream().map(e -> e.match(topicName)).filter(Optional::isPresent).map(Optional::get).findFirst();
        if (internalTopic.isPresent()) {
            // Internal topic:
            final QueryId queryId = internalTopic.get().queryId();
            final PersistentQueryMetadata query = ksqlEngine.getPersistentQuery(queryId).orElseThrow(() -> new TestFrameworkException("Unknown queryId for internal topic: " + queryId));
            final QuerySchemas.MultiSchemaInfo schemasInfo = query.getQuerySchemas().getTopicInfo(topicName);
            final Set<KeyFormat> keyFormats = schemasInfo.getKeyFormats();
            final Set<ValueFormat> valueFormats = schemasInfo.getValueFormats();
            // foreign key joins once the above github issue is implemented.
            if (keyFormats.size() != 1) {
                final String result = keyFormats.size() == 0 ? "Zero" : "Multiple";
                throw new Exception(result + " key formats registered for topic." + System.lineSeparator() + "topic: " + topicName + "formats: " + keyFormats.stream().map(KeyFormat::getFormat).sorted().collect(Collectors.toList()));
            }
            // for stream-stream left/outer joins once the above github issue is implemented.
            if (valueFormats.size() > 1) {
                throw new Exception("Multiple value formats registered for topic." + System.lineSeparator() + "topic: " + topicName + "formats: " + valueFormats.stream().map(ValueFormat::getFormat).sorted().collect(Collectors.toList()));
            }
            return new TopicInfo(topicName, query.getLogicalSchema(), internalTopic.get().keyFormat(Iterables.getOnlyElement(keyFormats), query), Iterables.getOnlyElement(valueFormats, NONE_VALUE_FORMAT), internalTopic.get().changeLogWindowSize(query));
        }
        // Source / sink topic:
        final Set<TopicInfo> keyTypes = ksqlEngine.getMetaStore().getAllDataSources().values().stream().filter(source -> source.getKafkaTopicName().equals(topicName)).map(source -> new TopicInfo(topicName, source.getSchema(), source.getKsqlTopic().getKeyFormat(), source.getKsqlTopic().getValueFormat(), OptionalLong.empty())).collect(Collectors.toSet());
        if (keyTypes.isEmpty()) {
            throw new TestFrameworkException("No information found for topic '" + topicName + "'. Available topics: " + cache.asMap().keySet());
        }
        return Iterables.get(keyTypes, 0);
    } catch (final Exception e) {
        throw new TestFrameworkException("Failed to determine key type for" + System.lineSeparator() + "topic: " + topicName + System.lineSeparator() + "reason: " + e.getMessage(), e);
    }
}

Example 4 with ValueFormat

use of io.confluent.ksql.serde.ValueFormat in project ksql by confluentinc.

the class QueryBuilder method buildPersistentQueryInDedicatedRuntime.

@SuppressWarnings("ParameterNumber")
PersistentQueryMetadata buildPersistentQueryInDedicatedRuntime(final KsqlConfig ksqlConfig, final KsqlConstants.PersistentQueryType persistentQueryType, final String statementText, final QueryId queryId, final Optional<DataSource> sinkDataSource, final Set<DataSource> sources, final ExecutionStep<?> physicalPlan, final String planSummary, final QueryMetadata.Listener listener, final Supplier<List<PersistentQueryMetadata>> allPersistentQueries, final StreamsBuilder streamsBuilder, final MetricCollectors metricCollectors) {
    final String applicationId = QueryApplicationId.build(ksqlConfig, true, queryId);
    final Map<String, Object> streamsProperties = buildStreamsProperties(applicationId, Optional.of(queryId), metricCollectors, config.getConfig(true), processingLogContext);
    final LogicalSchema logicalSchema;
    final KeyFormat keyFormat;
    final ValueFormat valueFormat;
    final KsqlTopic ksqlTopic;
    switch(persistentQueryType) {
        // CREATE_SOURCE does not have a sink, so the schema is obtained from the query source
        case CREATE_SOURCE:
            final DataSource dataSource = Iterables.getOnlyElement(sources);
            logicalSchema = dataSource.getSchema();
            keyFormat = dataSource.getKsqlTopic().getKeyFormat();
            valueFormat = dataSource.getKsqlTopic().getValueFormat();
            ksqlTopic = dataSource.getKsqlTopic();
            break;
        default:
            logicalSchema = sinkDataSource.get().getSchema();
            keyFormat = sinkDataSource.get().getKsqlTopic().getKeyFormat();
            valueFormat = sinkDataSource.get().getKsqlTopic().getValueFormat();
            ksqlTopic = sinkDataSource.get().getKsqlTopic();
            break;
    }
    final PhysicalSchema querySchema = PhysicalSchema.from(logicalSchema, keyFormat.getFeatures(), valueFormat.getFeatures());
    final RuntimeBuildContext runtimeBuildContext = buildContext(applicationId, queryId, streamsBuilder);
    final Object result = buildQueryImplementation(physicalPlan, runtimeBuildContext);
    final Topology topology = streamsBuilder.build(PropertiesUtil.asProperties(streamsProperties));
    final Optional<MaterializationProviderBuilderFactory.MaterializationProviderBuilder> materializationProviderBuilder = getMaterializationInfo(result).map(info -> materializationProviderBuilderFactory.materializationProviderBuilder(info, querySchema, keyFormat, streamsProperties, applicationId, queryId.toString()));
    final Optional<ScalablePushRegistry> scalablePushRegistry = applyScalablePushProcessor(querySchema.logicalSchema(), result, allPersistentQueries, streamsProperties, applicationId, ksqlConfig, ksqlTopic, serviceContext);
    return new PersistentQueryMetadataImpl(persistentQueryType, statementText, querySchema, sources.stream().map(DataSource::getName).collect(Collectors.toSet()), sinkDataSource, planSummary, queryId, materializationProviderBuilder, applicationId, topology, kafkaStreamsBuilder, runtimeBuildContext.getSchemas(), streamsProperties, config.getOverrides(), ksqlConfig.getLong(KSQL_SHUTDOWN_TIMEOUT_MS_CONFIG), getConfiguredQueryErrorClassifier(ksqlConfig, applicationId), physicalPlan, ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_ERROR_MAX_QUEUE_SIZE), getUncaughtExceptionProcessingLogger(queryId), ksqlConfig.getLong(KsqlConfig.KSQL_QUERY_RETRY_BACKOFF_INITIAL_MS), ksqlConfig.getLong(KsqlConfig.KSQL_QUERY_RETRY_BACKOFF_MAX_MS), listener, scalablePushRegistry);
}