Examples with FormatInfo io.confluent.ksql.serde.FormatInfo used on opensource projects

Example 6 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class LogicalPlanner method getSinkTopic.

private KsqlTopic getSinkTopic(final Into into, final LogicalSchema schema) {
    if (into.getExistingTopic().isPresent()) {
        return into.getExistingTopic().get();
    }
    final NewTopic newTopic = into.getNewTopic().orElseThrow(IllegalStateException::new);
    final FormatInfo keyFormat = getSinkKeyFormat(schema, newTopic);
    final SerdeFeatures keyFeatures = SerdeFeaturesFactory.buildKeyFeatures(schema, FormatFactory.of(keyFormat));
    final SerdeFeatures valFeatures = SerdeFeaturesFactory.buildValueFeatures(schema, FormatFactory.of(newTopic.getValueFormat()), analysis.getProperties().getValueSerdeFeatures(), ksqlConfig);
    return new KsqlTopic(newTopic.getTopicName(), KeyFormat.of(keyFormat, keyFeatures, newTopic.getWindowInfo()), ValueFormat.of(newTopic.getValueFormat(), valFeatures));
}

Example 7 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class CreateSourcePropertiesTest method shouldSetValueFullSchemaName.

@Test
public void shouldSetValueFullSchemaName() {
    // When:
    final CreateSourceProperties properties = CreateSourceProperties.from(ImmutableMap.<String, Literal>builder().putAll(MINIMUM_VALID_PROPS).put(CommonCreateConfigs.VALUE_FORMAT_PROPERTY, new StringLiteral("Protobuf")).put(CommonCreateConfigs.VALUE_SCHEMA_FULL_NAME, new StringLiteral("schema")).build());
    // Then:
    assertThat(properties.getValueFormat().map(FormatInfo::getProperties).map(props -> props.get(ConnectProperties.FULL_SCHEMA_NAME)), is(Optional.of("schema")));
}

Example 8 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class StreamAggregateBuilderTest method shouldReturnCorrectSerdeForWindowedAggregate.

@Test
public void shouldReturnCorrectSerdeForWindowedAggregate() {
    for (final Runnable given : given()) {
        // Given:
        clearInvocations(groupedStream, timeWindowedStream, sessionWindowedStream, aggregated, buildContext);
        given.run();
        // When:
        final KTableHolder<Windowed<GenericKey>> tableHolder = windowedAggregate.build(planBuilder, planInfo);
        // Then:
        final ExecutionKeyFactory<Windowed<GenericKey>> serdeFactory = tableHolder.getExecutionKeyFactory();
        final FormatInfo mockFormat = mock(FormatInfo.class);
        final PhysicalSchema mockSchema = mock(PhysicalSchema.class);
        final QueryContext mockCtx = mock(QueryContext.class);
        serdeFactory.buildKeySerde(mockFormat, mockSchema, mockCtx);
        verify(buildContext).buildKeySerde(same(mockFormat), eq(windowedAggregate.getWindowExpression().getWindowInfo()), same(mockSchema), same(mockCtx));
    }
}

Example 9 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class SchemaKGroupedTable method aggregate.

@Override
public SchemaKTable<GenericKey> aggregate(final List<ColumnName> nonAggregateColumns, final List<FunctionCall> aggregations, final Optional<WindowExpression> windowExpression, final FormatInfo valueFormat, final Stacker contextStacker) {
    if (windowExpression.isPresent()) {
        throw new KsqlException("Windowing not supported for table aggregations.");
    }
    final List<String> unsupportedFunctionNames = aggregations.stream().map(call -> UdafUtil.resolveAggregateFunction(functionRegistry, call, schema, ksqlConfig)).filter(function -> !(function instanceof TableAggregationFunction)).map(KsqlAggregateFunction::name).map(FunctionName::text).distinct().collect(Collectors.toList());
    if (!unsupportedFunctionNames.isEmpty()) {
        final String postfix = unsupportedFunctionNames.size() == 1 ? "" : "s";
        throw new KsqlException("The aggregation function" + postfix + " " + GrammaticalJoiner.and().join(unsupportedFunctionNames) + " cannot be applied to a table source, only to a stream source.");
    }
    final TableAggregate step = ExecutionStepFactory.tableAggregate(contextStacker, sourceTableStep, InternalFormats.of(keyFormat, valueFormat), nonAggregateColumns, aggregations);
    return new SchemaKTable<>(step, resolveSchema(step), keyFormat, ksqlConfig, functionRegistry);
}

Example 10 with FormatInfo

use of io.confluent.ksql.serde.FormatInfo in project ksql by confluentinc.

the class SchemaKStream method groupBy.

public SchemaKGroupedStream groupBy(final FormatInfo valueFormat, final List<Expression> groupByExpressions, final Stacker contextStacker) {
    if (!repartitionNeeded(groupByExpressions)) {
        return groupByKey(keyFormat, valueFormat, contextStacker);
    }
    final KeyFormat rekeyedFormat;
    if (keyFormat.getFormatInfo().getFormat().equals(NoneFormat.NAME)) {
        final FormatInfo defaultFormat = FormatInfo.of(ksqlConfig.getString(KsqlConfig.KSQL_DEFAULT_KEY_FORMAT_CONFIG));
        rekeyedFormat = KeyFormat.nonWindowed(defaultFormat, SerdeFeatures.of());
    } else {
        rekeyedFormat = keyFormat;
    }
    final KeyFormat sanitizedKeyFormat = SerdeFeaturesFactory.sanitizeKeyFormat(rekeyedFormat, toSqlTypes(groupByExpressions), true);
    final StreamGroupBy<K> source = ExecutionStepFactory.streamGroupBy(contextStacker, sourceStep, InternalFormats.of(sanitizedKeyFormat, valueFormat), groupByExpressions);
    return new SchemaKGroupedStream(source, resolveSchema(source), sanitizedKeyFormat, ksqlConfig, functionRegistry);
}