Examples with LogicalSchema io.confluent.ksql.schema.ksql.LogicalSchema used on opensource projects

Example 61 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class PartitionByParamsFactoryTest method shouldBuildResultSchemaWhenPartitioningByStructField.

@Test
public void shouldBuildResultSchemaWhenPartitioningByStructField() {
    // Given:
    final List<Expression> partitionBy = ImmutableList.of(new DereferenceExpression(Optional.empty(), new UnqualifiedColumnReferenceExp(COL3), "someField"));
    // When:
    final LogicalSchema resultSchema = PartitionByParamsFactory.buildSchema(SCHEMA, partitionBy, functionRegistry);
    // Then:
    assertThat(resultSchema, is(LogicalSchema.builder().keyColumn(ColumnName.of("someField"), SqlTypes.BIGINT).valueColumn(COL1, SqlTypes.INTEGER).valueColumn(COL2, SqlTypes.INTEGER).valueColumn(COL3, COL3_TYPE).valueColumn(SystemColumns.ROWTIME_NAME, SqlTypes.BIGINT).valueColumn(COL0, SqlTypes.STRING).valueColumn(ColumnName.of("someField"), SqlTypes.BIGINT).build()));
}

Example 62 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class PartitionByParamsFactoryTest method shouldBuildResultSchemaWhenPartitioningByNull.

@Test
public void shouldBuildResultSchemaWhenPartitioningByNull() {
    // Given:
    final List<Expression> partitionBy = ImmutableList.of(new NullLiteral());
    // When:
    final LogicalSchema resultSchema = PartitionByParamsFactory.buildSchema(SCHEMA, partitionBy, functionRegistry);
    // Then:
    assertThat(resultSchema, is(LogicalSchema.builder().valueColumn(COL1, SqlTypes.INTEGER).valueColumn(COL2, SqlTypes.INTEGER).valueColumn(COL3, COL3_TYPE).valueColumn(SystemColumns.ROWTIME_NAME, SqlTypes.BIGINT).valueColumn(COL0, SqlTypes.STRING).build()));
}

Example 63 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class TableGroupByBuilderBase method build.

public <K> KGroupedTableHolder build(final KTableHolder<K> table, final QueryContext queryContext, final Formats formats, final List<Expression> groupByExpressions) {
    final LogicalSchema sourceSchema = table.getSchema();
    final List<CompiledExpression> groupBy = CodeGenRunner.compileExpressions(groupByExpressions.stream(), "Group By", sourceSchema, buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final GroupByParams params = paramsFactory.build(sourceSchema, groupBy, logger);
    final PhysicalSchema physicalSchema = PhysicalSchema.from(params.getSchema(), formats.getKeyFeatures(), formats.getValueFeatures());
    final Serde<GenericKey> keySerde = buildContext.buildKeySerde(formats.getKeyFormat(), physicalSchema, queryContext);
    final Serde<GenericRow> valSerde = buildContext.buildValueSerde(formats.getValueFormat(), physicalSchema, queryContext);
    final Grouped<GenericKey, GenericRow> grouped = groupedFactory.create(StreamsUtil.buildOpName(queryContext), keySerde, valSerde);
    final KGroupedTable<GenericKey, GenericRow> groupedTable = table.getTable().filter((k, v) -> v != null).groupBy(new TableKeyValueMapper<>(params.getMapper()), grouped);
    return KGroupedTableHolder.of(groupedTable, params.getSchema());
}

Example 64 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class TableSelectBuilder method build.

@SuppressWarnings("unchecked")
public static <K> KTableHolder<K> build(final KTableHolder<K> table, final TableSelect<K> step, final RuntimeBuildContext buildContext, final Optional<Formats> formats, final MaterializedFactory materializedFactory) {
    final LogicalSchema sourceSchema = table.getSchema();
    final QueryContext queryContext = step.getProperties().getQueryContext();
    final Selection<K> selection = Selection.of(sourceSchema, step.getKeyColumnNames(), step.getSelectExpressions(), buildContext.getKsqlConfig(), buildContext.getFunctionRegistry());
    final SelectValueMapper<K> selectMapper = selection.getMapper();
    final ProcessingLogger logger = buildContext.getProcessingLogger(queryContext);
    final Named selectName = Named.as(StreamsUtil.buildOpName(queryContext));
    final Optional<MaterializationInfo.Builder> matBuilder = table.getMaterializationBuilder();
    final boolean forceMaterialize = !matBuilder.isPresent();
    final Serde<K> keySerde;
    final Serde<GenericRow> valSerde;
    if (formats.isPresent()) {
        final Formats materializationFormat = formats.get();
        final PhysicalSchema physicalSchema = PhysicalSchema.from(selection.getSchema(), materializationFormat.getKeyFeatures(), materializationFormat.getValueFeatures());
        keySerde = (Serde<K>) buildContext.buildKeySerde(materializationFormat.getKeyFormat(), physicalSchema, queryContext);
        valSerde = buildContext.buildValueSerde(materializationFormat.getValueFormat(), physicalSchema, queryContext);
        if (forceMaterialize) {
            final Stacker stacker = Stacker.of(step.getProperties().getQueryContext());
            final String stateStoreName = StreamsUtil.buildOpName(stacker.push(PROJECT_OP).getQueryContext());
            final Materialized<K, GenericRow, KeyValueStore<Bytes, byte[]>> materialized = materializedFactory.create(keySerde, valSerde, stateStoreName);
            final KTable<K, GenericRow> transFormedTable = table.getTable().transformValues(() -> new KsTransformer<>(selectMapper.getTransformer(logger)), materialized);
            return KTableHolder.materialized(transFormedTable, selection.getSchema(), table.getExecutionKeyFactory(), MaterializationInfo.builder(stateStoreName, selection.getSchema()));
        }
    } else {
        keySerde = null;
        valSerde = null;
    }
    final KTable<K, GenericRow> transFormedTable = table.getTable().transformValues(() -> new KsTransformer<>(selectMapper.getTransformer(logger)), Materialized.with(keySerde, valSerde), selectName);
    final Optional<MaterializationInfo.Builder> materialization = matBuilder.map(b -> b.map(pl -> (KsqlTransformer<Object, GenericRow>) selectMapper.getTransformer(pl), selection.getSchema(), queryContext));
    return table.withTable(transFormedTable, selection.getSchema()).withMaterialization(materialization);
}

Example 65 with LogicalSchema

use of io.confluent.ksql.schema.ksql.LogicalSchema in project ksql by confluentinc.

the class GroupByParamsFactoryTest method shouldSetKeyNameFromFieldName.

@Test
public void shouldSetKeyNameFromFieldName() {
    // Given:
    when(groupBy0.getExpression()).thenReturn(new DereferenceExpression(Optional.empty(), new UnqualifiedColumnReferenceExp(COL3), "someField"));
    // When:
    final LogicalSchema schema = GroupByParamsFactory.buildSchema(SOURCE_SCHEMA, ImmutableList.of(groupBy0, groupBy1));
    // Then:
    assertThat(schema, is(LogicalSchema.builder().keyColumn(ColumnName.of("someField"), SqlTypes.INTEGER).keyColumn(ColumnName.of("K1"), SqlTypes.INTEGER).valueColumns(SOURCE_SCHEMA.value()).build()));
}