Search in sources :

Example 1 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class KafkaConsumerFactory method create.

public static KafkaConsumer<Object, GenericRow> create(final KsqlTopic ksqlTopic, final LogicalSchema logicalSchema, final ServiceContext serviceContext, final Map<String, Object> consumerProperties, final KsqlConfig ksqlConfig, final String consumerGroupId) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(logicalSchema, ksqlTopic.getKeyFormat().getFeatures(), ksqlTopic.getValueFormat().getFeatures());
    final KeySerdeFactory keySerdeFactory = new GenericKeySerDe();
    final Deserializer<Object> keyDeserializer;
    if (ksqlTopic.getKeyFormat().getWindowInfo().isPresent()) {
        final Serde<Windowed<GenericKey>> keySerde = keySerdeFactory.create(ksqlTopic.getKeyFormat().getFormatInfo(), ksqlTopic.getKeyFormat().getWindowInfo().get(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
        keyDeserializer = getDeserializer(keySerde.deserializer());
    } else {
        final Serde<GenericKey> keySerde = keySerdeFactory.create(ksqlTopic.getKeyFormat().getFormatInfo(), physicalSchema.keySchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
        keyDeserializer = getDeserializer(keySerde.deserializer());
    }
    final ValueSerdeFactory valueSerdeFactory = new GenericRowSerDe();
    final Serde<GenericRow> valueSerde = valueSerdeFactory.create(ksqlTopic.getValueFormat().getFormatInfo(), physicalSchema.valueSchema(), ksqlConfig, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
    return new KafkaConsumer<>(consumerConfig(consumerProperties, ksqlConfig, consumerGroupId), keyDeserializer, valueSerde.deserializer());
}
Also used : ValueSerdeFactory(io.confluent.ksql.serde.ValueSerdeFactory) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) GenericKeySerDe(io.confluent.ksql.serde.GenericKeySerDe) GenericRowSerDe(io.confluent.ksql.serde.GenericRowSerDe) Windowed(org.apache.kafka.streams.kstream.Windowed) GenericRow(io.confluent.ksql.GenericRow) PhysicalSchema(io.confluent.ksql.schema.ksql.PhysicalSchema) GenericKey(io.confluent.ksql.GenericKey) KeySerdeFactory(io.confluent.ksql.serde.KeySerdeFactory)

Example 2 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class InsertValuesExecutor method serializeKey.

private byte[] serializeKey(final GenericKey keyValue, final DataSource dataSource, final KsqlConfig config, final ServiceContext serviceContext) {
    final PhysicalSchema physicalSchema = PhysicalSchema.from(dataSource.getSchema(), dataSource.getKsqlTopic().getKeyFormat().getFeatures(), dataSource.getKsqlTopic().getValueFormat().getFeatures());
    ensureKeySchemasMatch(physicalSchema.keySchema(), dataSource, serviceContext);
    final Serde<GenericKey> keySerde = keySerdeFactory.create(dataSource.getKsqlTopic().getKeyFormat().getFormatInfo(), physicalSchema.keySchema(), config, serviceContext.getSchemaRegistryClientFactory(), "", NoopProcessingLogContext.INSTANCE, Optional.empty());
    final String topicName = dataSource.getKafkaTopicName();
    try {
        return keySerde.serializer().serialize(topicName, keyValue);
    } catch (final Exception e) {
        maybeThrowSchemaRegistryAuthError(FormatFactory.fromName(dataSource.getKsqlTopic().getKeyFormat().getFormat()), topicName, true, AclOperation.WRITE, e);
        LOG.error("Could not serialize key.", e);
        throw new KsqlException("Could not serialize key: " + keyValue, e);
    }
}
Also used : PhysicalSchema(io.confluent.ksql.schema.ksql.PhysicalSchema) GenericKey(io.confluent.ksql.GenericKey) KsqlException(io.confluent.ksql.util.KsqlException) KsqlTopicAuthorizationException(io.confluent.ksql.exception.KsqlTopicAuthorizationException) RestClientException(io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException) KsqlException(io.confluent.ksql.util.KsqlException) KsqlStatementException(io.confluent.ksql.util.KsqlStatementException) ExecutionException(java.util.concurrent.ExecutionException) KsqlSchemaAuthorizationException(io.confluent.ksql.exception.KsqlSchemaAuthorizationException) TopicAuthorizationException(org.apache.kafka.common.errors.TopicAuthorizationException)

Example 3 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class ForeignKeyTableTableJoinBuilderTest method shouldDoLeftJoinOnSubKey.

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void shouldDoLeftJoinOnSubKey() {
    // Given:
    givenLeftJoin(leftMultiKey, L_KEY_2);
    // When:
    final KTableHolder<Struct> result = join.build(planBuilder, planInfo);
    // Then:
    final ArgumentCaptor<KsqlKeyExtractor> ksqlKeyExtractor = ArgumentCaptor.forClass(KsqlKeyExtractor.class);
    verify(leftKTableMultiKey).leftJoin(same(rightKTable), ksqlKeyExtractor.capture(), eq(new KsqlValueJoiner(LEFT_SCHEMA_MULTI_KEY.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class));
    verifyNoMoreInteractions(leftKTable, rightKTable, resultKTable);
    final GenericKey extractedKey = GenericKey.genericKey(LEFT_KEY_2);
    assertThat(ksqlKeyExtractor.getValue().apply(LEFT_ROW_MULTI), is(extractedKey));
    assertThat(result.getTable(), is(resultKTable));
    assertThat(result.getExecutionKeyFactory(), is(executionKeyFactory));
}
Also used : GenericKey(io.confluent.ksql.GenericKey) Materialized(org.apache.kafka.streams.kstream.Materialized) Struct(org.apache.kafka.connect.data.Struct) Test(org.junit.Test)

Example 4 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class ForeignKeyTableTableJoinBuilderTest method shouldDoLeftJoinOnKey.

// this is actually a PK-PK join and the logical planner would not compile a FK-join plan
// for this case atm
// however, from a physical plan POV this should still work, so we would like to keep this test
// 
// it might be possible to actually change the logical planner to compile a PK-PK join as
// FK-join if input tables are not co-partitioned (instead of throwing an error an rejecting
// the query), ie, if key-format or partition-count do not match -- it's an open question
// if it would be a good idea to do this though
@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void shouldDoLeftJoinOnKey() {
    // Given:
    givenLeftJoin(left, L_KEY);
    // When:
    final KTableHolder<Struct> result = join.build(planBuilder, planInfo);
    // Then:
    final ArgumentCaptor<KsqlKeyExtractor> ksqlKeyExtractor = ArgumentCaptor.forClass(KsqlKeyExtractor.class);
    verify(leftKTable).leftJoin(same(rightKTable), ksqlKeyExtractor.capture(), eq(new KsqlValueJoiner(LEFT_SCHEMA.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class));
    verifyNoMoreInteractions(leftKTable, rightKTable, resultKTable);
    final GenericKey extractedKey = GenericKey.genericKey(LEFT_KEY);
    assertThat(ksqlKeyExtractor.getValue().apply(LEFT_ROW), is(extractedKey));
    assertThat(result.getTable(), is(resultKTable));
    assertThat(result.getExecutionKeyFactory(), is(executionKeyFactory));
}
Also used : GenericKey(io.confluent.ksql.GenericKey) Materialized(org.apache.kafka.streams.kstream.Materialized) Struct(org.apache.kafka.connect.data.Struct) Test(org.junit.Test)

Example 5 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class ForeignKeyTableTableJoinBuilderTest method shouldDoInnerJoinOnSubKey.

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void shouldDoInnerJoinOnSubKey() {
    // Given:
    givenInnerJoin(leftMultiKey, L_KEY_2);
    // When:
    final KTableHolder<Struct> result = join.build(planBuilder, planInfo);
    // Then:
    final ArgumentCaptor<KsqlKeyExtractor> ksqlKeyExtractor = ArgumentCaptor.forClass(KsqlKeyExtractor.class);
    verify(leftKTableMultiKey).join(same(rightKTable), ksqlKeyExtractor.capture(), eq(new KsqlValueJoiner(LEFT_SCHEMA_MULTI_KEY.value().size(), RIGHT_SCHEMA.value().size(), 0)), any(Materialized.class));
    verifyNoMoreInteractions(leftKTable, rightKTable, resultKTable);
    final GenericKey extractedKey = GenericKey.genericKey(LEFT_KEY_2);
    assertThat(ksqlKeyExtractor.getValue().apply(LEFT_ROW_MULTI), is(extractedKey));
    assertThat(result.getTable(), is(resultKTable));
    assertThat(result.getExecutionKeyFactory(), is(executionKeyFactory));
}
Also used : GenericKey(io.confluent.ksql.GenericKey) Materialized(org.apache.kafka.streams.kstream.Materialized) Struct(org.apache.kafka.connect.data.Struct) Test(org.junit.Test)

Aggregations

GenericKey (io.confluent.ksql.GenericKey)147 GenericRow (io.confluent.ksql.GenericRow)100 Test (org.junit.Test)93 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)24 Windowed (org.apache.kafka.streams.kstream.Windowed)20 WindowedRow (io.confluent.ksql.execution.streams.materialization.WindowedRow)14 PhysicalSchema (io.confluent.ksql.schema.ksql.PhysicalSchema)14 Materialized (org.apache.kafka.streams.kstream.Materialized)13 ValueAndTimestamp (org.apache.kafka.streams.state.ValueAndTimestamp)13 UnqualifiedColumnReferenceExp (io.confluent.ksql.execution.expression.tree.UnqualifiedColumnReferenceExp)12 MaterializationException (io.confluent.ksql.execution.streams.materialization.MaterializationException)9 IntegrationTest (io.confluent.common.utils.IntegrationTest)8 Materialization (io.confluent.ksql.execution.streams.materialization.Materialization)8 Row (io.confluent.ksql.execution.streams.materialization.Row)8 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)8 TimeWindow (org.apache.kafka.streams.kstream.internals.TimeWindow)8 IntegrationTest (org.apache.kafka.test.IntegrationTest)8 InOrder (org.mockito.InOrder)8 MaterializedTable (io.confluent.ksql.execution.streams.materialization.MaterializedTable)7 Objects (java.util.Objects)7