Search in sources :

Example 6 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class KsqlResource method describeTopic.

private TopicDescription describeTopic(String statementText, String name) throws KsqlException {
    KsqlTopic ksqlTopic = ksqlEngine.getMetaStore().getTopic(name);
    if (ksqlTopic == null) {
        throw new KsqlException(String.format("Could not find Topic '%s' in the Metastore", name));
    }
    String schemaString = null;
    TopicDescription topicDescription = new TopicDescription(statementText, name, ksqlTopic.getKafkaTopicName(), ksqlTopic.getKsqlTopicSerDe().getSerDe().toString(), schemaString);
    return topicDescription;
}
Also used : TopicDescription(io.confluent.ksql.rest.entity.TopicDescription) KsqlException(io.confluent.ksql.util.KsqlException) KsqlTopic(io.confluent.ksql.metastore.KsqlTopic)

Example 7 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class KsqlStructuredDataOutputNodeTest method getKsqlStructuredDataOutputNode.

private KsqlStructuredDataOutputNode getKsqlStructuredDataOutputNode(boolean isWindowed) {
    final Map<String, Object> props = new HashMap<>();
    props.put(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY, 4);
    props.put(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY, (short) 3);
    StructuredDataSourceNode tableSourceNode = new StructuredDataSourceNode(new PlanNodeId("0"), new KsqlTable("sqlExpression", "datasource", schema, schema.field("key"), schema.field("timestamp"), new KsqlTopic("input", "input", new KsqlJsonTopicSerDe()), "TableStateStore", isWindowed), schema);
    return new KsqlStructuredDataOutputNode(new PlanNodeId("0"), tableSourceNode, schema, schema.field("timestamp"), schema.field("key"), new KsqlTopic("output", "output", new KsqlJsonTopicSerDe()), "output", props, Optional.empty());
}
Also used : HashMap(java.util.HashMap) KsqlJsonTopicSerDe(io.confluent.ksql.serde.json.KsqlJsonTopicSerDe) KsqlTable(io.confluent.ksql.metastore.KsqlTable) KsqlTopic(io.confluent.ksql.metastore.KsqlTopic)

Example 8 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class StructuredDataSourceNodeTest method shouldBuildSchemaKTableWhenKTableSource.

@Test
public void shouldBuildSchemaKTableWhenKTableSource() {
    StructuredDataSourceNode node = new StructuredDataSourceNode(new PlanNodeId("0"), new KsqlTable("sqlExpression", "datasource", schema, schema.field("field"), schema.field("timestamp"), new KsqlTopic("topic2", "topic2", new KsqlJsonTopicSerDe()), "statestore", false), schema);
    final SchemaKStream result = build(node);
    assertThat(result.getClass(), equalTo(SchemaKTable.class));
}
Also used : SchemaKTable(io.confluent.ksql.structured.SchemaKTable) KsqlJsonTopicSerDe(io.confluent.ksql.serde.json.KsqlJsonTopicSerDe) KsqlTable(io.confluent.ksql.metastore.KsqlTable) SchemaKStream(io.confluent.ksql.structured.SchemaKStream) KsqlTopic(io.confluent.ksql.metastore.KsqlTopic) Test(org.junit.Test)

Example 9 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class MetaStoreFixture method getNewMetaStore.

public static MetaStore getNewMetaStore() {
    MetaStore metaStore = new MetaStoreImpl();
    SchemaBuilder schemaBuilder1 = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.array(SchemaBuilder.FLOAT64_SCHEMA)).field("COL5", SchemaBuilder.map(SchemaBuilder.STRING_SCHEMA, SchemaBuilder.FLOAT64_SCHEMA));
    KsqlTopic ksqlTopic1 = new KsqlTopic("TEST1", "test1", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStream = new KsqlStream("sqlexpression", "TEST1", schemaBuilder1, schemaBuilder1.field("COL0"), null, ksqlTopic1);
    metaStore.putTopic(ksqlTopic1);
    metaStore.putSource(ksqlStream);
    SchemaBuilder schemaBuilder2 = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.BOOLEAN_SCHEMA);
    KsqlTopic ksqlTopic2 = new KsqlTopic("TEST2", "test2", new KsqlJsonTopicSerDe());
    KsqlTable ksqlTable = new KsqlTable("sqlexpression", "TEST2", schemaBuilder2, schemaBuilder2.field("COL0"), null, ksqlTopic2, "TEST2", false);
    metaStore.putTopic(ksqlTopic2);
    metaStore.putSource(ksqlTable);
    SchemaBuilder schemaBuilderOrders = SchemaBuilder.struct().field("ORDERTIME", SchemaBuilder.INT64_SCHEMA).field("ORDERID", SchemaBuilder.STRING_SCHEMA).field("ITEMID", SchemaBuilder.STRING_SCHEMA).field("ORDERUNITS", SchemaBuilder.FLOAT64_SCHEMA);
    KsqlTopic ksqlTopicOrders = new KsqlTopic("ORDERS_TOPIC", "orders_topic", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStreamOrders = new KsqlStream("sqlexpression", "ORDERS", schemaBuilderOrders, schemaBuilderOrders.field("ORDERTIME"), null, ksqlTopicOrders);
    metaStore.putTopic(ksqlTopicOrders);
    metaStore.putSource(ksqlStreamOrders);
    return metaStore;
}
Also used : MetaStore(io.confluent.ksql.metastore.MetaStore) KsqlStream(io.confluent.ksql.metastore.KsqlStream) MetaStoreImpl(io.confluent.ksql.metastore.MetaStoreImpl) KsqlJsonTopicSerDe(io.confluent.ksql.serde.json.KsqlJsonTopicSerDe) KsqlTable(io.confluent.ksql.metastore.KsqlTable) SchemaBuilder(org.apache.kafka.connect.data.SchemaBuilder) KsqlTopic(io.confluent.ksql.metastore.KsqlTopic)

Example 10 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class AvroUtilTest method shouldValidatePersistentQueryResultCorrectly.

@Test
public void shouldValidatePersistentQueryResultCorrectly() throws IOException, RestClientException {
    SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
    KsqlTopic resultTopic = new KsqlTopic("testTopic", "testTopic", new KsqlAvroTopicSerDe());
    Schema resultSchema = SerDeUtil.getSchemaFromAvro(ordersAveroSchemaStr);
    PersistentQueryMetadata persistentQueryMetadata = new PersistentQueryMetadata("", null, null, "", null, DataSource.DataSourceType.KSTREAM, "", mock(KafkaTopicClient.class), resultSchema, resultTopic, null);
    org.apache.avro.Schema.Parser parser = new org.apache.avro.Schema.Parser();
    org.apache.avro.Schema avroSchema = parser.parse(ordersAveroSchemaStr);
    expect(schemaRegistryClient.testCompatibility(anyString(), EasyMock.isA(avroSchema.getClass()))).andReturn(true);
    replay(schemaRegistryClient);
    avroUtil.validatePersistentQueryResults(persistentQueryMetadata, schemaRegistryClient);
}
Also used : KsqlAvroTopicSerDe(io.confluent.ksql.serde.avro.KsqlAvroTopicSerDe) Schema(org.apache.kafka.connect.data.Schema) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) KsqlTopic(io.confluent.ksql.metastore.KsqlTopic) KsqlParser(io.confluent.ksql.parser.KsqlParser) Test(org.junit.Test)

Aggregations

KsqlTopic (io.confluent.ksql.metastore.KsqlTopic)15 KsqlStream (io.confluent.ksql.metastore.KsqlStream)5 KsqlJsonTopicSerDe (io.confluent.ksql.serde.json.KsqlJsonTopicSerDe)5 KsqlAvroTopicSerDe (io.confluent.ksql.serde.avro.KsqlAvroTopicSerDe)4 KsqlException (io.confluent.ksql.util.KsqlException)4 HashMap (java.util.HashMap)4 Test (org.junit.Test)4 KsqlTable (io.confluent.ksql.metastore.KsqlTable)3 Schema (org.apache.kafka.connect.data.Schema)3 SchemaBuilder (org.apache.kafka.connect.data.SchemaBuilder)3 SchemaRegistryClient (io.confluent.kafka.schemaregistry.client.SchemaRegistryClient)2 StructuredDataSource (io.confluent.ksql.metastore.StructuredDataSource)2 SelectItem (io.confluent.ksql.parser.tree.SelectItem)2 SingleColumn (io.confluent.ksql.parser.tree.SingleColumn)2 KafkaConsumerGroupClient (io.confluent.ksql.util.KafkaConsumerGroupClient)2 KafkaConsumerGroupClientImpl (io.confluent.ksql.util.KafkaConsumerGroupClientImpl)2 KsqlConfig (io.confluent.ksql.util.KsqlConfig)2 Pair (io.confluent.ksql.util.Pair)2 TopicDescription (org.apache.kafka.clients.admin.TopicDescription)2 TopicPartition (org.apache.kafka.common.TopicPartition)2