Examples with KsqlTopic io.confluent.ksql.metastore.KsqlTopic used on opensource projects

Example 1 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class CodeGenRunnerTest method init.

@Before
public void init() {
    metaStore = MetaStoreFixture.getNewMetaStore();
    functionRegistry = new FunctionRegistry();
    schema = SchemaBuilder.struct().field("CODEGEN_TEST.COL0", SchemaBuilder.INT64_SCHEMA).field("CODEGEN_TEST.COL1", SchemaBuilder.STRING_SCHEMA).field("CODEGEN_TEST.COL2", SchemaBuilder.STRING_SCHEMA).field("CODEGEN_TEST.COL3", SchemaBuilder.FLOAT64_SCHEMA).field("CODEGEN_TEST.COL4", SchemaBuilder.FLOAT64_SCHEMA).field("CODEGEN_TEST.COL5", SchemaBuilder.INT32_SCHEMA).field("CODEGEN_TEST.COL6", SchemaBuilder.BOOLEAN_SCHEMA).field("CODEGEN_TEST.COL7", SchemaBuilder.BOOLEAN_SCHEMA).field("CODEGEN_TEST.COL8", SchemaBuilder.INT64_SCHEMA).field("CODEGEN_TEST.COL9", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL10", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL11", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA)).field("CODEGEN_TEST.COL12", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA));
    Schema metaStoreSchema = SchemaBuilder.struct().field("COL0", SchemaBuilder.INT64_SCHEMA).field("COL1", SchemaBuilder.STRING_SCHEMA).field("COL2", SchemaBuilder.STRING_SCHEMA).field("COL3", SchemaBuilder.FLOAT64_SCHEMA).field("COL4", SchemaBuilder.FLOAT64_SCHEMA).field("COL5", SchemaBuilder.INT32_SCHEMA).field("COL6", SchemaBuilder.BOOLEAN_SCHEMA).field("COL7", SchemaBuilder.BOOLEAN_SCHEMA).field("COL8", SchemaBuilder.INT64_SCHEMA).field("COL9", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("COL10", SchemaBuilder.array(SchemaBuilder.INT32_SCHEMA)).field("COL11", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA)).field("COL12", SchemaBuilder.map(SchemaBuilder.INT32_SCHEMA, SchemaBuilder.INT32_SCHEMA));
    KsqlTopic ksqlTopic = new KsqlTopic("CODEGEN_TEST", "codegen_test", new KsqlJsonTopicSerDe());
    KsqlStream ksqlStream = new KsqlStream("sqlexpression", "CODEGEN_TEST", metaStoreSchema, metaStoreSchema.field("COL0"), null, ksqlTopic);
    metaStore.putTopic(ksqlTopic);
    metaStore.putSource(ksqlStream);
    codeGenRunner = new CodeGenRunner(schema, functionRegistry);
}

Example 2 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class AvroUtilTest method shouldFailForInvalidResultAvroSchema.

@Test
public void shouldFailForInvalidResultAvroSchema() throws IOException, RestClientException {
    SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
    KsqlTopic resultTopic = new KsqlTopic("testTopic", "testTopic", new KsqlAvroTopicSerDe());
    Schema resultSchema = SerDeUtil.getSchemaFromAvro(ordersAveroSchemaStr);
    PersistentQueryMetadata persistentQueryMetadata = new PersistentQueryMetadata("", null, null, "", null, DataSource.DataSourceType.KSTREAM, "", mock(KafkaTopicClient.class), resultSchema, resultTopic, null);
    expect(schemaRegistryClient.testCompatibility(anyString(), anyObject())).andReturn(false);
    replay(schemaRegistryClient);
    try {
        avroUtil.validatePersistentQueryResults(persistentQueryMetadata, schemaRegistryClient);
        fail();
    } catch (Exception e) {
        assertThat("Incorrect exception message", "Cannot register avro schema for testTopic since " + "it is not valid for schema registry.", equalTo(e.getMessage()));
    }
}

Example 3 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class AstBuilder method getResultDatasource.

private StructuredDataSource getResultDatasource(Select select, Table into) {
    SchemaBuilder dataSource = SchemaBuilder.struct().name(into.toString());
    for (SelectItem selectItem : select.getSelectItems()) {
        if (selectItem instanceof SingleColumn) {
            SingleColumn singleColumn = (SingleColumn) selectItem;
            String fieldName = singleColumn.getAlias().get();
            dataSource = dataSource.field(fieldName, Schema.BOOLEAN_SCHEMA);
        }
    }
    KsqlTopic ksqlTopic = new KsqlTopic(into.getName().toString(), into.getName().toString(), null);
    StructuredDataSource resultStream = new KsqlStream("AstBuilder-Into", into.getName().toString(), dataSource.schema(), dataSource.fields().get(0), null, ksqlTopic);
    return resultStream;
}

Example 4 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class KafkaTopicsList method build.

public static KafkaTopicsList build(String statementText, Collection<KsqlTopic> ksqlTopics, Map<String, TopicDescription> kafkaTopicDescriptions, KsqlConfig ksqlConfig, KafkaConsumerGroupClient consumerGroupClient) {
    Set<String> registeredNames = getRegisteredKafkaTopicNames(ksqlTopics);
    List<KafkaTopicInfo> kafkaTopicInfoList = new ArrayList<>();
    kafkaTopicDescriptions = new TreeMap<>(filterKsqlInternalTopics(kafkaTopicDescriptions, ksqlConfig));
    Map<String, List<Integer>> topicConsumersAndGroupCount = getTopicConsumerAndGroupCounts(consumerGroupClient);
    for (TopicDescription desp : kafkaTopicDescriptions.values()) {
        kafkaTopicInfoList.add(new KafkaTopicInfo(desp.name(), registeredNames.contains(desp.name()), desp.partitions().stream().map(partition -> partition.replicas().size()).collect(Collectors.toList()), topicConsumersAndGroupCount.getOrDefault(desp.name(), Arrays.asList(0, 0)).get(0), topicConsumersAndGroupCount.getOrDefault(desp.name(), Arrays.asList(0, 0)).get(1)));
    }
    return new KafkaTopicsList(statementText, kafkaTopicInfoList);
}

Example 5 with KsqlTopic

use of io.confluent.ksql.metastore.KsqlTopic in project ksql by confluentinc.

the class RegisterTopicCommand method run.

@Override
public DdlCommandResult run(MetaStore metaStore) {
    if (metaStore.getTopic(topicName) != null) {
        // Check IF NOT EXIST is set, if set, do not create topic if one exists.
        if (notExists) {
            return new DdlCommandResult(true, "Topic is not registered because it already registered" + ".");
        } else {
            throw new KsqlException("Topic already registered.");
        }
    }
    KsqlTopic ksqlTopic = new KsqlTopic(topicName, kafkaTopicName, topicSerDe);
    // TODO: Need to check if the topic exists.
    // Add the topic to the metastore
    metaStore.putTopic(ksqlTopic);
    return new DdlCommandResult(true, "Topic registered");
}