Examples with KsqlTopic io.confluent.ksql.execution.ddl.commands.KsqlTopic used on opensource projects

Example 21 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class MetaStoreFixture method getNewMetaStore.

public static MutableMetaStore getNewMetaStore(final FunctionRegistry functionRegistry, final ValueFormat valueFormat) {
    final MutableMetaStore metaStore = new MetaStoreImpl(functionRegistry);
    final KeyFormat keyFormat = KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of());
    final LogicalSchema test1Schema = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.array(SqlTypes.DOUBLE)).valueColumn(ColumnName.of("COL5"), SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).headerColumn(ColumnName.of("HEAD"), Optional.empty()).build();
    final KsqlTopic ksqlTopic0 = new KsqlTopic("test0", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream0 = new KsqlStream<>("sqlexpression", SourceName.of("TEST0"), test1Schema, Optional.empty(), false, ksqlTopic0, false);
    metaStore.putSource(ksqlStream0, false);
    final KsqlTopic ksqlTopic1 = new KsqlTopic("test1", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream1 = new KsqlStream<>("sqlexpression", SourceName.of("TEST1"), test1Schema, Optional.empty(), false, ksqlTopic1, false);
    metaStore.putSource(ksqlStream1, false);
    final LogicalSchema test2Schema = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic2 = new KsqlTopic("test2", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable = new KsqlTable<>("sqlexpression", SourceName.of("TEST2"), test2Schema, Optional.empty(), false, ksqlTopic2, false);
    metaStore.putSource(ksqlTable, false);
    final SqlType addressSchema = SqlTypes.struct().field("NUMBER", SqlTypes.BIGINT).field("STREET", SqlTypes.STRING).field("CITY", SqlTypes.STRING).field("STATE", SqlTypes.STRING).field("ZIPCODE", SqlTypes.BIGINT).build();
    final SqlType categorySchema = SqlTypes.struct().field("ID", SqlTypes.BIGINT).field("NAME", SqlTypes.STRING).build();
    final SqlType itemInfoSchema = SqlTypes.struct().field("ITEMID", SqlTypes.BIGINT).field("NAME", SqlTypes.STRING).field("CATEGORY", categorySchema).build();
    final LogicalSchema ordersSchema = LogicalSchema.builder().keyColumn(ColumnName.of("ORDERTIME"), SqlTypes.BIGINT).valueColumn(ColumnName.of("ORDERID"), SqlTypes.BIGINT).valueColumn(ColumnName.of("ITEMID"), SqlTypes.STRING).valueColumn(ColumnName.of("ITEMINFO"), itemInfoSchema).valueColumn(ColumnName.of("ORDERUNITS"), SqlTypes.INTEGER).valueColumn(ColumnName.of("ARRAYCOL"), SqlTypes.array(SqlTypes.DOUBLE)).valueColumn(ColumnName.of("MAPCOL"), SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).valueColumn(ColumnName.of("ADDRESS"), addressSchema).valueColumn(ColumnName.of("TIMESTAMPCOL"), SqlTypes.TIMESTAMP).valueColumn(ColumnName.of("TIMECOL"), SqlTypes.TIME).valueColumn(ColumnName.of("DATECOL"), SqlTypes.DATE).valueColumn(ColumnName.of("BYTESCOL"), SqlTypes.BYTES).build();
    final KsqlTopic ksqlTopicOrders = new KsqlTopic("orders_topic", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStreamOrders = new KsqlStream<>("sqlexpression", SourceName.of("ORDERS"), ordersSchema, Optional.empty(), false, ksqlTopicOrders, false);
    metaStore.putSource(ksqlStreamOrders, false);
    final LogicalSchema testTable3 = LogicalSchema.builder().keyColumn(ColumnName.of("COL0"), SqlTypes.BIGINT).valueColumn(ColumnName.of("COL1"), SqlTypes.STRING).valueColumn(ColumnName.of("COL2"), SqlTypes.STRING).valueColumn(ColumnName.of("COL3"), SqlTypes.DOUBLE).valueColumn(ColumnName.of("COL4"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic3 = new KsqlTopic("test3", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable3 = new KsqlTable<>("sqlexpression", SourceName.of("TEST3"), testTable3, Optional.empty(), false, ksqlTopic3, false);
    metaStore.putSource(ksqlTable3, false);
    final SqlType nestedOrdersSchema = SqlTypes.struct().field("ORDERTIME", SqlTypes.BIGINT).field("ORDERID", SqlTypes.BIGINT).field("ITEMID", SqlTypes.STRING).field("ITEMINFO", itemInfoSchema).field("ORDERUNITS", SqlTypes.INTEGER).field("ARRAYCOL", SqlTypes.array(SqlTypes.DOUBLE)).field("MAPCOL", SqlTypes.map(SqlTypes.STRING, SqlTypes.DOUBLE)).field("ADDRESS", addressSchema).build();
    final LogicalSchema nestedArrayStructMapSchema = LogicalSchema.builder().keyColumn(ColumnName.of("K"), SqlTypes.STRING).valueColumn(ColumnName.of("ARRAYCOL"), SqlTypes.array(itemInfoSchema)).valueColumn(ColumnName.of("MAPCOL"), SqlTypes.map(SqlTypes.STRING, itemInfoSchema)).valueColumn(ColumnName.of("NESTED_ORDER_COL"), nestedOrdersSchema).valueColumn(ColumnName.of("ITEM"), itemInfoSchema).build();
    final KsqlTopic nestedArrayStructMapTopic = new KsqlTopic("NestedArrayStructMap_topic", keyFormat, valueFormat);
    final KsqlStream<?> nestedArrayStructMapOrders = new KsqlStream<>("sqlexpression", SourceName.of("NESTED_STREAM"), nestedArrayStructMapSchema, Optional.empty(), false, nestedArrayStructMapTopic, false);
    metaStore.putSource(nestedArrayStructMapOrders, false);
    final KsqlTopic ksqlTopic4 = new KsqlTopic("test4", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStream4 = new KsqlStream<>("sqlexpression4", SourceName.of("TEST4"), test1Schema, Optional.empty(), false, ksqlTopic4, false);
    metaStore.putSource(ksqlStream4, false);
    final LogicalSchema sensorReadingsSchema = LogicalSchema.builder().keyColumn(ColumnName.of("ID"), SqlTypes.BIGINT).valueColumn(ColumnName.of("SENSOR_NAME"), SqlTypes.STRING).valueColumn(ColumnName.of("ARR1"), SqlTypes.array(SqlTypes.BIGINT)).valueColumn(ColumnName.of("ARR2"), SqlTypes.array(SqlTypes.STRING)).build();
    final KsqlTopic ksqlTopicSensorReadings = new KsqlTopic("sensor_readings_topic", keyFormat, valueFormat);
    final KsqlStream<?> ksqlStreamSensorReadings = new KsqlStream<>("sqlexpression", SourceName.of("SENSOR_READINGS"), sensorReadingsSchema, Optional.empty(), false, ksqlTopicSensorReadings, false);
    metaStore.putSource(ksqlStreamSensorReadings, false);
    final LogicalSchema testTable5 = LogicalSchema.builder().keyColumn(ColumnName.of("A"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("B"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("C"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("D"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("E"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("F"), SqlTypes.BOOLEAN).valueColumn(ColumnName.of("G"), SqlTypes.BOOLEAN).build();
    final KsqlTopic ksqlTopic5 = new KsqlTopic("test5", keyFormat, valueFormat);
    final KsqlTable<String> ksqlTable5 = new KsqlTable<>("sqlexpression", SourceName.of("TEST5"), testTable5, Optional.empty(), false, ksqlTopic5, false);
    metaStore.putSource(ksqlTable5, false);
    return metaStore;
}

Example 22 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class TestExecutorTest method givenDataSourceTopic.

private void givenDataSourceTopic(final LogicalSchema schema) {
    final KsqlTopic topic = mock(KsqlTopic.class);
    when(topic.getKeyFormat()).thenReturn(KeyFormat.of(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of(), Optional.empty()));
    when(topic.getValueFormat()).thenReturn(ValueFormat.of(FormatInfo.of(FormatFactory.JSON.name()), SerdeFeatures.of()));
    final SourceName sourceName = SourceName.of(TestExecutorTest.SINK_TOPIC_NAME + "_source");
    final DataSource dataSource = mock(DataSource.class);
    when(dataSource.getKsqlTopic()).thenReturn(topic);
    when(dataSource.getSchema()).thenReturn(schema);
    when(dataSource.getKafkaTopicName()).thenReturn(TestExecutorTest.SINK_TOPIC_NAME);
    when(dataSource.getName()).thenReturn(sourceName);
    when(dataSource.getDataSourceType()).thenReturn(DataSourceType.KSTREAM);
    allSources.put(sourceName, dataSource);
}

Example 23 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class TemporaryEngine method givenSource.

@SuppressWarnings("unchecked")
public <T extends DataSource> T givenSource(final DataSourceType type, final String name, final Set<SourceName> sourceReferences) {
    givenKafkaTopic(name);
    final KsqlTopic topic = new KsqlTopic(name, KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of()), ValueFormat.of(FormatInfo.of(FormatFactory.JSON.name()), SerdeFeatures.of()));
    final DataSource source;
    switch(type) {
        case KSTREAM:
            source = new KsqlStream<>("statement", SourceName.of(name), SCHEMA, Optional.empty(), false, topic, false);
            break;
        case KTABLE:
            source = new KsqlTable<>("statement", SourceName.of(name), SCHEMA, Optional.empty(), false, topic, false);
            break;
        default:
            throw new IllegalArgumentException(type.toString());
    }
    metaStore.putSource(source, false);
    metaStore.addSourceReferences(source.getName(), sourceReferences);
    return (T) source;
}

Example 24 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class InsertValuesExecutorTest method givenDataSourceWithSchema.

private void givenDataSourceWithSchema(final String topicName, final LogicalSchema schema, final SerdeFeatures keyFeatures, final SerdeFeatures valFeatures, final FormatInfo keyFormat, final FormatInfo valueFormat, final boolean table, final boolean isSource) {
    final KsqlTopic topic = new KsqlTopic(topicName, KeyFormat.nonWindowed(keyFormat, keyFeatures), ValueFormat.of(valueFormat, valFeatures));
    final DataSource dataSource;
    if (table) {
        dataSource = new KsqlTable<>("", SourceName.of("TOPIC"), schema, Optional.empty(), false, topic, isSource);
    } else {
        dataSource = new KsqlStream<>("", SourceName.of("TOPIC"), schema, Optional.empty(), false, topic, isSource);
    }
    final MetaStoreImpl metaStore = new MetaStoreImpl(TestFunctionRegistry.INSTANCE.get());
    metaStore.putSource(dataSource, false);
    when(engine.getMetaStore()).thenReturn(metaStore);
}

Example 25 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class DescribeConnectorExecutorTest method setUp.

@Before
public void setUp() {
    final MetricCollectors metricCollectors = new MetricCollectors();
    when(engine.getMetaStore()).thenReturn(metaStore);
    when(engine.metricCollectors()).thenReturn(metricCollectors);
    when(serviceContext.getConnectClient()).thenReturn(connectClient);
    when(metaStore.getAllDataSources()).thenReturn(ImmutableMap.of(SourceName.of("source"), source));
    when(source.getKafkaTopicName()).thenReturn(TOPIC);
    when(source.getSqlExpression()).thenReturn(STATEMENT);
    when(source.getKsqlTopic()).thenReturn(new KsqlTopic(TOPIC, KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.AVRO.name()), SerdeFeatures.of()), ValueFormat.of(FormatInfo.of(FormatFactory.AVRO.name()), SerdeFeatures.of())));
    when(source.getSchema()).thenReturn(LogicalSchema.builder().keyColumn(SystemColumns.ROWKEY_NAME, SqlTypes.STRING).valueColumn(ColumnName.of("foo"), SqlPrimitiveType.of(SqlBaseType.STRING)).build());
    when(source.getDataSourceType()).thenReturn(DataSourceType.KTABLE);
    when(source.getName()).thenReturn(SourceName.of("source"));
    when(connectClient.status(CONNECTOR_NAME)).thenReturn(ConnectResponse.success(STATUS, HttpStatus.SC_OK));
    when(connectClient.describe("connector")).thenReturn(ConnectResponse.success(INFO, HttpStatus.SC_OK));
    connectorFactory = info -> Optional.of(connector);
    executor = new DescribeConnectorExecutor(connectorFactory, new DefaultConnectServerErrors());
    final DescribeConnector describeConnector = new DescribeConnector(Optional.empty(), "connector");
    final KsqlConfig ksqlConfig = new KsqlConfig(ImmutableMap.of());
    describeStatement = ConfiguredStatement.of(PreparedStatement.of("statementText", describeConnector), SessionConfig.of(ksqlConfig, ImmutableMap.of()));
}