Examples with KsqlTopic io.confluent.ksql.execution.ddl.commands.KsqlTopic used on opensource projects

Example 6 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class EngineExecutor method sourceTablePlan.

@SuppressFBWarnings(value = "NP_NULL_PARAM_DEREF_NONVIRTUAL")
private KsqlPlan sourceTablePlan(final ConfiguredStatement<?> statement) {
    final CreateTable createTable = (CreateTable) statement.getStatement();
    final CreateTableCommand ddlCommand = (CreateTableCommand) engineContext.createDdlCommand(statement.getStatementText(), (ExecutableDdlStatement) statement.getStatement(), config);
    final Relation from = new AliasedRelation(new Table(createTable.getName()), createTable.getName());
    // Only VALUE or HEADER columns must be selected from the source table. When running a
    // pull query, the keys are added if selecting all columns.
    final Select select = new Select(createTable.getElements().stream().filter(column -> !column.getConstraints().isKey() && !column.getConstraints().isPrimaryKey()).map(column -> new SingleColumn(new UnqualifiedColumnReferenceExp(column.getName()), Optional.of(column.getName()))).collect(Collectors.toList()));
    // Source table need to keep emitting changes so every new record is materialized for
    // pull query availability.
    final RefinementInfo refinementInfo = RefinementInfo.of(OutputRefinement.CHANGES);
    // This is a plan for a `select * from <source-table> emit changes` statement,
    // without a sink topic to write the results. The query is just made to materialize the
    // source table.
    final Query query = new Query(Optional.empty(), select, from, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(refinementInfo), false, OptionalInt.empty());
    // The source table does not exist in the current metastore, so a temporary metastore that
    // contains only the source table is created here. This metastore is used later to create
    // ExecutorsPlan.
    final MutableMetaStore tempMetastore = new MetaStoreImpl(new InternalFunctionRegistry());
    final Formats formats = ddlCommand.getFormats();
    tempMetastore.putSource(new KsqlTable<>(statement.getStatementText(), createTable.getName(), ddlCommand.getSchema(), Optional.empty(), false, new KsqlTopic(ddlCommand.getTopicName(), KeyFormat.of(formats.getKeyFormat(), formats.getKeyFeatures(), Optional.empty()), ValueFormat.of(formats.getValueFormat(), formats.getValueFeatures())), true), false);
    final ExecutorPlans plans = planQuery(statement, query, Optional.empty(), Optional.empty(), tempMetastore);
    final KsqlBareOutputNode outputNode = (KsqlBareOutputNode) plans.logicalPlan.getNode().get();
    final QueryPlan queryPlan = new QueryPlan(getSourceNames(outputNode), Optional.empty(), plans.physicalPlan.getPhysicalPlan(), plans.physicalPlan.getQueryId(), getApplicationId(plans.physicalPlan.getQueryId(), getSourceNames(outputNode)));
    engineContext.createQueryValidator().validateQuery(config, plans.physicalPlan, engineContext.getQueryRegistry().getAllLiveQueries());
    return KsqlPlan.queryPlanCurrent(statement.getStatementText(), Optional.of(ddlCommand), queryPlan);
}

Example 7 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class CreateSourceFactoryTest method shouldCreateTableCommandFromNodeOutput.

@Test
public void shouldCreateTableCommandFromNodeOutput() {
    // Given:
    final KsqlTopic ksqlTopic = mock(KsqlTopic.class);
    when(ksqlTopic.getKafkaTopicName()).thenReturn(TOPIC_NAME);
    when(ksqlTopic.getKeyFormat()).thenReturn(SOME_KEY_FORMAT);
    when(ksqlTopic.getValueFormat()).thenReturn(SOME_VALUE_FORMAT);
    final KsqlStructuredDataOutputNode outputNode = mock(KsqlStructuredDataOutputNode.class);
    when(outputNode.getSinkName()).thenReturn(Optional.of(SOME_NAME));
    when(outputNode.getSchema()).thenReturn(EXPECTED_SCHEMA);
    when(outputNode.getTimestampColumn()).thenReturn(Optional.of(TIMESTAMP_COLUMN));
    when(outputNode.getKsqlTopic()).thenReturn(ksqlTopic);
    // When:
    final CreateTableCommand result = createSourceFactory.createTableCommand(outputNode);
    // Then:
    assertThat(result.getSourceName(), is(SOME_NAME));
    assertThat(result.getSchema(), is(EXPECTED_SCHEMA));
    assertThat(result.getTimestampColumn(), is(Optional.of(TIMESTAMP_COLUMN)));
    assertThat(result.getTopicName(), is(TOPIC_NAME));
    assertThat(result.getFormats(), is(Formats.from(ksqlTopic)));
    assertThat(result.getWindowInfo(), is(Optional.empty()));
    assertThat(result.isOrReplace(), is(false));
}

Example 8 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class SchemaRegisterInjectorTest method setUp.

@Before
public void setUp() throws IOException, RestClientException {
    metaStore = new MetaStoreImpl(new InternalFunctionRegistry());
    config = new KsqlConfig(ImmutableMap.of(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY, "foo:8081"));
    injector = new SchemaRegisterInjector(executionContext, serviceContext);
    when(serviceContext.getSchemaRegistryClient()).thenReturn(schemaRegistryClient);
    when(serviceContext.getTopicClient()).thenReturn(topicClient);
    when(serviceContext.getConsumerGroupClient()).thenReturn(consumerGroupClient);
    when(executionContext.createSandbox(any())).thenReturn(executionSandbox);
    when(keyFeatures.enabled(SerdeFeature.UNWRAP_SINGLES)).thenReturn(true);
    when(ddlCommand.getSchema()).thenReturn(SCHEMA);
    when(ddlCommand.getTopicName()).thenReturn("SINK");
    when(ddlCommand.getFormats()).thenReturn(formats);
    when(formats.getKeyFormat()).thenReturn(FormatInfo.of(FormatFactory.AVRO.name()));
    when(formats.getKeyFeatures()).thenReturn(keyFeatures);
    when(formats.getValueFormat()).thenReturn(FormatInfo.of(FormatFactory.AVRO.name()));
    when(formats.getValueFeatures()).thenReturn(valFeatures);
    when(schemaRegistryClient.getLatestSchemaMetadata(any())).thenThrow(new RestClientException("foo", 404, SchemaRegistryUtil.SUBJECT_NOT_FOUND_ERROR_CODE));
    final KsqlTopic sourceTopic = new KsqlTopic("source", KeyFormat.nonWindowed(FormatInfo.of(FormatFactory.KAFKA.name()), SerdeFeatures.of()), ValueFormat.of(FormatInfo.of(FormatFactory.JSON.name()), valFeatures));
    final KsqlStream<?> source = new KsqlStream<>("", SourceName.of("SOURCE"), SCHEMA, Optional.empty(), false, sourceTopic, false);
    metaStore.putSource(source, false);
}

Example 9 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class TestExecutorUtil method buildSinkTopic.

private static Topic buildSinkTopic(final DataSource sinkDataSource, final StubKafkaService stubKafkaService, final SchemaRegistryClient schemaRegistryClient) {
    final String kafkaTopicName = sinkDataSource.getKafkaTopicName();
    final KsqlTopic ksqlTopic = sinkDataSource.getKsqlTopic();
    final Optional<ParsedSchema> keySchema = getSchema(ksqlTopic.getKeyFormat().getFormat(), getSRSubject(ksqlTopic.getKafkaTopicName(), true), schemaRegistryClient);
    final Optional<ParsedSchema> valueSchema = getSchema(ksqlTopic.getValueFormat().getFormat(), getSRSubject(ksqlTopic.getKafkaTopicName(), false), schemaRegistryClient);
    final Topic sinkTopic = new Topic(kafkaTopicName, keySchema, valueSchema);
    stubKafkaService.ensureTopic(sinkTopic);
    return sinkTopic;
}

Example 10 with KsqlTopic

use of io.confluent.ksql.execution.ddl.commands.KsqlTopic in project ksql by confluentinc.

the class AnalysisTest method givenWindowedSource.

private static void givenWindowedSource(final KsqlStream<?> dataSource) {
    final KsqlTopic topic = mock(KsqlTopic.class);
    when(topic.getKeyFormat()).thenReturn(KeyFormat.windowed(A_FORMAT, SerdeFeatures.of(), A_WINDOW));
    when(dataSource.getKsqlTopic()).thenReturn(topic);
}