Examples with DataSource io.confluent.ksql.metastore.model.DataSource used on opensource projects

Example 41 with DataSource

use of io.confluent.ksql.metastore.model.DataSource in project ksql by confluentinc.

the class SourceDescriptionFactoryTest method shouldReturnEmptyTimestampColumn.

@Test
public void shouldReturnEmptyTimestampColumn() {
    // Given:
    final String kafkaTopicName = "kafka";
    final DataSource dataSource = buildDataSource(kafkaTopicName, Optional.empty());
    // When
    final SourceDescription sourceDescription = SourceDescriptionFactory.create(dataSource, true, Collections.emptyList(), Collections.emptyList(), Optional.empty(), Collections.emptyList(), Collections.emptyList(), new MetricCollectors());
    // Then:
    assertThat(sourceDescription.getTimestamp(), is(""));
}

Example 42 with DataSource

use of io.confluent.ksql.metastore.model.DataSource in project ksql by confluentinc.

the class SourceDescriptionFactoryTest method shouldReturnSourceConstraints.

@Test
public void shouldReturnSourceConstraints() {
    // Given:
    final String kafkaTopicName = "kafka";
    final DataSource dataSource = buildDataSource(kafkaTopicName, Optional.empty());
    // When
    final SourceDescription sourceDescription = SourceDescriptionFactory.create(dataSource, true, Collections.emptyList(), Collections.emptyList(), Optional.empty(), Collections.emptyList(), ImmutableList.of("s1", "s2"), new MetricCollectors());
    // Then:
    assertThat(sourceDescription.getSourceConstraints(), hasItems("s1", "s2"));
}

Example 43 with DataSource

use of io.confluent.ksql.metastore.model.DataSource in project ksql by confluentinc.

the class KsqlTesterTest method pipeInput.

private void pipeInput(final ConfiguredStatement<InsertValues> statement) {
    final InsertValues insertValues = statement.getStatement();
    final DataSource dataSource = engine.getMetaStore().getSource(insertValues.getTarget());
    if (dataSource == null) {
        throw new KsqlException("Unknown data source " + insertValues.getTarget());
    }
    final KsqlGenericRecord record = new GenericRecordFactory(config, engine.getMetaStore(), System::currentTimeMillis).build(insertValues.getColumns(), insertValues.getValues(), dataSource.getSchema(), dataSource.getDataSourceType());
    driverPipeline.pipeInput(dataSource.getKafkaTopicName(), record.key, record.value, record.ts);
}

Example 44 with DataSource

use of io.confluent.ksql.metastore.model.DataSource in project ksql by confluentinc.

the class KsqlTesterTest method execute.

@SuppressWarnings("unchecked")
private void execute(final ParsedStatement parsedStatement) {
    final PreparedStatement<?> engineStatement = engine.prepare(parsedStatement);
    final ConfiguredStatement<?> configured = ConfiguredStatement.of(engineStatement, SessionConfig.of(config, overrides));
    createTopics(engineStatement);
    if (engineStatement.getStatement() instanceof InsertValues) {
        pipeInput((ConfiguredStatement<InsertValues>) configured);
        return;
    } else if (engineStatement.getStatement() instanceof SetProperty) {
        PropertyOverrider.set((ConfiguredStatement<SetProperty>) configured, overrides);
        return;
    } else if (engineStatement.getStatement() instanceof UnsetProperty) {
        PropertyOverrider.unset((ConfiguredStatement<UnsetProperty>) configured, overrides);
        return;
    }
    final ConfiguredStatement<?> injected = formatInjector.inject(configured);
    final ExecuteResult result = engine.execute(serviceContext, injected);
    // is DDL statement
    if (!result.getQuery().isPresent()) {
        return;
    }
    final PersistentQueryMetadata query = (PersistentQueryMetadata) result.getQuery().get();
    final Topology topology = query.getTopology();
    final Properties properties = new Properties();
    properties.putAll(query.getStreamsProperties());
    properties.put(StreamsConfig.STATE_DIR_CONFIG, tmpFolder.getRoot().getAbsolutePath());
    final TopologyTestDriver driver = new TopologyTestDriver(topology, properties);
    final List<TopicInfo> inputTopics = query.getSourceNames().stream().map(sn -> engine.getMetaStore().getSource(sn)).map(ds -> new TopicInfo(ds.getKafkaTopicName(), keySerde(ds), valueSerde(ds))).collect(Collectors.toList());
    // Sink may be Optional for source tables. Once source table query execution is supported, then
    // we would need have a condition to not create an output topic info
    final DataSource output = engine.getMetaStore().getSource(query.getSinkName().get());
    final TopicInfo outputInfo = new TopicInfo(output.getKafkaTopicName(), keySerde(output), valueSerde(output));
    driverPipeline.addDriver(driver, inputTopics, outputInfo);
    drivers.put(query.getQueryId(), new DriverAndProperties(driver, properties));
}

Example 45 with DataSource

use of io.confluent.ksql.metastore.model.DataSource in project ksql by confluentinc.

the class PullQueryExecutionUtil method findMaterializingQuery.

static PersistentQueryMetadata findMaterializingQuery(final EngineContext engineContext, final ImmutableAnalysis analysis) {
    final DataSource source = analysis.getFrom().getDataSource();
    final SourceName sourceName = source.getName();
    final Set<QueryId> queries = engineContext.getQueryRegistry().getQueriesWithSink(sourceName);
    if (source.getDataSourceType() != DataSourceType.KTABLE) {
        throw new KsqlException("Unexpected data source type for table pull query: " + source.getDataSourceType() + " " + PullQueryValidator.PULL_QUERY_SYNTAX_HELP);
    }
    if (queries.isEmpty()) {
        throw notMaterializedException(sourceName);
    }
    if (queries.size() > 1) {
        throw new IllegalStateException("Tables do not support multiple queries writing into them, yet somehow this happened. " + "Source Name: " + sourceName + " Queries: " + queries + ". Please submit " + "a GitHub issue with the queries that were run.");
    }
    final QueryId queryId = Iterables.getOnlyElement(queries);
    return engineContext.getQueryRegistry().getPersistentQuery(queryId).orElseThrow(() -> new KsqlException("Materializing query has been stopped"));
}