Examples with Materialization io.confluent.ksql.execution.streams.materialization.Materialization used on opensource projects

Example 1 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class QueryBuilderTest method shouldStartPersistentQueryWithCorrectMaterializationProvider.

@Test
public void shouldStartPersistentQueryWithCorrectMaterializationProvider() {
    // Given:
    final PersistentQueryMetadata queryMetadata = buildPersistentQuery(SOURCES, KsqlConstants.PersistentQueryType.CREATE_AS, QUERY_ID);
    queryMetadata.initialize();
    queryMetadata.start();
    // When:
    final Optional<Materialization> result = queryMetadata.getMaterialization(QUERY_ID, stacker);
    // Then:
    assertThat(result.get(), is(materialization));
}

Example 2 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class QueryBuilderTest method shouldStartCreateSourceQueryWithMaterializationProvider.

@Test
public void shouldStartCreateSourceQueryWithMaterializationProvider() {
    when(ksqlConfig.getBoolean(KsqlConfig.KSQL_SHARED_RUNTIME_ENABLED)).thenReturn(true);
    // Given:
    final DataSource source = givenSource("foo");
    when(source.getSchema()).thenReturn(SINK_SCHEMA);
    when(source.getKsqlTopic()).thenReturn(ksqlTopic);
    final PersistentQueryMetadata queryMetadata = buildPersistentQuery(ImmutableSet.of(source), KsqlConstants.PersistentQueryType.CREATE_SOURCE, QUERY_ID, Optional.empty());
    queryMetadata.initialize();
    queryMetadata.register();
    queryMetadata.start();
    // When:
    final Optional<Materialization> result = queryMetadata.getMaterialization(QUERY_ID, stacker);
    // Then:
    assertThat(result.get(), is(materialization));
}

Example 3 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableWithKeyFieldsInProjection.

@Test
public void shouldQueryMaterializedTableWithKeyFieldsInProjection() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*), AS_VALUE(USERID) AS USERID_2 FROM " + USER_TABLE + " GROUP BY USERID;");
    final LogicalSchema schema = schema("KSQL_COL_0", SqlTypes.BIGINT, "USERID_2", SqlTypes.STRING);
    final Map<String, GenericRow> rows = waitForUniqueUserRows(STRING_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.empty()));
    final MaterializedTable table = materialization.nonWindowed();
    rows.forEach((rowKey, value) -> {
        final GenericKey key = genericKey(rowKey);
        final List<Row> rowList = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION)));
        assertThat(rowList.size(), is(1));
        assertThat(rowList.get(0).schema(), is(schema));
        assertThat(rowList.get(0).key(), is(key));
        assertThat(rowList.get(0).value(), is(value));
    });
}

Example 4 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableForAggregatedTable.

@Test
public void shouldQueryMaterializedTableForAggregatedTable() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) FROM " + USER_TABLE + " GROUP BY USERID;");
    final LogicalSchema schema = schema("KSQL_COL_0", SqlTypes.BIGINT);
    final Map<String, GenericRow> rows = waitForUniqueUserRows(STRING_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.empty()));
    final MaterializedTable table = materialization.nonWindowed();
    rows.forEach((rowKey, value) -> {
        final GenericKey key = genericKey(rowKey);
        final Iterator<Row> rowIterator = withRetry(() -> table.get(key, PARTITION));
        assertThat(rowIterator.hasNext(), is(true));
        final Row row = rowIterator.next();
        assertThat(row.schema(), is(schema));
        assertThat(row.key(), is(key));
        assertThat(row.value(), is(value));
    });
    final GenericKey key = genericKey("Won't find me");
    assertThat("unknown key", withRetry(() -> table.get(key, PARTITION).hasNext()), is(false));
}

Example 5 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldHandleHavingClause.

@Test
public void shouldHandleHavingClause() {
    // Note: HAVING clause are handled centrally by KsqlMaterialization. This logic will have been
    // installed as part of building the below statement:
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_TABLE + " GROUP BY USERID" + " HAVING SUM(REGISTERTIME) > 2;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final int matches = (int) USER_DATA_PROVIDER.data().values().stream().filter(row -> ((Long) row.get(0)) > 2).count();
    final Map<String, GenericRow> rows = waitForUniqueUserRows(matches, STRING_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    final MaterializedTable table = materialization.nonWindowed();
    rows.forEach((rowKey, value) -> {
        // Rows passing the HAVING clause:
        final GenericKey key = genericKey(rowKey);
        final List<Row> rowList = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION)));
        assertThat(rowList.size(), is(1));
        assertThat(rowList.get(0).schema(), is(schema));
        assertThat(rowList.get(0).key(), is(key));
        assertThat(rowList.get(0).value(), is(value));
    });
    USER_DATA_PROVIDER.data().entries().stream().filter(e -> !rows.containsKey(e.getKey().get(0))).forEach(e -> {
        // Rows filtered by the HAVING clause:
        final List<Row> rowList = withRetry(() -> Lists.newArrayList(table.get(e.getKey(), PARTITION)));
        assertThat(rowList.isEmpty(), is(true));
    });
}