Examples with Materialization io.confluent.ksql.execution.streams.materialization.Materialization used on opensource projects

Example 6 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableForTumblingWindowed.

@Test
public void shouldQueryMaterializedTableForTumblingWindowed() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_STREAM + " WINDOW TUMBLING (SIZE " + WINDOW_SIZE.getSeconds() + " SECONDS)" + " GROUP BY USERID;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final Map<Windowed<String>, GenericRow> rows = waitForUniqueUserRows(TIME_WINDOWED_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.of(WindowType.TUMBLING)));
    final MaterializedWindowedTable table = materialization.windowed();
    rows.forEach((k, v) -> {
        final Window w = Window.of(k.window().startTime(), k.window().endTime());
        final GenericKey key = genericKey(k.key());
        final List<WindowedRow> resultAtWindowStart = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.singleton(w.start()), Range.all())));
        assertThat("at exact window start", resultAtWindowStart, hasSize(1));
        assertThat(resultAtWindowStart.get(0).schema(), is(schema));
        assertThat(resultAtWindowStart.get(0).window(), is(Optional.of(w)));
        assertThat(resultAtWindowStart.get(0).key(), is(key));
        assertThat(resultAtWindowStart.get(0).value(), is(v));
        final List<WindowedRow> resultAtWindowEnd = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.all(), Range.singleton(w.end()))));
        assertThat("at exact window end", resultAtWindowEnd, hasSize(1));
        final List<WindowedRow> resultFromRange = withRetry(() -> withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().minusMillis(1), w.start().plusMillis(1)), Range.all()))));
        assertThat("range including window start", resultFromRange, is(resultAtWindowStart));
        final List<WindowedRow> resultPast = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().plusMillis(1), w.start().plusMillis(1)), Range.all())));
        assertThat("past start", resultPast, is(empty()));
    });
}

Example 7 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldReturnEmptyIfNotMaterializedStream.

@Test
public void shouldReturnEmptyIfNotMaterializedStream() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE STREAM " + output + " AS" + " SELECT * FROM " + USER_STREAM + ";");
    // When:
    final Optional<Materialization> result = query.getMaterialization(queryId, contextStacker);
    // Then:
    assertThat(result, is(Optional.empty()));
}

Example 8 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryTumblingWindowMaterializedTableWithRetention.

@Test
public void shouldQueryTumblingWindowMaterializedTableWithRetention() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT PAGEID, COUNT(*) AS COUNT FROM " + PAGE_VIEWS_STREAM + " WINDOW TUMBLING (SIZE " + WINDOW_SEGMENT_DURATION.getSeconds() + " SECONDS," + " RETENTION " + (WINDOW_SEGMENT_DURATION.getSeconds() * 2) + " SECONDS," + " GRACE PERIOD 0 SECONDS)" + " GROUP BY PAGEID;");
    final List<ConsumerRecord<Windowed<String>, GenericRow>> rows = waitForPageViewRows(TIME_WINDOWED_DESERIALIZER, query.getPhysicalSchema());
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.of(WindowType.TUMBLING)));
    final MaterializedWindowedTable table = materialization.windowed();
    final Set<Optional<Window>> expectedWindows = Stream.of(Window.of(WINDOW_START_INSTANTS.get(1), WINDOW_START_INSTANTS.get(1).plusSeconds(WINDOW_SEGMENT_DURATION.getSeconds())), Window.of(WINDOW_START_INSTANTS.get(2), WINDOW_START_INSTANTS.get(2).plusSeconds(WINDOW_SEGMENT_DURATION.getSeconds())), Window.of(WINDOW_START_INSTANTS.get(3), WINDOW_START_INSTANTS.get(3).plusSeconds(WINDOW_SEGMENT_DURATION.getSeconds()))).map(Optional::of).collect(Collectors.toSet());
    verifyRetainedWindows(rows, table, expectedWindows);
}

Example 9 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldReturnEmptyIfAppServerNotConfigured.

@Test
public void shouldReturnEmptyIfAppServerNotConfigured() {
    // Given:
    try (TestKsqlContext ksqlNoAppServer = TEST_HARNESS.ksqlContextBuilder().build()) {
        initializeKsql(ksqlNoAppServer);
        final PersistentQueryMetadata query = executeQuery(ksqlNoAppServer, "CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_TABLE + " GROUP BY USERID;");
        // When:
        final Optional<Materialization> result = query.getMaterialization(queryId, contextStacker);
        // Then:
        assertThat(result, is(Optional.empty()));
    }
}

Example 10 with Materialization

use of io.confluent.ksql.execution.streams.materialization.Materialization in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableForSessionWindowed.

@Test
public void shouldQueryMaterializedTableForSessionWindowed() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_STREAM + " WINDOW SESSION (" + WINDOW_SIZE.getSeconds() + " SECONDS)" + " GROUP BY USERID;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final Map<Windowed<String>, GenericRow> rows = waitForUniqueUserRows(SESSION_WINDOWED_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.of(WindowType.SESSION)));
    final MaterializedWindowedTable table = materialization.windowed();
    rows.forEach((k, v) -> {
        final Window w = Window.of(k.window().startTime(), k.window().endTime());
        final GenericKey key = genericKey(k.key());
        final List<WindowedRow> resultAtWindowStart = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.singleton(w.start()), Range.all())));
        assertThat("at exact window start", resultAtWindowStart, hasSize(1));
        assertThat(resultAtWindowStart.get(0).schema(), is(schema));
        assertThat(resultAtWindowStart.get(0).window(), is(Optional.of(w)));
        assertThat(resultAtWindowStart.get(0).key(), is(key));
        assertThat(resultAtWindowStart.get(0).value(), is(v));
        final List<WindowedRow> resultAtWindowEnd = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.all(), Range.singleton(w.end()))));
        assertThat("at exact window end", resultAtWindowEnd, hasSize(1));
        final List<WindowedRow> resultFromRange = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().minusMillis(1), w.start().plusMillis(1)), Range.all())));
        assertThat("range including window start", resultFromRange, is(resultAtWindowStart));
        final List<WindowedRow> resultPast = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().plusMillis(1), w.start().plusMillis(1)), Range.all())));
        assertThat("past start", resultPast, is(empty()));
    });
}