Examples with WindowedRow io.confluent.ksql.execution.streams.materialization.WindowedRow used on opensource projects

Example 21 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableIQv2Test method shouldReturnValuesForClosedEndBounds_fetchAll.

@Test
public void shouldReturnValuesForClosedEndBounds_fetchAll() {
    // Given:
    final Range<Instant> end = Range.closed(NOW, NOW.plusSeconds(10));
    final Range<Instant> startEqiv = Range.closed(end.lowerEndpoint().minus(WINDOW_SIZE), end.lowerEndpoint().minus(WINDOW_SIZE));
    final StateQueryResult partitionResult = new StateQueryResult();
    final QueryResult queryResult = QueryResult.forResult(keyValueIterator);
    queryResult.setPosition(POSITION);
    partitionResult.addResult(PARTITION, queryResult);
    when(kafkaStreams.query(any())).thenReturn(partitionResult);
    when(keyValueIterator.hasNext()).thenReturn(true, true, false);
    when(keyValueIterator.next()).thenReturn(new KeyValue<>(new Windowed<>(A_KEY, new TimeWindow(startEqiv.lowerEndpoint().toEpochMilli(), startEqiv.lowerEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_1)).thenReturn(new KeyValue<>(new Windowed<>(A_KEY2, new TimeWindow(startEqiv.upperEndpoint().toEpochMilli(), startEqiv.upperEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_2)).thenThrow(new AssertionError());
    // When:
    final KsMaterializedQueryResult<WindowedRow> result = table.get(PARTITION, Range.all(), end);
    // Then:
    final Iterator<WindowedRow> rowIterator = result.getRowIterator();
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(startEqiv.lowerEndpoint()), VALUE_1.value(), VALUE_1.timestamp())));
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(A_KEY2, startEqiv.upperEndpoint()), VALUE_2.value(), VALUE_2.timestamp())));
    assertThat(rowIterator.hasNext(), is(false));
    assertThat(result.getPosition(), not(Optional.empty()));
    assertThat(result.getPosition().get(), is(POSITION));
}

Example 22 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method verifyRetainedWindows.

private static void verifyRetainedWindows(final List<ConsumerRecord<Windowed<String>, GenericRow>> rows, final MaterializedWindowedTable table, final Set<Optional<Window>> expectedWindows) {
    rows.forEach(record -> {
        final GenericKey key = genericKey(record.key().key());
        final List<WindowedRow> resultAtWindowStart = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.all(), Range.all())));
        assertThat("Should have fewer windows retained", resultAtWindowStart, hasSize(expectedWindows.size()));
        final Set<Optional<Window>> actualWindows = resultAtWindowStart.stream().map(WindowedRow::window).collect(Collectors.toSet());
        assertThat("Should retain the latest windows", actualWindows, equalTo(expectedWindows));
    });
}

Example 23 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializationFunctionalTest method shouldQueryMaterializedTableForTumblingWindowed.

@Test
public void shouldQueryMaterializedTableForTumblingWindowed() {
    // Given:
    final PersistentQueryMetadata query = executeQuery("CREATE TABLE " + output + " AS" + " SELECT USERID, COUNT(*) AS COUNT FROM " + USER_STREAM + " WINDOW TUMBLING (SIZE " + WINDOW_SIZE.getSeconds() + " SECONDS)" + " GROUP BY USERID;");
    final LogicalSchema schema = schema("COUNT", SqlTypes.BIGINT);
    final Map<Windowed<String>, GenericRow> rows = waitForUniqueUserRows(TIME_WINDOWED_DESERIALIZER, schema);
    // When:
    final Materialization materialization = query.getMaterialization(queryId, contextStacker).get();
    // Then:
    assertThat(materialization.windowType(), is(Optional.of(WindowType.TUMBLING)));
    final MaterializedWindowedTable table = materialization.windowed();
    rows.forEach((k, v) -> {
        final Window w = Window.of(k.window().startTime(), k.window().endTime());
        final GenericKey key = genericKey(k.key());
        final List<WindowedRow> resultAtWindowStart = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.singleton(w.start()), Range.all())));
        assertThat("at exact window start", resultAtWindowStart, hasSize(1));
        assertThat(resultAtWindowStart.get(0).schema(), is(schema));
        assertThat(resultAtWindowStart.get(0).window(), is(Optional.of(w)));
        assertThat(resultAtWindowStart.get(0).key(), is(key));
        assertThat(resultAtWindowStart.get(0).value(), is(v));
        final List<WindowedRow> resultAtWindowEnd = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.all(), Range.singleton(w.end()))));
        assertThat("at exact window end", resultAtWindowEnd, hasSize(1));
        final List<WindowedRow> resultFromRange = withRetry(() -> withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().minusMillis(1), w.start().plusMillis(1)), Range.all()))));
        assertThat("range including window start", resultFromRange, is(resultAtWindowStart));
        final List<WindowedRow> resultPast = withRetry(() -> Lists.newArrayList(table.get(key, PARTITION, Range.closed(w.start().plusMillis(1), w.start().plusMillis(1)), Range.all())));
        assertThat("past start", resultPast, is(empty()));
    });
}

Example 24 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableIQv2Test method shouldMaintainResultOrder.

@Test
public void shouldMaintainResultOrder() {
    // Given:
    when(fetchIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    final Instant start = WINDOW_START_BOUNDS.lowerEndpoint();
    final StateQueryResult partitionResult = new StateQueryResult();
    final QueryResult result = QueryResult.forResult(fetchIterator);
    result.setPosition(POSITION);
    partitionResult.addResult(PARTITION, result);
    when(kafkaStreams.query(any())).thenReturn(partitionResult);
    when(fetchIterator.next()).thenReturn(new KeyValue<>(start.toEpochMilli(), VALUE_1)).thenReturn(new KeyValue<>(start.plusMillis(1).toEpochMilli(), VALUE_2)).thenReturn(new KeyValue<>(start.plusMillis(2).toEpochMilli(), VALUE_3)).thenThrow(new AssertionError());
    // When:
    final Iterator<WindowedRow> rowIterator = table.get(A_KEY, PARTITION, Range.all(), Range.all()).rowIterator;
    // Then:
    assertThat(rowIterator.hasNext(), is(true));
    final List<WindowedRow> resultList = Lists.newArrayList(rowIterator);
    assertThat(resultList, contains(WindowedRow.of(SCHEMA, windowedKey(start), VALUE_1.value(), VALUE_1.timestamp()), WindowedRow.of(SCHEMA, windowedKey(start.plusMillis(1)), VALUE_2.value(), VALUE_2.timestamp()), WindowedRow.of(SCHEMA, windowedKey(start.plusMillis(2)), VALUE_3.value(), VALUE_3.timestamp())));
}

Example 25 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableIQv2Test method shouldReturnValuesForOpenEndBounds.

@Test
public void shouldReturnValuesForOpenEndBounds() {
    // Given:
    final Range<Instant> end = Range.open(NOW, NOW.plusSeconds(10));
    final Range<Instant> startEquiv = Range.open(end.lowerEndpoint().minus(WINDOW_SIZE), end.upperEndpoint().minus(WINDOW_SIZE));
    final StateQueryResult partitionResult = new StateQueryResult();
    final QueryResult queryResult = QueryResult.forResult(fetchIterator);
    queryResult.setPosition(POSITION);
    partitionResult.addResult(PARTITION, queryResult);
    when(kafkaStreams.query(any())).thenReturn(partitionResult);
    when(fetchIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fetchIterator.next()).thenReturn(new KeyValue<>(startEquiv.lowerEndpoint().toEpochMilli(), VALUE_1)).thenReturn(new KeyValue<>(startEquiv.lowerEndpoint().plusMillis(1).toEpochMilli(), VALUE_2)).thenReturn(new KeyValue<>(startEquiv.upperEndpoint().toEpochMilli(), VALUE_3)).thenThrow(new AssertionError());
    // When:
    final KsMaterializedQueryResult<WindowedRow> result = table.get(A_KEY, PARTITION, Range.all(), end);
    // Then:
    final Iterator<WindowedRow> rowIterator = result.getRowIterator();
    assertThat(rowIterator.hasNext(), is(true));
    final List<WindowedRow> resultList = Lists.newArrayList(rowIterator);
    assertThat(resultList, contains(WindowedRow.of(SCHEMA, windowedKey(startEquiv.lowerEndpoint().plusMillis(1)), VALUE_2.value(), VALUE_2.timestamp())));
    assertThat(result.getPosition(), not(Optional.empty()));
    assertThat(result.getPosition().get(), is(POSITION));
}