Examples with WindowedRow io.confluent.ksql.execution.streams.materialization.WindowedRow used on opensource projects

Example 31 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableIQv2Test method shouldReturnValuesForOpenStartBounds.

@Test
public void shouldReturnValuesForOpenStartBounds() {
    // Given:
    final Range<Instant> start = Range.open(NOW, NOW.plusSeconds(10));
    final StateQueryResult partitionResult = new StateQueryResult();
    final QueryResult queryResult = QueryResult.forResult(fetchIterator);
    queryResult.setPosition(POSITION);
    partitionResult.addResult(PARTITION, queryResult);
    when(kafkaStreams.query(any())).thenReturn(partitionResult);
    when(fetchIterator.hasNext()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(fetchIterator.next()).thenReturn(new KeyValue<>(start.lowerEndpoint().toEpochMilli(), VALUE_1)).thenReturn(new KeyValue<>(start.lowerEndpoint().plusMillis(1).toEpochMilli(), VALUE_2)).thenReturn(new KeyValue<>(start.upperEndpoint().toEpochMilli(), VALUE_3)).thenThrow(new AssertionError());
    // When:
    final KsMaterializedQueryResult<WindowedRow> result = table.get(A_KEY, PARTITION, start, Range.all());
    // Then:
    final Iterator<WindowedRow> rowIterator = result.getRowIterator();
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(start.lowerEndpoint().plusMillis(1)), VALUE_2.value(), VALUE_2.timestamp())));
    assertThat(result.getPosition(), not(Optional.empty()));
    assertThat(result.getPosition().get(), is(POSITION));
}

Example 32 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableIQv2Test method shouldReturnValuesForClosedStartBounds_fetchAll.

@Test
public void shouldReturnValuesForClosedStartBounds_fetchAll() {
    // Given:
    final Range<Instant> start = Range.closed(Instant.ofEpochMilli(System.currentTimeMillis()), NOW.plusSeconds(10));
    final StateQueryResult partitionResult = new StateQueryResult();
    final QueryResult queryResult = QueryResult.forResult(keyValueIterator);
    queryResult.setPosition(POSITION);
    partitionResult.addResult(PARTITION, queryResult);
    when(kafkaStreams.query(any())).thenReturn(partitionResult);
    when(keyValueIterator.hasNext()).thenReturn(true, true, false);
    when(keyValueIterator.next()).thenReturn(new KeyValue<>(new Windowed<>(A_KEY, new TimeWindow(start.lowerEndpoint().toEpochMilli(), start.lowerEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_1)).thenReturn(new KeyValue<>(new Windowed<>(A_KEY2, new TimeWindow(start.upperEndpoint().toEpochMilli(), start.upperEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_2)).thenThrow(new AssertionError());
    // When:
    final KsMaterializedQueryResult<WindowedRow> result = table.get(PARTITION, start, Range.all());
    // Then:
    final Iterator<WindowedRow> rowIterator = result.getRowIterator();
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(A_KEY, start.lowerEndpoint()), VALUE_1.value(), VALUE_1.timestamp())));
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(A_KEY2, start.upperEndpoint()), VALUE_2.value(), VALUE_2.timestamp())));
    assertThat(rowIterator.hasNext(), is(false));
    assertThat(result.getPosition(), not(Optional.empty()));
    assertThat(result.getPosition().get(), is(POSITION));
}

Example 33 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTableTest method shouldReturnValuesForClosedStartBounds_fetchAll.

@Test
public void shouldReturnValuesForClosedStartBounds_fetchAll() {
    // Given:
    final Range<Instant> start = Range.closed(NOW, NOW.plusSeconds(10));
    when(keyValueIterator.hasNext()).thenReturn(true, true, false);
    when(keyValueIterator.next()).thenReturn(new KeyValue<>(new Windowed<>(A_KEY, new TimeWindow(start.lowerEndpoint().toEpochMilli(), start.lowerEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_1)).thenReturn(new KeyValue<>(new Windowed<>(A_KEY2, new TimeWindow(start.upperEndpoint().toEpochMilli(), start.upperEndpoint().toEpochMilli() + WINDOW_SIZE.toMillis())), VALUE_2)).thenThrow(new AssertionError());
    // When:
    final Iterator<WindowedRow> rowIterator = table.get(PARTITION, start, Range.all()).rowIterator;
    // Then:
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(start.lowerEndpoint()), VALUE_1.value(), VALUE_1.timestamp())));
    assertThat(rowIterator.hasNext(), is(true));
    assertThat(rowIterator.next(), is(WindowedRow.of(SCHEMA, windowedKey(A_KEY2, start.upperEndpoint()), VALUE_2.value(), VALUE_2.timestamp())));
    assertThat(rowIterator.hasNext(), is(false));
}

Example 34 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

public KsMaterializedQueryResult<WindowedRow> get(final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        final KeyValueIterator<Windowed<GenericKey>, ValueAndTimestamp<GenericRow>> iterator = cacheBypassFetcherAll.fetchAll(store, lower, upper);
        return KsMaterializedQueryResult.rowIterator(Streams.stream(IteratorUtil.onComplete(iterator, iterator::close)).map(next -> {
            final Instant windowStart = next.key.window().startTime();
            if (!windowStartBounds.contains(windowStart)) {
                return null;
            }
            final Instant windowEnd = next.key.window().endTime();
            if (!windowEndBounds.contains(windowEnd)) {
                return null;
            }
            final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
            final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(next.key.key(), window), next.value.value(), next.value.timestamp());
            return row;
        }).filter(Objects::nonNull).iterator());
    } catch (final Exception e) {
        throw new MaterializationException("Failed to scan materialized table", e);
    }
}

Example 35 with WindowedRow

use of io.confluent.ksql.execution.streams.materialization.WindowedRow in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

@Override
public KsMaterializedQueryResult<WindowedRow> get(final GenericKey key, final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        try (WindowStoreIterator<ValueAndTimestamp<GenericRow>> it = cacheBypassFetcher.fetch(store, key, lower, upper)) {
            final Builder<WindowedRow> builder = ImmutableList.builder();
            while (it.hasNext()) {
                final KeyValue<Long, ValueAndTimestamp<GenericRow>> next = it.next();
                final Instant windowStart = Instant.ofEpochMilli(next.key);
                if (!windowStartBounds.contains(windowStart)) {
                    continue;
                }
                final Instant windowEnd = windowStart.plus(windowSize);
                if (!windowEndBounds.contains(windowEnd)) {
                    continue;
                }
                final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
                final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(key, window), next.value.value(), next.value.timestamp());
                builder.add(row);
            }
            return KsMaterializedQueryResult.rowIterator(builder.build().iterator());
        }
    } catch (final Exception e) {
        throw new MaterializationException("Failed to get value from materialized table", e);
    }
}