Examples with GenericKey io.confluent.ksql.GenericKey used on opensource projects

Example 86 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class RowGeneratorTest method shouldGenerateCorrectKey.

@Test
public void shouldGenerateCorrectKey() throws IOException {
    final Generator generator = new Generator(new File("./src/main/resources/pageviews_schema.avro"), new Random());
    final RowGenerator rowGenerator = new RowGenerator(generator, "viewtime", Optional.empty());
    final Pair<GenericKey, GenericRow> rowPair = rowGenerator.generateRow();
    final GenericKey key = rowPair.getLeft();
    final GenericRow value = rowPair.getRight();
    assertThat(key, is(notNullValue()));
    assertThat(key.get(0), is(instanceOf(Long.class)));
    assertThat("must match copy of key in value", key.get(0), is(value.get(0)));
}

Example 87 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class SchemaKSourceFactory method buildWindowedTable.

private static SchemaKTable<?> buildWindowedTable(final PlanBuildContext buildContext, final DataSource dataSource, final Stacker contextStacker) {
    final WindowInfo windowInfo = dataSource.getKsqlTopic().getKeyFormat().getWindowInfo().orElseThrow(IllegalArgumentException::new);
    final int pseudoColumnVersionToUse = determinePseudoColumnVersionToUse(buildContext);
    final SourceStep<KTableHolder<Windowed<GenericKey>>> step = ExecutionStepFactory.tableSourceWindowed(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), windowInfo, dataSource.getTimestampColumn(), pseudoColumnVersionToUse);
    return schemaKTable(buildContext, resolveSchema(buildContext, step, dataSource), dataSource.getKsqlTopic().getKeyFormat(), step);
}

Example 88 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class SchemaKSourceFactory method buildTable.

private static SchemaKTable<?> buildTable(final PlanBuildContext buildContext, final DataSource dataSource, final Stacker contextStacker) {
    final KeyFormat keyFormat = dataSource.getKsqlTopic().getKeyFormat();
    if (keyFormat.isWindowed()) {
        throw new IllegalArgumentException("windowed");
    }
    final SourceStep<KTableHolder<GenericKey>> step;
    final int pseudoColumnVersionToUse = determinePseudoColumnVersionToUse(buildContext);
    // If the old query has a v1 table step, continue to use it.
    // See https://github.com/confluentinc/ksql/pull/7990
    boolean useOldExecutionStepVersion = false;
    if (buildContext.getPlanInfo().isPresent()) {
        final Set<ExecutionStep<?>> sourceSteps = buildContext.getPlanInfo().get().getSources();
        useOldExecutionStepVersion = sourceSteps.stream().anyMatch(executionStep -> executionStep instanceof TableSourceV1);
    }
    if (useOldExecutionStepVersion && pseudoColumnVersionToUse != SystemColumns.LEGACY_PSEUDOCOLUMN_VERSION_NUMBER) {
        throw new IllegalStateException("TableSourceV2 was released in conjunction with pseudocolumn" + "version 1. Something has gone very wrong");
    }
    if (buildContext.getKsqlConfig().getBoolean(KsqlConfig.KSQL_ROWPARTITION_ROWOFFSET_ENABLED) && !useOldExecutionStepVersion) {
        step = ExecutionStepFactory.tableSource(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), dataSource.getTimestampColumn(), InternalFormats.of(keyFormat, Formats.from(dataSource.getKsqlTopic()).getValueFormat()), pseudoColumnVersionToUse);
    } else {
        step = ExecutionStepFactory.tableSourceV1(contextStacker, dataSource.getSchema(), dataSource.getKafkaTopicName(), Formats.from(dataSource.getKsqlTopic()), dataSource.getTimestampColumn(), pseudoColumnVersionToUse);
    }
    return schemaKTable(buildContext, resolveSchema(buildContext, step, dataSource), dataSource.getKsqlTopic().getKeyFormat(), step);
}

Example 89 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

public KsMaterializedQueryResult<WindowedRow> get(final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        final KeyValueIterator<Windowed<GenericKey>, ValueAndTimestamp<GenericRow>> iterator = cacheBypassFetcherAll.fetchAll(store, lower, upper);
        return KsMaterializedQueryResult.rowIterator(Streams.stream(IteratorUtil.onComplete(iterator, iterator::close)).map(next -> {
            final Instant windowStart = next.key.window().startTime();
            if (!windowStartBounds.contains(windowStart)) {
                return null;
            }
            final Instant windowEnd = next.key.window().endTime();
            if (!windowEndBounds.contains(windowEnd)) {
                return null;
            }
            final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
            final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(next.key.key(), window), next.value.value(), next.value.timestamp());
            return row;
        }).filter(Objects::nonNull).iterator());
    } catch (final Exception e) {
        throw new MaterializationException("Failed to scan materialized table", e);
    }
}

Example 90 with GenericKey

use of io.confluent.ksql.GenericKey in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

@Override
public KsMaterializedQueryResult<WindowedRow> get(final GenericKey key, final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        try (WindowStoreIterator<ValueAndTimestamp<GenericRow>> it = cacheBypassFetcher.fetch(store, key, lower, upper)) {
            final Builder<WindowedRow> builder = ImmutableList.builder();
            while (it.hasNext()) {
                final KeyValue<Long, ValueAndTimestamp<GenericRow>> next = it.next();
                final Instant windowStart = Instant.ofEpochMilli(next.key);
                if (!windowStartBounds.contains(windowStart)) {
                    continue;
                }
                final Instant windowEnd = windowStart.plus(windowSize);
                if (!windowEndBounds.contains(windowEnd)) {
                    continue;
                }
                final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
                final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(key, window), next.value.value(), next.value.timestamp());
                builder.add(row);
            }
            return KsMaterializedQueryResult.rowIterator(builder.build().iterator());
        }
    } catch (final Exception e) {
        throw new MaterializationException("Failed to get value from materialized table", e);
    }
}