Examples with MaterializationException io.confluent.ksql.execution.streams.materialization.MaterializationException used on opensource projects

Example 6 with MaterializationException

use of io.confluent.ksql.execution.streams.materialization.MaterializationException in project ksql by confluentinc.

the class KsLocator method getMetadataForKeys.

/**
 * Gets the Metadata when looking up a list of keys.  This is used when the set of keys are known.
 * @param keys The non-empty set of keys to lookup metadata for
 * @param filterPartitions The partitions to limit lookups to, if non empty. Partitions which
 *                         exist by are not listed here are omitted. If empty, no filtering is
 *                         done.
 * @return The metadata associated with the keys
 */
private List<PartitionMetadata> getMetadataForKeys(final List<KsqlKey> keys, final Set<Integer> filterPartitions) {
    // Maintain request order for reproducibility by using a LinkedHashMap, even though it's
    // not a guarantee of the API.
    final Map<Integer, KeyQueryMetadata> metadataByPartition = new LinkedHashMap<>();
    final Map<Integer, Set<KsqlKey>> keysByPartition = new HashMap<>();
    for (KsqlKey key : keys) {
        final KeyQueryMetadata metadata = getKeyQueryMetadata(key);
        // Fail fast if Streams not ready. Let client handle it
        if (metadata.equals(KeyQueryMetadata.NOT_AVAILABLE)) {
            LOG.debug("KeyQueryMetadata not available for state store '{}' and key {}", storeName, key);
            throw new MaterializationException(String.format("Materialized data for key %s is not available yet. " + "Please try again later.", key));
        }
        LOG.debug("Handling pull query for key {} in partition {} of state store {}.", key, metadata.partition(), storeName);
        if (filterPartitions.size() > 0 && !filterPartitions.contains(metadata.partition())) {
            LOG.debug("Ignoring key {} in partition {} because parition is not included in lookup.", key, metadata.partition());
            continue;
        }
        keysByPartition.computeIfAbsent(metadata.partition(), k -> new LinkedHashSet<>());
        keysByPartition.get(metadata.partition()).add(key);
        metadataByPartition.putIfAbsent(metadata.partition(), metadata);
    }
    return metadataByPartition.values().stream().map(metadata -> {
        final HostInfo activeHost = metadata.activeHost();
        final Set<HostInfo> standByHosts = metadata.standbyHosts();
        return new PartitionMetadata(activeHost, standByHosts, metadata.partition(), Optional.of(keysByPartition.get(metadata.partition())));
    }).collect(Collectors.toList());
}

Example 7 with MaterializationException

use of io.confluent.ksql.execution.streams.materialization.MaterializationException in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

public KsMaterializedQueryResult<WindowedRow> get(final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        final KeyValueIterator<Windowed<GenericKey>, ValueAndTimestamp<GenericRow>> iterator = cacheBypassFetcherAll.fetchAll(store, lower, upper);
        return KsMaterializedQueryResult.rowIterator(Streams.stream(IteratorUtil.onComplete(iterator, iterator::close)).map(next -> {
            final Instant windowStart = next.key.window().startTime();
            if (!windowStartBounds.contains(windowStart)) {
                return null;
            }
            final Instant windowEnd = next.key.window().endTime();
            if (!windowEndBounds.contains(windowEnd)) {
                return null;
            }
            final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
            final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(next.key.key(), window), next.value.value(), next.value.timestamp());
            return row;
        }).filter(Objects::nonNull).iterator());
    } catch (final Exception e) {
        throw new MaterializationException("Failed to scan materialized table", e);
    }
}

Example 8 with MaterializationException

use of io.confluent.ksql.execution.streams.materialization.MaterializationException in project ksql by confluentinc.

the class KsMaterializedWindowTable method get.

@Override
public KsMaterializedQueryResult<WindowedRow> get(final GenericKey key, final int partition, final Range<Instant> windowStartBounds, final Range<Instant> windowEndBounds, final Optional<Position> position) {
    try {
        final ReadOnlyWindowStore<GenericKey, ValueAndTimestamp<GenericRow>> store = stateStore.store(QueryableStoreTypes.timestampedWindowStore(), partition);
        final Instant lower = calculateLowerBound(windowStartBounds, windowEndBounds);
        final Instant upper = calculateUpperBound(windowStartBounds, windowEndBounds);
        try (WindowStoreIterator<ValueAndTimestamp<GenericRow>> it = cacheBypassFetcher.fetch(store, key, lower, upper)) {
            final Builder<WindowedRow> builder = ImmutableList.builder();
            while (it.hasNext()) {
                final KeyValue<Long, ValueAndTimestamp<GenericRow>> next = it.next();
                final Instant windowStart = Instant.ofEpochMilli(next.key);
                if (!windowStartBounds.contains(windowStart)) {
                    continue;
                }
                final Instant windowEnd = windowStart.plus(windowSize);
                if (!windowEndBounds.contains(windowEnd)) {
                    continue;
                }
                final TimeWindow window = new TimeWindow(windowStart.toEpochMilli(), windowEnd.toEpochMilli());
                final WindowedRow row = WindowedRow.of(stateStore.schema(), new Windowed<>(key, window), next.value.value(), next.value.timestamp());
                builder.add(row);
            }
            return KsMaterializedQueryResult.rowIterator(builder.build().iterator());
        }
    } catch (final Exception e) {
        throw new MaterializationException("Failed to get value from materialized table", e);
    }
}

Example 9 with MaterializationException

use of io.confluent.ksql.execution.streams.materialization.MaterializationException in project ksql by confluentinc.

the class KsMaterializedTableIQv2 method get.

@Override
public KsMaterializedQueryResult<Row> get(final int partition, final Optional<Position> position) {
    try {
        final RangeQuery<GenericKey, ValueAndTimestamp<GenericRow>> query = RangeQuery.withNoBounds();
        StateQueryRequest<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> request = inStore(stateStore.getStateStoreName()).withQuery(query).withPartitions(ImmutableSet.of(partition));
        if (position.isPresent()) {
            request = request.withPositionBound(PositionBound.at(position.get()));
        }
        final StateQueryResult<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> result = stateStore.getKafkaStreams().query(request);
        final QueryResult<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> queryResult = result.getPartitionResults().get(partition);
        if (queryResult.isFailure()) {
            throw failedQueryException(queryResult);
        } else if (queryResult.getResult() == null) {
            return KsMaterializedQueryResult.rowIteratorWithPosition(Collections.emptyIterator(), queryResult.getPosition());
        } else {
            final KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>> iterator = queryResult.getResult();
            return KsMaterializedQueryResult.rowIteratorWithPosition(Streams.stream(IteratorUtil.onComplete(iterator, iterator::close)).map(keyValue -> Row.of(stateStore.schema(), keyValue.key, keyValue.value.value(), keyValue.value.timestamp())).iterator(), queryResult.getPosition());
        }
    } catch (final NotUpToBoundException | MaterializationException e) {
        throw e;
    } catch (final Exception e) {
        throw new MaterializationException("Failed to scan materialized table", e);
    }
}

Example 10 with MaterializationException

use of io.confluent.ksql.execution.streams.materialization.MaterializationException in project ksql by confluentinc.

the class KsMaterializedTableIQv2 method get.

// CHECKSTYLE_RULES.OFF: CyclomaticComplexity
@Override
public KsMaterializedQueryResult<Row> get(final int partition, final GenericKey from, final GenericKey to, final Optional<Position> position) {
    // CHECKSTYLE_RULES.ON: CyclomaticComplexity
    try {
        final RangeQuery<GenericKey, ValueAndTimestamp<GenericRow>> query;
        if (from != null && to != null) {
            query = RangeQuery.withRange(from, to);
        } else if (from == null && to != null) {
            query = RangeQuery.withUpperBound(to);
        } else if (from != null && to == null) {
            query = RangeQuery.withLowerBound(from);
        } else {
            query = RangeQuery.withNoBounds();
        }
        StateQueryRequest<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> request = inStore(stateStore.getStateStoreName()).withQuery(query).withPartitions(ImmutableSet.of(partition));
        if (position.isPresent()) {
            request = request.withPositionBound(PositionBound.at(position.get()));
        }
        final StateQueryResult<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> result = stateStore.getKafkaStreams().query(request);
        final QueryResult<KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>>> queryResult = result.getPartitionResults().get(partition);
        if (queryResult.isFailure()) {
            throw failedQueryException(queryResult);
        } else if (queryResult.getResult() == null) {
            return KsMaterializedQueryResult.rowIteratorWithPosition(Collections.emptyIterator(), queryResult.getPosition());
        } else {
            final KeyValueIterator<GenericKey, ValueAndTimestamp<GenericRow>> iterator = queryResult.getResult();
            return KsMaterializedQueryResult.rowIteratorWithPosition(Streams.stream(IteratorUtil.onComplete(iterator, iterator::close)).map(keyValue -> Row.of(stateStore.schema(), keyValue.key, keyValue.value.value(), keyValue.value.timestamp())).iterator(), queryResult.getPosition());
        }
    } catch (final NotUpToBoundException | MaterializationException e) {
        throw e;
    } catch (final Exception e) {
        throw new MaterializationException("Failed to range scan materialized table", e);
    }
}