Examples with ConsistencyOffsetVector io.confluent.ksql.util.ConsistencyOffsetVector used on opensource projects

Example 6 with ConsistencyOffsetVector

use of io.confluent.ksql.util.ConsistencyOffsetVector in project ksql by confluentinc.

the class HARouting method executeOrRouteQuery.

@SuppressWarnings("ParameterNumber")
@VisibleForTesting
static PartitionFetchResult executeOrRouteQuery(final KsqlNode node, final KsqlPartitionLocation location, final ConfiguredStatement<Query> statement, final ServiceContext serviceContext, final RoutingOptions routingOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final PullPhysicalPlan pullPhysicalPlan, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory = (rawRow, schema) -> new PullQueryRow(rawRow, schema, Optional.ofNullable(routingOptions.getIsDebugRequest() ? node : null), Optional.empty());
    if (node.isLocal()) {
        try {
            LOG.debug("Query {} executed locally at host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordLocalRequests(1));
            synchronized (pullPhysicalPlan) {
                pullPhysicalPlan.execute(ImmutableList.of(location), pullQueryQueue, rowFactory);
                return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
            }
        } catch (StandbyFallbackException | NotUpToBoundException e) {
            LOG.warn("Error executing query locally at node {}. Falling back to standby state which " + "may return stale results. Cause {}", node, e.getMessage());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error executing query locally at node %s: %s", node.location(), e.getMessage()), e);
        }
    } else {
        try {
            LOG.debug("Query {} routed to host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordRemoteRequests(1));
            forwardTo(node, ImmutableList.of(location), statement, serviceContext, pullQueryQueue, rowFactory, outputSchema, shouldCancelRequests, consistencyOffsetVector);
            return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
        } catch (StandbyFallbackException e) {
            LOG.warn("Error forwarding query to node {}. Falling back to standby state which may " + "return stale results", node.location(), e.getCause());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error forwarding query to node %s: %s", node.location(), e.getMessage()), e);
        }
    }
}

Example 7 with ConsistencyOffsetVector

use of io.confluent.ksql.util.ConsistencyOffsetVector in project ksql by confluentinc.

the class HARouting method handlePullQuery.

public CompletableFuture<Void> handlePullQuery(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final List<KsqlPartitionLocation> allLocations = pullPhysicalPlan.getMaterialization().locator().locate(pullPhysicalPlan.getKeys(), routingOptions, routingFilterFactory, pullPhysicalPlan.getPlanType() == PullPhysicalPlanType.RANGE_SCAN);
    final Map<Integer, List<Host>> emptyPartitions = allLocations.stream().filter(loc -> loc.getNodes().stream().noneMatch(node -> node.getHost().isSelected())).collect(Collectors.toMap(KsqlPartitionLocation::getPartition, loc -> loc.getNodes().stream().map(KsqlNode::getHost).collect(Collectors.toList())));
    if (!emptyPartitions.isEmpty()) {
        final MaterializationException materializationException = new MaterializationException("Unable to execute pull query. " + emptyPartitions.entrySet().stream().map(kv -> String.format("Partition %s failed to find valid host. Hosts scanned: %s", kv.getKey(), kv.getValue())).collect(Collectors.joining(", ", "[", "]")));
        LOG.debug(materializationException.getMessage());
        throw materializationException;
    }
    // at this point we should filter out the hosts that we should not route to
    final List<KsqlPartitionLocation> locations = allLocations.stream().map(KsqlPartitionLocation::removeFilteredHosts).collect(Collectors.toList());
    final CompletableFuture<Void> completableFuture = new CompletableFuture<>();
    coordinatorExecutorService.submit(() -> {
        try {
            executeRounds(serviceContext, pullPhysicalPlan, statement, routingOptions, outputSchema, queryId, locations, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
            completableFuture.complete(null);
        } catch (Throwable t) {
            completableFuture.completeExceptionally(t);
        }
    });
    return completableFuture;
}

Example 8 with ConsistencyOffsetVector

use of io.confluent.ksql.util.ConsistencyOffsetVector in project ksql by confluentinc.

the class HARouting method streamedRowsHandler.

private static Consumer<List<StreamedRow>> streamedRowsHandler(final KsqlNode owner, final PullQueryQueue pullQueryQueue, final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory, final LogicalSchema outputSchema, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final AtomicInteger processedRows = new AtomicInteger(0);
    final AtomicReference<Header> header = new AtomicReference<>();
    return streamedRows -> {
        try {
            if (streamedRows == null || streamedRows.isEmpty()) {
                return;
            }
            final List<PullQueryRow> rows = new ArrayList<>();
            // If this is the first row overall, skip the header
            final int previousProcessedRows = processedRows.getAndAdd(streamedRows.size());
            for (int i = 0; i < streamedRows.size(); i++) {
                final StreamedRow row = streamedRows.get(i);
                if (i == 0 && previousProcessedRows == 0) {
                    final Optional<Header> optionalHeader = row.getHeader();
                    optionalHeader.ifPresent(h -> validateSchema(outputSchema, h.getSchema(), owner));
                    optionalHeader.ifPresent(header::set);
                    continue;
                }
                if (row.getErrorMessage().isPresent()) {
                    // If we receive an error that's not a network error, we let that bubble up.
                    throw new KsqlException(row.getErrorMessage().get().getMessage());
                }
                if (!row.getRow().isPresent()) {
                    parseNonDataRows(row, i, consistencyOffsetVector);
                    continue;
                }
                final List<?> r = row.getRow().get().getColumns();
                Preconditions.checkNotNull(header.get());
                rows.add(rowFactory.apply(r, header.get().getSchema()));
            }
            if (!pullQueryQueue.acceptRows(rows)) {
                LOG.error("Failed to queue all rows");
            }
        } catch (Exception e) {
            throw new KsqlException(e.getMessage(), e);
        }
    };
}

Example 9 with ConsistencyOffsetVector

use of io.confluent.ksql.util.ConsistencyOffsetVector in project ksql by confluentinc.

the class EngineExecutor method executeTablePullQuery.

/**
 * Evaluates a pull query by first analyzing it, then building the logical plan and finally
 * the physical plan. The execution is then done using the physical plan in a pipelined manner.
 * @param statement The pull query
 * @param routingOptions Configuration parameters used for HA routing
 * @param pullQueryMetrics JMX metrics
 * @return the rows that are the result of evaluating the pull query
 */
PullQueryResult executeTablePullQuery(final ImmutableAnalysis analysis, final ConfiguredStatement<Query> statement, final HARouting routing, final RoutingOptions routingOptions, final QueryPlannerOptions queryPlannerOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final boolean startImmediately, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (!statement.getStatement().isPullQuery()) {
        throw new IllegalArgumentException("Executor can only handle pull queries");
    }
    final SessionConfig sessionConfig = statement.getSessionConfig();
    // If we ever change how many hops a request can do, we'll need to update this for correct
    // metrics.
    final RoutingNodeType routingNodeType = routingOptions.getIsSkipForwardRequest() ? RoutingNodeType.REMOTE_NODE : RoutingNodeType.SOURCE_NODE;
    PullPhysicalPlan plan = null;
    try {
        // Do not set sessionConfig.getConfig to true! The copying is inefficient and slows down pull
        // query performance significantly.  Instead use QueryPlannerOptions which check overrides
        // deliberately.
        final KsqlConfig ksqlConfig = sessionConfig.getConfig(false);
        final LogicalPlanNode logicalPlan = buildAndValidateLogicalPlan(statement, analysis, ksqlConfig, queryPlannerOptions, false);
        // This is a cancel signal that is used to stop both local operations and requests
        final CompletableFuture<Void> shouldCancelRequests = new CompletableFuture<>();
        plan = buildPullPhysicalPlan(logicalPlan, analysis, queryPlannerOptions, shouldCancelRequests, consistencyOffsetVector);
        final PullPhysicalPlan physicalPlan = plan;
        final PullQueryQueue pullQueryQueue = new PullQueryQueue(analysis.getLimitClause());
        final PullQueryQueuePopulator populator = () -> routing.handlePullQuery(serviceContext, physicalPlan, statement, routingOptions, physicalPlan.getOutputSchema(), physicalPlan.getQueryId(), pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
        final PullQueryResult result = new PullQueryResult(physicalPlan.getOutputSchema(), populator, physicalPlan.getQueryId(), pullQueryQueue, pullQueryMetrics, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType, physicalPlan::getRowsReadFromDataSource, shouldCancelRequests, consistencyOffsetVector);
        if (startImmediately) {
            result.start();
        }
        return result;
    } catch (final Exception e) {
        if (plan == null) {
            pullQueryMetrics.ifPresent(m -> m.recordErrorRateForNoResult(1));
        } else {
            final PullPhysicalPlan physicalPlan = plan;
            pullQueryMetrics.ifPresent(metrics -> metrics.recordErrorRate(1, physicalPlan.getSourceType(), physicalPlan.getPlanType(), routingNodeType));
        }
        final String stmtLower = statement.getStatementText().toLowerCase(Locale.ROOT);
        final String messageLower = e.getMessage().toLowerCase(Locale.ROOT);
        final String stackLower = Throwables.getStackTraceAsString(e).toLowerCase(Locale.ROOT);
        // the contents of the query
        if (messageLower.contains(stmtLower) || stackLower.contains(stmtLower)) {
            final StackTraceElement loc = Iterables.getLast(Throwables.getCausalChain(e)).getStackTrace()[0];
            LOG.error("Failure to execute pull query {} {}, not logging the error message since it " + "contains the query string, which may contain sensitive information. If you " + "see this LOG message, please submit a GitHub ticket and we will scrub " + "the statement text from the error at {}", routingOptions.debugString(), queryPlannerOptions.debugString(), loc);
        } else {
            LOG.error("Failure to execute pull query. {} {}", routingOptions.debugString(), queryPlannerOptions.debugString(), e);
        }
        LOG.debug("Failed pull query text {}, {}", statement.getStatementText(), e);
        throw new KsqlStatementException(e.getMessage() == null ? "Server Error" + Arrays.toString(e.getStackTrace()) : e.getMessage(), statement.getStatementText(), e);
    }
}

Example 10 with ConsistencyOffsetVector

use of io.confluent.ksql.util.ConsistencyOffsetVector in project ksql by confluentinc.

the class HARouting method parseNonDataRows.

private static void parseNonDataRows(final StreamedRow row, final int i, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    if (row.getConsistencyToken().isPresent()) {
        if (consistencyOffsetVector.isPresent()) {
            final String token = row.getConsistencyToken().get().getConsistencyToken();
            final ConsistencyOffsetVector received = ConsistencyOffsetVector.deserialize(token);
            consistencyOffsetVector.get().merge(received);
        }
    } else if (!row.getFinalMessage().isPresent()) {
        throw new KsqlException("Missing row data on row " + i + " of chunk");
    }
}