Examples with ServiceContext io.confluent.ksql.services.ServiceContext used on opensource projects

Example 16 with ServiceContext

use of io.confluent.ksql.services.ServiceContext in project ksql by confluentinc.

the class HARouting method executeRounds.

private void executeRounds(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final List<KsqlPartitionLocation> locations, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) throws InterruptedException {
    final ExecutorCompletionService<PartitionFetchResult> completionService = new ExecutorCompletionService<>(routerExecutorService);
    final int totalPartitions = locations.size();
    int processedPartitions = 0;
    final Map<Integer, List<Exception>> exceptionsPerPartition = new HashMap<>();
    for (final KsqlPartitionLocation partition : locations) {
        final KsqlNode node = getNodeForRound(partition, routingOptions);
        pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordPartitionFetchRequest(1));
        completionService.submit(() -> routeQuery.routeQuery(node, partition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
    }
    while (processedPartitions < totalPartitions) {
        final Future<PartitionFetchResult> future = completionService.take();
        try {
            final PartitionFetchResult fetchResult = future.get();
            if (fetchResult.isError()) {
                exceptionsPerPartition.computeIfAbsent(fetchResult.location.getPartition(), v -> new ArrayList<>()).add(fetchResult.exception.get());
                final KsqlPartitionLocation nextRoundPartition = nextNode(fetchResult.getLocation());
                final KsqlNode node = getNodeForRound(nextRoundPartition, routingOptions);
                pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordResubmissionRequest(1));
                completionService.submit(() -> routeQuery.routeQuery(node, nextRoundPartition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
            } else {
                Preconditions.checkState(fetchResult.getResult() == RoutingResult.SUCCESS);
                processedPartitions++;
            }
        } catch (final Exception e) {
            final MaterializationException exception = new MaterializationException("Unable to execute pull query: " + e.getMessage());
            for (Entry<Integer, List<Exception>> entry : exceptionsPerPartition.entrySet()) {
                for (Exception excp : entry.getValue()) {
                    exception.addSuppressed(excp);
                }
            }
            throw exception;
        }
    }
    pullQueryQueue.close();
}

Example 17 with ServiceContext

use of io.confluent.ksql.services.ServiceContext in project ksql by confluentinc.

the class HARouting method executeOrRouteQuery.

@SuppressWarnings("ParameterNumber")
@VisibleForTesting
static PartitionFetchResult executeOrRouteQuery(final KsqlNode node, final KsqlPartitionLocation location, final ConfiguredStatement<Query> statement, final ServiceContext serviceContext, final RoutingOptions routingOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final PullPhysicalPlan pullPhysicalPlan, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory = (rawRow, schema) -> new PullQueryRow(rawRow, schema, Optional.ofNullable(routingOptions.getIsDebugRequest() ? node : null), Optional.empty());
    if (node.isLocal()) {
        try {
            LOG.debug("Query {} executed locally at host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordLocalRequests(1));
            synchronized (pullPhysicalPlan) {
                pullPhysicalPlan.execute(ImmutableList.of(location), pullQueryQueue, rowFactory);
                return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
            }
        } catch (StandbyFallbackException | NotUpToBoundException e) {
            LOG.warn("Error executing query locally at node {}. Falling back to standby state which " + "may return stale results. Cause {}", node, e.getMessage());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error executing query locally at node %s: %s", node.location(), e.getMessage()), e);
        }
    } else {
        try {
            LOG.debug("Query {} routed to host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordRemoteRequests(1));
            forwardTo(node, ImmutableList.of(location), statement, serviceContext, pullQueryQueue, rowFactory, outputSchema, shouldCancelRequests, consistencyOffsetVector);
            return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
        } catch (StandbyFallbackException e) {
            LOG.warn("Error forwarding query to node {}. Falling back to standby state which may " + "return stale results", node.location(), e.getCause());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error forwarding query to node %s: %s", node.location(), e.getMessage()), e);
        }
    }
}

Example 18 with ServiceContext

use of io.confluent.ksql.services.ServiceContext in project ksql by confluentinc.

the class HARouting method handlePullQuery.

public CompletableFuture<Void> handlePullQuery(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final List<KsqlPartitionLocation> allLocations = pullPhysicalPlan.getMaterialization().locator().locate(pullPhysicalPlan.getKeys(), routingOptions, routingFilterFactory, pullPhysicalPlan.getPlanType() == PullPhysicalPlanType.RANGE_SCAN);
    final Map<Integer, List<Host>> emptyPartitions = allLocations.stream().filter(loc -> loc.getNodes().stream().noneMatch(node -> node.getHost().isSelected())).collect(Collectors.toMap(KsqlPartitionLocation::getPartition, loc -> loc.getNodes().stream().map(KsqlNode::getHost).collect(Collectors.toList())));
    if (!emptyPartitions.isEmpty()) {
        final MaterializationException materializationException = new MaterializationException("Unable to execute pull query. " + emptyPartitions.entrySet().stream().map(kv -> String.format("Partition %s failed to find valid host. Hosts scanned: %s", kv.getKey(), kv.getValue())).collect(Collectors.joining(", ", "[", "]")));
        LOG.debug(materializationException.getMessage());
        throw materializationException;
    }
    // at this point we should filter out the hosts that we should not route to
    final List<KsqlPartitionLocation> locations = allLocations.stream().map(KsqlPartitionLocation::removeFilteredHosts).collect(Collectors.toList());
    final CompletableFuture<Void> completableFuture = new CompletableFuture<>();
    coordinatorExecutorService.submit(() -> {
        try {
            executeRounds(serviceContext, pullPhysicalPlan, statement, routingOptions, outputSchema, queryId, locations, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
            completableFuture.complete(null);
        } catch (Throwable t) {
            completableFuture.completeExceptionally(t);
        }
    });
    return completableFuture;
}

Example 19 with ServiceContext

use of io.confluent.ksql.services.ServiceContext in project ksql by confluentinc.

the class PersistentQueryCleanupImpl method cleanupLeakedQueries.

public void cleanupLeakedQueries(final List<PersistentQueryMetadata> persistentQueries) {
    final Set<String> stateStoreNames = persistentQueries.stream().flatMap(s -> {
        final List<String> doNotDelete = new ArrayList<>(Collections.singletonList(s.getQueryApplicationId()));
        if (s instanceof BinPackedPersistentQueryMetadataImpl) {
            doNotDelete.add(s.getQueryApplicationId() + "/__" + s.getQueryId().toString() + "__");
        }
        return doNotDelete.stream();
    }).collect(Collectors.toSet());
    final String[] stateDirFileNames = new File(stateDir).list();
    if (stateDirFileNames == null) {
        LOG.info("No state stores to clean up");
    } else {
        final Set<String> allStateStores = Arrays.stream(stateDirFileNames).flatMap(f -> {
            final String[] fileNames = new File(stateDir + "/" + f).list();
            if (null == fileNames) {
                return Stream.of(f);
            } else if (Arrays.stream(fileNames).anyMatch(t -> t.matches("__*__"))) {
                return Arrays.stream(fileNames).filter(t -> t.matches("__*__")).map(s -> f + "/" + s);
            } else {
                return Stream.of(f);
            }
        }).collect(Collectors.toSet());
        allStateStores.removeAll(stateStoreNames);
        allStateStores.forEach((storeName) -> queryCleanupService.addCleanupTask(new QueryCleanupService.QueryCleanupTask(serviceContext, storeName.split("/")[0], 1 < storeName.split("__").length ? Optional.of(storeName.split("__")[1]) : Optional.empty(), false, stateDir, ksqlConfig.getString(KsqlConfig.KSQL_SERVICE_ID_CONFIG), ksqlConfig.getString(KsqlConfig.KSQL_PERSISTENT_QUERY_NAME_PREFIX_CONFIG))));
    }
}

Example 20 with ServiceContext

use of io.confluent.ksql.services.ServiceContext in project ksql by confluentinc.

the class PrintTopicUtil method createTopicConsumer.

static KafkaConsumer<Bytes, Bytes> createTopicConsumer(final ServiceContext serviceContext, final Map<String, Object> consumerProperties, final PrintTopic printTopic) {
    final KafkaConsumer<Bytes, Bytes> topicConsumer = new KafkaConsumer<>(injectSupplierProperties(serviceContext, consumerProperties), new BytesDeserializer(), new BytesDeserializer());
    final List<TopicPartition> topicPartitions = topicConsumer.partitionsFor(printTopic.getTopic()).stream().map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition())).collect(Collectors.toList());
    topicConsumer.assign(topicPartitions);
    if (printTopic.getFromBeginning()) {
        topicConsumer.seekToBeginning(topicPartitions);
    }
    return topicConsumer;
}