use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class PushRouting method connectToHosts.
/**
* Connects to all of the hosts provided.
* @return A future for a PushConnectionsHandle, which can be used to terminate connections.
*/
@SuppressWarnings("checkstyle:ParameterNumber")
private CompletableFuture<PushConnectionsHandle> connectToHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final Collection<KsqlNode> hosts, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final boolean dynamicallyAddedNode, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final Set<KsqlNode> catchupHosts, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
final Map<KsqlNode, CompletableFuture<RoutingResult>> futureMap = new LinkedHashMap<>();
for (final KsqlNode node : hosts) {
pushConnectionsHandle.add(node, new RoutingResult(RoutingResultStatus.IN_PROGRESS, () -> {
}));
final CompletableFuture<Void> callback = new CompletableFuture<>();
callback.handle((v, t) -> {
if (t == null) {
pushConnectionsHandle.get(node).ifPresent(result -> {
result.close();
result.updateStatus(RoutingResultStatus.COMPLETE);
});
LOG.info("Host {} completed request {}.", node, pushPhysicalPlanManager.getQueryId());
} else if (t instanceof GapFoundException) {
pushConnectionsHandle.get(node).ifPresent(result -> {
result.close();
result.updateStatus(RoutingResultStatus.OFFSET_GAP_FOUND);
});
} else {
pushConnectionsHandle.completeExceptionally(t);
}
return null;
});
futureMap.put(node, executeOrRouteQuery(node, statement, serviceContext, pushPhysicalPlanManager, outputSchema, transientQueryQueue, callback, scalablePushQueryMetrics, pushConnectionsHandle.getOffsetsTracker(), catchupHosts.contains(node), pushRoutingOptions, thisHostName));
}
return CompletableFuture.allOf(futureMap.values().toArray(new CompletableFuture[0])).thenApply(v -> {
for (final KsqlNode node : hosts) {
final CompletableFuture<RoutingResult> future = futureMap.get(node);
final RoutingResult routingResult = future.join();
pushConnectionsHandle.add(node, routingResult);
}
return pushConnectionsHandle;
}).exceptionally(t -> {
final KsqlNode node = futureMap.entrySet().stream().filter(e -> e.getValue().isCompletedExceptionally()).map(Entry::getKey).findFirst().orElse(null);
for (KsqlNode n : hosts) {
final CompletableFuture<RoutingResult> future = futureMap.get(n);
// Take whatever completed exceptionally and mark it as failed
if (future.isCompletedExceptionally()) {
pushConnectionsHandle.get(n).ifPresent(result -> result.updateStatus(RoutingResultStatus.FAILED));
} else {
final RoutingResult routingResult = future.join();
pushConnectionsHandle.add(node, routingResult);
}
}
LOG.warn("Error routing query {} id {} to host {} at timestamp {} with exception {}", statement.getStatementText(), pushPhysicalPlanManager.getQueryId(), node, System.currentTimeMillis(), t.getCause());
// retries in that case and don't fail the original request.
if (!dynamicallyAddedNode) {
pushConnectionsHandle.completeExceptionally(new KsqlException(String.format("Unable to execute push query \"%s\". %s", statement.getStatementText(), t.getCause().getMessage())));
}
return pushConnectionsHandle;
}).exceptionally(t -> {
LOG.error("Unexpected error handing exception", t);
return pushConnectionsHandle;
});
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class PushRouting method checkForNewHosts.
private void checkForNewHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
VertxUtils.checkContext(pushPhysicalPlanManager.getContext());
if (pushConnectionsHandle.isClosed()) {
return;
}
final Set<KsqlNode> updatedHosts = registryToNodes.apply(pushPhysicalPlanManager.getScalablePushRegistry());
final Set<KsqlNode> hosts = pushConnectionsHandle.getActiveHosts();
final Set<KsqlNode> newHosts = Sets.difference(updatedHosts, hosts).stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() != RoutingResultStatus.IN_PROGRESS).orElse(true)).collect(Collectors.toSet());
final Set<KsqlNode> removedHosts = Sets.difference(hosts, updatedHosts);
if (newHosts.size() > 0) {
LOG.info("Dynamically adding new hosts {} for {}", newHosts, pushPhysicalPlanManager.getQueryId());
final Set<KsqlNode> catchupHosts = newHosts.stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() == RoutingResultStatus.OFFSET_GAP_FOUND).orElse(false)).collect(Collectors.toSet());
connectToHosts(serviceContext, pushPhysicalPlanManager, statement, newHosts, outputSchema, transientQueryQueue, pushConnectionsHandle, true, scalablePushQueryMetrics, catchupHosts, pushRoutingOptions, thisHostName);
}
if (removedHosts.size() > 0) {
LOG.info("Dynamically removing hosts {} for {}", removedHosts, pushPhysicalPlanManager.getQueryId());
for (final KsqlNode node : removedHosts) {
final RoutingResult result = pushConnectionsHandle.remove(node);
result.close();
result.updateStatus(RoutingResultStatus.REMOVED);
}
}
pushPhysicalPlanManager.getContext().owner().setTimer(clusterCheckInterval, timerId -> checkForNewHosts(serviceContext, pushPhysicalPlanManager, statement, outputSchema, transientQueryQueue, pushConnectionsHandle, scalablePushQueryMetrics, pushRoutingOptions, thisHostName));
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class ListConnectorsExecutor method execute.
@SuppressWarnings("OptionalGetWithoutIsPresent")
public StatementExecutorResponse execute(final ConfiguredStatement<ListConnectors> configuredStatement, final SessionProperties sessionProperties, final KsqlExecutionContext ksqlExecutionContext, final ServiceContext serviceContext) {
final ConnectClient connectClient = serviceContext.getConnectClient();
final ConnectResponse<List<String>> connectors = serviceContext.getConnectClient().connectors();
if (connectors.error().isPresent()) {
return StatementExecutorResponse.handled(connectErrorHandler.handle(configuredStatement, connectors));
}
final List<SimpleConnectorInfo> infos = new ArrayList<>();
final List<KsqlWarning> warnings = new ArrayList<>();
final Scope scope = configuredStatement.getStatement().getScope();
for (final String name : connectors.datum().get()) {
final ConnectResponse<ConnectorInfo> response = connectClient.describe(name);
if (response.datum().filter(i -> inScope(i.type(), scope)).isPresent()) {
final ConnectResponse<ConnectorStateInfo> status = connectClient.status(name);
infos.add(fromConnectorInfoResponse(name, response, status));
} else if (response.error().isPresent()) {
if (scope == Scope.ALL) {
infos.add(new SimpleConnectorInfo(name, ConnectorType.UNKNOWN, null, null));
}
warnings.add(new KsqlWarning(String.format("Could not describe connector %s: %s", name, response.error().get())));
}
}
return StatementExecutorResponse.handled(Optional.of(new ConnectorList(configuredStatement.getStatementText(), warnings, infos)));
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class ListTopicsExecutor method execute.
public static StatementExecutorResponse execute(final ConfiguredStatement<ListTopics> statement, final SessionProperties sessionProperties, final KsqlExecutionContext executionContext, final ServiceContext serviceContext) {
final KafkaTopicClient client = serviceContext.getTopicClient();
final Map<String, TopicDescription> topicDescriptions = listTopics(client, statement);
if (statement.getStatement().getShowExtended()) {
final KafkaConsumerGroupClient consumerGroupClient = new KafkaConsumerGroupClientImpl(serviceContext::getAdminClient);
final Map<String, List<Integer>> topicConsumersAndGroupCount = getTopicConsumerAndGroupCounts(consumerGroupClient);
final List<KafkaTopicInfoExtended> topicInfoExtendedList = topicDescriptions.values().stream().map(desc -> topicDescriptionToTopicInfoExtended(desc, topicConsumersAndGroupCount)).collect(Collectors.toList());
return StatementExecutorResponse.handled(Optional.of(new KafkaTopicsListExtended(statement.getStatementText(), topicInfoExtendedList)));
} else {
final List<KafkaTopicInfo> topicInfoList = topicDescriptions.values().stream().map(ListTopicsExecutor::topicDescriptionToTopicInfo).collect(Collectors.toList());
return StatementExecutorResponse.handled(Optional.of(new KafkaTopicsList(statement.getStatementText(), topicInfoList)));
}
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class DescribeConnectorExecutor method execute.
@SuppressWarnings("OptionalGetWithoutIsPresent")
public StatementExecutorResponse execute(final ConfiguredStatement<DescribeConnector> configuredStatement, final SessionProperties sessionProperties, final KsqlExecutionContext ksqlExecutionContext, final ServiceContext serviceContext) {
final String connectorName = configuredStatement.getStatement().getConnectorName();
final ConnectResponse<ConnectorStateInfo> statusResponse = serviceContext.getConnectClient().status(connectorName);
if (statusResponse.error().isPresent()) {
return StatementExecutorResponse.handled(connectErrorHandler.handle(configuredStatement, statusResponse));
}
final ConnectResponse<ConnectorInfo> infoResponse = serviceContext.getConnectClient().describe(connectorName);
if (infoResponse.error().isPresent()) {
return StatementExecutorResponse.handled(connectErrorHandler.handle(configuredStatement, infoResponse));
}
final ConnectorStateInfo status = statusResponse.datum().get();
final ConnectorInfo info = infoResponse.datum().get();
final Optional<Connector> connector = connectorFactory.apply(info);
final List<KsqlWarning> warnings;
final List<String> topics;
if (connector.isPresent()) {
// Small optimization. If a connector's info is not found in the response, don't query for
// active topics with the given connectorName
final ConnectResponse<Map<String, Map<String, List<String>>>> topicsResponse = serviceContext.getConnectClient().topics(connectorName);
// server logs.
if (topicsResponse.error().isPresent() && topicsResponse.httpCode() == HttpStatus.SC_NOT_FOUND) {
topics = ImmutableList.of();
warnings = ImmutableList.of();
LOG.warn("Could not list related topics due to error: " + topicsResponse.error().get());
} else if (topicsResponse.error().isPresent()) {
topics = ImmutableList.of();
warnings = ImmutableList.of(new KsqlWarning("Could not list related topics due to error: " + topicsResponse.error().get()));
} else {
topics = topicsResponse.datum().get().get(connectorName).getOrDefault(TOPICS_KEY, ImmutableList.of());
warnings = ImmutableList.of();
}
} else {
topics = ImmutableList.of();
warnings = ImmutableList.of();
}
final List<SourceDescription> sources;
if (connector.isPresent()) {
sources = ksqlExecutionContext.getMetaStore().getAllDataSources().values().stream().filter(source -> topics.contains(source.getKafkaTopicName())).map(source -> SourceDescriptionFactory.create(source, false, ImmutableList.of(), ImmutableList.of(), Optional.empty(), ImmutableList.of(), ImmutableList.of(), ksqlExecutionContext.metricCollectors())).collect(Collectors.toList());
} else {
sources = ImmutableList.of();
}
final ConnectorDescription description = new ConnectorDescription(configuredStatement.getStatementText(), info.config().get(ConnectorConfig.CONNECTOR_CLASS_CONFIG), status, sources, topics, warnings);
return StatementExecutorResponse.handled(Optional.of(description));
}
Aggregations