use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class KsqlEngineTestUtil method execute.
@SuppressWarnings({ "rawtypes", "unchecked" })
private static ExecuteResult execute(final ServiceContext serviceContext, final KsqlExecutionContext executionContext, final ParsedStatement stmt, final KsqlConfig ksqlConfig, final Map<String, Object> overriddenProperties, final Optional<DefaultSchemaInjector> schemaInjector) {
final PreparedStatement<?> prepared = executionContext.prepare(stmt);
final ConfiguredStatement<?> configured = ConfiguredStatement.of(prepared, SessionConfig.of(ksqlConfig, overriddenProperties));
final ConfiguredStatement<?> withFormats = new DefaultFormatInjector().inject(configured);
final ConfiguredStatement<?> withSchema = schemaInjector.map(injector -> injector.inject(withFormats)).orElse((ConfiguredStatement) withFormats);
final ConfiguredStatement<?> reformatted = new SqlFormatInjector(executionContext).inject(withSchema);
try {
return executionContext.execute(serviceContext, reformatted);
} catch (final KsqlStatementException e) {
// can easily check that the failed statement is the input statement
throw new KsqlStatementException(e.getRawMessage(), stmt.getStatementText(), e.getCause());
}
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class QueryEndpoint method createStatement.
private ConfiguredStatement<Query> createStatement(final String queryString, final Map<String, Object> properties, final Map<String, Object> sessionVariables) {
final List<ParsedStatement> statements = ksqlEngine.parse(queryString);
if ((statements.size() != 1)) {
throw new KsqlStatementException(String.format("Expected exactly one KSQL statement; found %d instead", statements.size()), queryString);
}
final PreparedStatement<?> ps = ksqlEngine.prepare(statements.get(0), sessionVariables.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().toString())));
final Statement statement = ps.getStatement();
if (!(statement instanceof Query)) {
throw new KsqlStatementException("Not a query", queryString);
}
@SuppressWarnings("unchecked") final PreparedStatement<Query> psq = (PreparedStatement<Query>) ps;
return ConfiguredStatement.of(psq, SessionConfig.of(ksqlConfig, properties));
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class HARouting method executeRounds.
private void executeRounds(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final List<KsqlPartitionLocation> locations, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) throws InterruptedException {
final ExecutorCompletionService<PartitionFetchResult> completionService = new ExecutorCompletionService<>(routerExecutorService);
final int totalPartitions = locations.size();
int processedPartitions = 0;
final Map<Integer, List<Exception>> exceptionsPerPartition = new HashMap<>();
for (final KsqlPartitionLocation partition : locations) {
final KsqlNode node = getNodeForRound(partition, routingOptions);
pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordPartitionFetchRequest(1));
completionService.submit(() -> routeQuery.routeQuery(node, partition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
}
while (processedPartitions < totalPartitions) {
final Future<PartitionFetchResult> future = completionService.take();
try {
final PartitionFetchResult fetchResult = future.get();
if (fetchResult.isError()) {
exceptionsPerPartition.computeIfAbsent(fetchResult.location.getPartition(), v -> new ArrayList<>()).add(fetchResult.exception.get());
final KsqlPartitionLocation nextRoundPartition = nextNode(fetchResult.getLocation());
final KsqlNode node = getNodeForRound(nextRoundPartition, routingOptions);
pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordResubmissionRequest(1));
completionService.submit(() -> routeQuery.routeQuery(node, nextRoundPartition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
} else {
Preconditions.checkState(fetchResult.getResult() == RoutingResult.SUCCESS);
processedPartitions++;
}
} catch (final Exception e) {
final MaterializationException exception = new MaterializationException("Unable to execute pull query: " + e.getMessage());
for (Entry<Integer, List<Exception>> entry : exceptionsPerPartition.entrySet()) {
for (Exception excp : entry.getValue()) {
exception.addSuppressed(excp);
}
}
throw exception;
}
}
pullQueryQueue.close();
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class HARouting method executeOrRouteQuery.
@SuppressWarnings("ParameterNumber")
@VisibleForTesting
static PartitionFetchResult executeOrRouteQuery(final KsqlNode node, final KsqlPartitionLocation location, final ConfiguredStatement<Query> statement, final ServiceContext serviceContext, final RoutingOptions routingOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final PullPhysicalPlan pullPhysicalPlan, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory = (rawRow, schema) -> new PullQueryRow(rawRow, schema, Optional.ofNullable(routingOptions.getIsDebugRequest() ? node : null), Optional.empty());
if (node.isLocal()) {
try {
LOG.debug("Query {} executed locally at host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordLocalRequests(1));
synchronized (pullPhysicalPlan) {
pullPhysicalPlan.execute(ImmutableList.of(location), pullQueryQueue, rowFactory);
return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
}
} catch (StandbyFallbackException | NotUpToBoundException e) {
LOG.warn("Error executing query locally at node {}. Falling back to standby state which " + "may return stale results. Cause {}", node, e.getMessage());
return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
} catch (Exception e) {
throw new KsqlException(String.format("Error executing query locally at node %s: %s", node.location(), e.getMessage()), e);
}
} else {
try {
LOG.debug("Query {} routed to host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordRemoteRequests(1));
forwardTo(node, ImmutableList.of(location), statement, serviceContext, pullQueryQueue, rowFactory, outputSchema, shouldCancelRequests, consistencyOffsetVector);
return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
} catch (StandbyFallbackException e) {
LOG.warn("Error forwarding query to node {}. Falling back to standby state which may " + "return stale results", node.location(), e.getCause());
return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
} catch (Exception e) {
throw new KsqlException(String.format("Error forwarding query to node %s: %s", node.location(), e.getMessage()), e);
}
}
}
use of io.confluent.ksql.statement.ConfiguredStatement in project ksql by confluentinc.
the class HARouting method handlePullQuery.
public CompletableFuture<Void> handlePullQuery(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
final List<KsqlPartitionLocation> allLocations = pullPhysicalPlan.getMaterialization().locator().locate(pullPhysicalPlan.getKeys(), routingOptions, routingFilterFactory, pullPhysicalPlan.getPlanType() == PullPhysicalPlanType.RANGE_SCAN);
final Map<Integer, List<Host>> emptyPartitions = allLocations.stream().filter(loc -> loc.getNodes().stream().noneMatch(node -> node.getHost().isSelected())).collect(Collectors.toMap(KsqlPartitionLocation::getPartition, loc -> loc.getNodes().stream().map(KsqlNode::getHost).collect(Collectors.toList())));
if (!emptyPartitions.isEmpty()) {
final MaterializationException materializationException = new MaterializationException("Unable to execute pull query. " + emptyPartitions.entrySet().stream().map(kv -> String.format("Partition %s failed to find valid host. Hosts scanned: %s", kv.getKey(), kv.getValue())).collect(Collectors.joining(", ", "[", "]")));
LOG.debug(materializationException.getMessage());
throw materializationException;
}
// at this point we should filter out the hosts that we should not route to
final List<KsqlPartitionLocation> locations = allLocations.stream().map(KsqlPartitionLocation::removeFilteredHosts).collect(Collectors.toList());
final CompletableFuture<Void> completableFuture = new CompletableFuture<>();
coordinatorExecutorService.submit(() -> {
try {
executeRounds(serviceContext, pullPhysicalPlan, statement, routingOptions, outputSchema, queryId, locations, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
completableFuture.complete(null);
} catch (Throwable t) {
completableFuture.completeExceptionally(t);
}
});
return completableFuture;
}
Aggregations