Search in sources :

Example 1 with KsqlPartitionLocation

use of io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation in project ksql by confluentinc.

the class KsLocatorTest method shouldReturnLocalOwnerIfSameAsSuppliedLocalHost.

@Test
public void shouldReturnLocalOwnerIfSameAsSuppliedLocalHost() {
    // Given:
    final HostInfo localHostInfo = new HostInfo(LOCAL_HOST_URL.getHost(), LOCAL_HOST_URL.getPort());
    final KsqlHostInfo localHost = locator.asKsqlHost(localHostInfo);
    getActiveAndStandbyMetadata(localHostInfo);
    when(activeFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    when(livenessFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    // When:
    final List<KsqlPartitionLocation> result = locator.locate(ImmutableList.of(KEY), routingOptions, routingFilterFactoryActive, false);
    // Then:
    List<KsqlNode> nodeList = result.get(0).getNodes();
    assertThat(nodeList.stream().findFirst().map(KsqlNode::isLocal), is(Optional.of(true)));
}
Also used : KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) HostInfo(org.apache.kafka.streams.state.HostInfo) KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) Test(org.junit.Test)

Example 2 with KsqlPartitionLocation

use of io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation in project ksql by confluentinc.

the class KsLocatorTest method shouldReturnRemoteOwnerForDifferentHost.

@Test
public void shouldReturnRemoteOwnerForDifferentHost() {
    // Given:
    final HostInfo localHostInfo = new HostInfo("different", LOCAL_HOST_URL.getPort());
    final KsqlHostInfo localHost = locator.asKsqlHost(localHostInfo);
    getActiveAndStandbyMetadata(localHostInfo);
    when(activeFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    when(livenessFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    // When:
    final List<KsqlPartitionLocation> result = locator.locate(ImmutableList.of(KEY), routingOptions, routingFilterFactoryActive, false);
    // Then:
    List<KsqlNode> nodeList = result.get(0).getNodes();
    assertThat(nodeList.stream().findFirst().map(KsqlNode::isLocal), is(Optional.of(false)));
}
Also used : KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) HostInfo(org.apache.kafka.streams.state.HostInfo) KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) Test(org.junit.Test)

Example 3 with KsqlPartitionLocation

use of io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation in project ksql by confluentinc.

the class KsLocatorTest method shouldReturnRemoteOwnerForDifferentPortOnLocalHost.

@Test
public void shouldReturnRemoteOwnerForDifferentPortOnLocalHost() {
    // Given:
    final HostInfo localHostInfo = new HostInfo("LOCALhost", LOCAL_HOST_URL.getPort() + 1);
    final KsqlHostInfo localHost = locator.asKsqlHost(localHostInfo);
    getActiveAndStandbyMetadata(localHostInfo);
    when(activeFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    when(livenessFilter.filter(eq(localHost))).thenReturn(Host.include(localHost));
    // When:
    final List<KsqlPartitionLocation> result = locator.locate(ImmutableList.of(KEY), routingOptions, routingFilterFactoryActive, false);
    // Then:
    List<KsqlNode> nodeList = result.get(0).getNodes();
    assertThat(nodeList.stream().findFirst().map(KsqlNode::isLocal), is(Optional.of(false)));
}
Also used : KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) HostInfo(org.apache.kafka.streams.state.HostInfo) KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) Test(org.junit.Test)

Example 4 with KsqlPartitionLocation

use of io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation in project ksql by confluentinc.

the class HARouting method executeRounds.

private void executeRounds(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final List<KsqlPartitionLocation> locations, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) throws InterruptedException {
    final ExecutorCompletionService<PartitionFetchResult> completionService = new ExecutorCompletionService<>(routerExecutorService);
    final int totalPartitions = locations.size();
    int processedPartitions = 0;
    final Map<Integer, List<Exception>> exceptionsPerPartition = new HashMap<>();
    for (final KsqlPartitionLocation partition : locations) {
        final KsqlNode node = getNodeForRound(partition, routingOptions);
        pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordPartitionFetchRequest(1));
        completionService.submit(() -> routeQuery.routeQuery(node, partition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
    }
    while (processedPartitions < totalPartitions) {
        final Future<PartitionFetchResult> future = completionService.take();
        try {
            final PartitionFetchResult fetchResult = future.get();
            if (fetchResult.isError()) {
                exceptionsPerPartition.computeIfAbsent(fetchResult.location.getPartition(), v -> new ArrayList<>()).add(fetchResult.exception.get());
                final KsqlPartitionLocation nextRoundPartition = nextNode(fetchResult.getLocation());
                final KsqlNode node = getNodeForRound(nextRoundPartition, routingOptions);
                pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordResubmissionRequest(1));
                completionService.submit(() -> routeQuery.routeQuery(node, nextRoundPartition, statement, serviceContext, routingOptions, pullQueryMetrics, pullPhysicalPlan, outputSchema, queryId, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector));
            } else {
                Preconditions.checkState(fetchResult.getResult() == RoutingResult.SUCCESS);
                processedPartitions++;
            }
        } catch (final Exception e) {
            final MaterializationException exception = new MaterializationException("Unable to execute pull query: " + e.getMessage());
            for (Entry<Integer, List<Exception>> entry : exceptionsPerPartition.entrySet()) {
                for (Exception excp : entry.getValue()) {
                    exception.addSuppressed(excp);
                }
            }
            throw exception;
        }
    }
    pullQueryQueue.close();
}
Also used : ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) Query(io.confluent.ksql.parser.tree.Query) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) MaterializationException(io.confluent.ksql.execution.streams.materialization.MaterializationException) RoutingFilterFactory(io.confluent.ksql.execution.streams.RoutingFilter.RoutingFilterFactory) BiFunction(java.util.function.BiFunction) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) Header(io.confluent.ksql.rest.entity.StreamedRow.Header) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RestResponse(io.confluent.ksql.rest.client.RestResponse) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) ImmutableList(com.google.common.collect.ImmutableList) NotUpToBoundException(io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException) Host(io.confluent.ksql.execution.streams.RoutingFilter.Host) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) ExecutorService(java.util.concurrent.ExecutorService) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) Consumer(java.util.function.Consumer) List(java.util.List) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) MaterializationException(io.confluent.ksql.execution.streams.materialization.MaterializationException) NotUpToBoundException(io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException) KsqlException(io.confluent.ksql.util.KsqlException) MaterializationException(io.confluent.ksql.execution.streams.materialization.MaterializationException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Entry(java.util.Map.Entry) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation)

Example 5 with KsqlPartitionLocation

use of io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation in project ksql by confluentinc.

the class HARouting method executeOrRouteQuery.

@SuppressWarnings("ParameterNumber")
@VisibleForTesting
static PartitionFetchResult executeOrRouteQuery(final KsqlNode node, final KsqlPartitionLocation location, final ConfiguredStatement<Query> statement, final ServiceContext serviceContext, final RoutingOptions routingOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final PullPhysicalPlan pullPhysicalPlan, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory = (rawRow, schema) -> new PullQueryRow(rawRow, schema, Optional.ofNullable(routingOptions.getIsDebugRequest() ? node : null), Optional.empty());
    if (node.isLocal()) {
        try {
            LOG.debug("Query {} executed locally at host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordLocalRequests(1));
            synchronized (pullPhysicalPlan) {
                pullPhysicalPlan.execute(ImmutableList.of(location), pullQueryQueue, rowFactory);
                return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
            }
        } catch (StandbyFallbackException | NotUpToBoundException e) {
            LOG.warn("Error executing query locally at node {}. Falling back to standby state which " + "may return stale results. Cause {}", node, e.getMessage());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error executing query locally at node %s: %s", node.location(), e.getMessage()), e);
        }
    } else {
        try {
            LOG.debug("Query {} routed to host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordRemoteRequests(1));
            forwardTo(node, ImmutableList.of(location), statement, serviceContext, pullQueryQueue, rowFactory, outputSchema, shouldCancelRequests, consistencyOffsetVector);
            return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
        } catch (StandbyFallbackException e) {
            LOG.warn("Error forwarding query to node {}. Falling back to standby state which may " + "return stale results", node.location(), e.getCause());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error forwarding query to node %s: %s", node.location(), e.getMessage()), e);
        }
    }
}
Also used : ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) Query(io.confluent.ksql.parser.tree.Query) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) MaterializationException(io.confluent.ksql.execution.streams.materialization.MaterializationException) RoutingFilterFactory(io.confluent.ksql.execution.streams.RoutingFilter.RoutingFilterFactory) BiFunction(java.util.function.BiFunction) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RoutingOptions(io.confluent.ksql.execution.streams.RoutingOptions) KsqlNode(io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode) Header(io.confluent.ksql.rest.entity.StreamedRow.Header) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RestResponse(io.confluent.ksql.rest.client.RestResponse) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) ImmutableList(com.google.common.collect.ImmutableList) NotUpToBoundException(io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException) Host(io.confluent.ksql.execution.streams.RoutingFilter.Host) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) ExecutorService(java.util.concurrent.ExecutorService) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) Logger(org.slf4j.Logger) ImmutableMap(com.google.common.collect.ImmutableMap) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) Consumer(java.util.function.Consumer) List(java.util.List) PullQueryExecutorMetrics(io.confluent.ksql.internal.PullQueryExecutorMetrics) ConsistencyOffsetVector(io.confluent.ksql.util.ConsistencyOffsetVector) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) KsqlPartitionLocation(io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation) PullPhysicalPlanType(io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType) PullQueryQueue(io.confluent.ksql.query.PullQueryQueue) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) NotUpToBoundException(io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException) KsqlException(io.confluent.ksql.util.KsqlException) MaterializationException(io.confluent.ksql.execution.streams.materialization.MaterializationException) NotUpToBoundException(io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException) KsqlException(io.confluent.ksql.util.KsqlException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

KsqlNode (io.confluent.ksql.execution.streams.materialization.Locator.KsqlNode)8 KsqlPartitionLocation (io.confluent.ksql.execution.streams.materialization.Locator.KsqlPartitionLocation)8 KsqlHostInfo (io.confluent.ksql.util.KsqlHostInfo)5 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Preconditions (com.google.common.base.Preconditions)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)3 Host (io.confluent.ksql.execution.streams.RoutingFilter.Host)3 RoutingFilterFactory (io.confluent.ksql.execution.streams.RoutingFilter.RoutingFilterFactory)3 RoutingOptions (io.confluent.ksql.execution.streams.RoutingOptions)3 MaterializationException (io.confluent.ksql.execution.streams.materialization.MaterializationException)3 NotUpToBoundException (io.confluent.ksql.execution.streams.materialization.ks.NotUpToBoundException)3 PullQueryExecutorMetrics (io.confluent.ksql.internal.PullQueryExecutorMetrics)3 Query (io.confluent.ksql.parser.tree.Query)3 PullPhysicalPlanType (io.confluent.ksql.physical.pull.PullPhysicalPlan.PullPhysicalPlanType)3 PullQueryQueue (io.confluent.ksql.query.PullQueryQueue)3 QueryId (io.confluent.ksql.query.QueryId)3 RestResponse (io.confluent.ksql.rest.client.RestResponse)3 StreamedRow (io.confluent.ksql.rest.entity.StreamedRow)3