Search in sources :

Example 1 with KsqlNode

use of io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode in project ksql by confluentinc.

the class PushRouting method connectToHosts.

/**
 * Connects to all of the hosts provided.
 * @return A future for a PushConnectionsHandle, which can be used to terminate connections.
 */
@SuppressWarnings("checkstyle:ParameterNumber")
private CompletableFuture<PushConnectionsHandle> connectToHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final Collection<KsqlNode> hosts, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final boolean dynamicallyAddedNode, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final Set<KsqlNode> catchupHosts, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
    final Map<KsqlNode, CompletableFuture<RoutingResult>> futureMap = new LinkedHashMap<>();
    for (final KsqlNode node : hosts) {
        pushConnectionsHandle.add(node, new RoutingResult(RoutingResultStatus.IN_PROGRESS, () -> {
        }));
        final CompletableFuture<Void> callback = new CompletableFuture<>();
        callback.handle((v, t) -> {
            if (t == null) {
                pushConnectionsHandle.get(node).ifPresent(result -> {
                    result.close();
                    result.updateStatus(RoutingResultStatus.COMPLETE);
                });
                LOG.info("Host {} completed request {}.", node, pushPhysicalPlanManager.getQueryId());
            } else if (t instanceof GapFoundException) {
                pushConnectionsHandle.get(node).ifPresent(result -> {
                    result.close();
                    result.updateStatus(RoutingResultStatus.OFFSET_GAP_FOUND);
                });
            } else {
                pushConnectionsHandle.completeExceptionally(t);
            }
            return null;
        });
        futureMap.put(node, executeOrRouteQuery(node, statement, serviceContext, pushPhysicalPlanManager, outputSchema, transientQueryQueue, callback, scalablePushQueryMetrics, pushConnectionsHandle.getOffsetsTracker(), catchupHosts.contains(node), pushRoutingOptions, thisHostName));
    }
    return CompletableFuture.allOf(futureMap.values().toArray(new CompletableFuture[0])).thenApply(v -> {
        for (final KsqlNode node : hosts) {
            final CompletableFuture<RoutingResult> future = futureMap.get(node);
            final RoutingResult routingResult = future.join();
            pushConnectionsHandle.add(node, routingResult);
        }
        return pushConnectionsHandle;
    }).exceptionally(t -> {
        final KsqlNode node = futureMap.entrySet().stream().filter(e -> e.getValue().isCompletedExceptionally()).map(Entry::getKey).findFirst().orElse(null);
        for (KsqlNode n : hosts) {
            final CompletableFuture<RoutingResult> future = futureMap.get(n);
            // Take whatever completed exceptionally and mark it as failed
            if (future.isCompletedExceptionally()) {
                pushConnectionsHandle.get(n).ifPresent(result -> result.updateStatus(RoutingResultStatus.FAILED));
            } else {
                final RoutingResult routingResult = future.join();
                pushConnectionsHandle.add(node, routingResult);
            }
        }
        LOG.warn("Error routing query {} id {} to host {} at timestamp {} with exception {}", statement.getStatementText(), pushPhysicalPlanManager.getQueryId(), node, System.currentTimeMillis(), t.getCause());
        // retries in that case and don't fail the original request.
        if (!dynamicallyAddedNode) {
            pushConnectionsHandle.completeExceptionally(new KsqlException(String.format("Unable to execute push query \"%s\". %s", statement.getStatementText(), t.getCause().getMessage())));
        }
        return pushConnectionsHandle;
    }).exceptionally(t -> {
        LOG.error("Unexpected error handing exception", t);
        return pushConnectionsHandle;
    });
}
Also used : Query(io.confluent.ksql.parser.tree.Query) LoadingCache(com.google.common.cache.LoadingCache) KeyValue(io.confluent.ksql.util.KeyValue) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RowMetadata(io.confluent.ksql.util.RowMetadata) DataRow(io.confluent.ksql.rest.entity.StreamedRow.DataRow) RestResponse(io.confluent.ksql.rest.client.RestResponse) Context(io.vertx.core.Context) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) PushContinuationToken(io.confluent.ksql.rest.entity.PushContinuationToken) QueryRow(io.confluent.ksql.physical.common.QueryRow) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) BufferedPublisher(io.confluent.ksql.reactive.BufferedPublisher) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) CacheLoader(com.google.common.cache.CacheLoader) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) BaseSubscriber(io.confluent.ksql.reactive.BaseSubscriber) List(java.util.List) VertxUtils(io.confluent.ksql.util.VertxUtils) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) CacheBuilder(com.google.common.cache.CacheBuilder) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) KeyValueMetadata(io.confluent.ksql.util.KeyValueMetadata) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) OffsetVector(io.confluent.ksql.util.OffsetVector) BiConsumer(java.util.function.BiConsumer) Logger(org.slf4j.Logger) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) GenericRow(io.confluent.ksql.GenericRow) Subscription(org.reactivestreams.Subscription) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PushOffsetVector(io.confluent.ksql.util.PushOffsetVector) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) KsqlException(io.confluent.ksql.util.KsqlException) LinkedHashMap(java.util.LinkedHashMap)

Example 2 with KsqlNode

use of io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode in project ksql by confluentinc.

the class PushRouting method checkForNewHosts.

private void checkForNewHosts(final ServiceContext serviceContext, final PushPhysicalPlanManager pushPhysicalPlanManager, final ConfiguredStatement<Query> statement, final LogicalSchema outputSchema, final TransientQueryQueue transientQueryQueue, final PushConnectionsHandle pushConnectionsHandle, final Optional<ScalablePushQueryMetrics> scalablePushQueryMetrics, final PushRoutingOptions pushRoutingOptions, final String thisHostName) {
    VertxUtils.checkContext(pushPhysicalPlanManager.getContext());
    if (pushConnectionsHandle.isClosed()) {
        return;
    }
    final Set<KsqlNode> updatedHosts = registryToNodes.apply(pushPhysicalPlanManager.getScalablePushRegistry());
    final Set<KsqlNode> hosts = pushConnectionsHandle.getActiveHosts();
    final Set<KsqlNode> newHosts = Sets.difference(updatedHosts, hosts).stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() != RoutingResultStatus.IN_PROGRESS).orElse(true)).collect(Collectors.toSet());
    final Set<KsqlNode> removedHosts = Sets.difference(hosts, updatedHosts);
    if (newHosts.size() > 0) {
        LOG.info("Dynamically adding new hosts {} for {}", newHosts, pushPhysicalPlanManager.getQueryId());
        final Set<KsqlNode> catchupHosts = newHosts.stream().filter(node -> pushConnectionsHandle.get(node).map(routingResult -> routingResult.getStatus() == RoutingResultStatus.OFFSET_GAP_FOUND).orElse(false)).collect(Collectors.toSet());
        connectToHosts(serviceContext, pushPhysicalPlanManager, statement, newHosts, outputSchema, transientQueryQueue, pushConnectionsHandle, true, scalablePushQueryMetrics, catchupHosts, pushRoutingOptions, thisHostName);
    }
    if (removedHosts.size() > 0) {
        LOG.info("Dynamically removing hosts {} for {}", removedHosts, pushPhysicalPlanManager.getQueryId());
        for (final KsqlNode node : removedHosts) {
            final RoutingResult result = pushConnectionsHandle.remove(node);
            result.close();
            result.updateStatus(RoutingResultStatus.REMOVED);
        }
    }
    pushPhysicalPlanManager.getContext().owner().setTimer(clusterCheckInterval, timerId -> checkForNewHosts(serviceContext, pushPhysicalPlanManager, statement, outputSchema, transientQueryQueue, pushConnectionsHandle, scalablePushQueryMetrics, pushRoutingOptions, thisHostName));
}
Also used : Query(io.confluent.ksql.parser.tree.Query) LoadingCache(com.google.common.cache.LoadingCache) KeyValue(io.confluent.ksql.util.KeyValue) ServiceContext(io.confluent.ksql.services.ServiceContext) LoggerFactory(org.slf4j.LoggerFactory) RowMetadata(io.confluent.ksql.util.RowMetadata) DataRow(io.confluent.ksql.rest.entity.StreamedRow.DataRow) RestResponse(io.confluent.ksql.rest.client.RestResponse) Context(io.vertx.core.Context) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) PushContinuationToken(io.confluent.ksql.rest.entity.PushContinuationToken) QueryRow(io.confluent.ksql.physical.common.QueryRow) Map(java.util.Map) QueryId(io.confluent.ksql.query.QueryId) KsqlRequestConfig(io.confluent.ksql.util.KsqlRequestConfig) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ScalablePushQueryMetrics(io.confluent.ksql.internal.ScalablePushQueryMetrics) ConfiguredStatement(io.confluent.ksql.statement.ConfiguredStatement) BufferedPublisher(io.confluent.ksql.reactive.BufferedPublisher) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) CacheLoader(com.google.common.cache.CacheLoader) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) BaseSubscriber(io.confluent.ksql.reactive.BaseSubscriber) List(java.util.List) VertxUtils(io.confluent.ksql.util.VertxUtils) Entry(java.util.Map.Entry) KsqlException(io.confluent.ksql.util.KsqlException) Optional(java.util.Optional) TransientQueryQueue(io.confluent.ksql.query.TransientQueryQueue) CacheBuilder(com.google.common.cache.CacheBuilder) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) KeyValueMetadata(io.confluent.ksql.util.KeyValueMetadata) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) OffsetVector(io.confluent.ksql.util.OffsetVector) BiConsumer(java.util.function.BiConsumer) Logger(org.slf4j.Logger) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) GenericRow(io.confluent.ksql.GenericRow) Subscription(org.reactivestreams.Subscription) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) PushOffsetVector(io.confluent.ksql.util.PushOffsetVector) Collections(java.util.Collections) KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode)

Example 3 with KsqlNode

use of io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode in project ksql by confluentinc.

the class AllHostsLocatorTest method shouldLocate.

@Test
public void shouldLocate() throws MalformedURLException {
    // Given:
    final AllHostsLocator locator = new AllHostsLocator(() -> ImmutableList.of(metadata1, metadata2), new URL("http://localhost:8088"));
    when(metadata1.getAllStreamsHostMetadata()).thenReturn(ImmutableList.of(streamsMetadata1, streamsMetadata2));
    when(metadata2.getAllStreamsHostMetadata()).thenReturn(ImmutableList.of(streamsMetadata3));
    when(streamsMetadata1.hostInfo()).thenReturn(new HostInfo("abc", 101), new HostInfo("localhost", 8088));
    when(streamsMetadata2.hostInfo()).thenReturn(new HostInfo("localhost", 8088));
    when(streamsMetadata3.hostInfo()).thenReturn(new HostInfo("localhost", 8089));
    // When:
    final List<KsqlNode> nodes = ImmutableList.copyOf(locator.locate());
    // Then:
    assertThat(nodes.size(), is(3));
    assertThat(nodes.get(0).isLocal(), is(false));
    assertThat(nodes.get(0).location().toString(), is("http://abc:101/"));
    assertThat(nodes.get(1).isLocal(), is(true));
    assertThat(nodes.get(1).location().toString(), is("http://localhost:8088/"));
    assertThat(nodes.get(2).isLocal(), is(false));
    assertThat(nodes.get(2).location().toString(), is("http://localhost:8089/"));
}
Also used : KsqlNode(io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode) HostInfo(org.apache.kafka.streams.state.HostInfo) URL(java.net.URL) Test(org.junit.Test)

Aggregations

KsqlNode (io.confluent.ksql.physical.scalablepush.locator.PushLocator.KsqlNode)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 Preconditions (com.google.common.base.Preconditions)2 CacheBuilder (com.google.common.cache.CacheBuilder)2 CacheLoader (com.google.common.cache.CacheLoader)2 LoadingCache (com.google.common.cache.LoadingCache)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Sets (com.google.common.collect.Sets)2 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)2 GenericRow (io.confluent.ksql.GenericRow)2 ScalablePushQueryMetrics (io.confluent.ksql.internal.ScalablePushQueryMetrics)2 Query (io.confluent.ksql.parser.tree.Query)2 QueryRow (io.confluent.ksql.physical.common.QueryRow)2 QueryId (io.confluent.ksql.query.QueryId)2 TransientQueryQueue (io.confluent.ksql.query.TransientQueryQueue)2 BaseSubscriber (io.confluent.ksql.reactive.BaseSubscriber)2 BufferedPublisher (io.confluent.ksql.reactive.BufferedPublisher)2 RestResponse (io.confluent.ksql.rest.client.RestResponse)2 KsqlErrorMessage (io.confluent.ksql.rest.entity.KsqlErrorMessage)2