Examples with Host io.confluent.ksql.execution.streams.RoutingFilter.Host used on opensource projects

Example 1 with Host

use of io.confluent.ksql.execution.streams.RoutingFilter.Host in project ksql by confluentinc.

the class LivenessFilterTest method shouldFilterAllAlive.

@Test
public void shouldFilterAllAlive() {
    // Given:
    allHostsStatus = ImmutableMap.of(activeHost, HOST_ALIVE, standByHost1, HOST_ALIVE, standByHost2, HOST_ALIVE);
    when(heartbeatAgent.getHostsStatus()).thenReturn(allHostsStatus);
    // When:
    final Host filterActive = livenessFilter.filter(activeHost);
    final Host filterStandby1 = livenessFilter.filter(standByHost1);
    final Host filterStandby2 = livenessFilter.filter(standByHost2);
    // Then:
    assertThat(filterActive.isSelected(), is(true));
    assertThat(filterStandby1.isSelected(), is(true));
    assertThat(filterStandby2.isSelected(), is(true));
}

Example 2 with Host

use of io.confluent.ksql.execution.streams.RoutingFilter.Host in project ksql by confluentinc.

the class LivenessFilterTest method shouldFilterAllDead.

@Test
public void shouldFilterAllDead() {
    // Given:
    allHostsStatus = ImmutableMap.of(activeHost, HOST_DEAD, standByHost1, HOST_DEAD, standByHost2, HOST_DEAD);
    when(heartbeatAgent.getHostsStatus()).thenReturn(allHostsStatus);
    // When:
    final Host filterActive = livenessFilter.filter(activeHost);
    final Host filterStandby1 = livenessFilter.filter(standByHost1);
    final Host filterStandby2 = livenessFilter.filter(standByHost2);
    // Then:
    assertThat(filterActive.isSelected(), is(false));
    assertThat(filterActive.getReasonNotSelected(), is("Host is not alive as of time 0"));
    assertThat(filterStandby1.isSelected(), is(false));
    assertThat(filterActive.getReasonNotSelected(), is("Host is not alive as of time 0"));
    assertThat(filterStandby2.isSelected(), is(false));
    assertThat(filterActive.getReasonNotSelected(), is("Host is not alive as of time 0"));
}

Example 3 with Host

use of io.confluent.ksql.execution.streams.RoutingFilter.Host in project ksql by confluentinc.

the class HARouting method executeOrRouteQuery.

@SuppressWarnings("ParameterNumber")
@VisibleForTesting
static PartitionFetchResult executeOrRouteQuery(final KsqlNode node, final KsqlPartitionLocation location, final ConfiguredStatement<Query> statement, final ServiceContext serviceContext, final RoutingOptions routingOptions, final Optional<PullQueryExecutorMetrics> pullQueryMetrics, final PullPhysicalPlan pullPhysicalPlan, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final BiFunction<List<?>, LogicalSchema, PullQueryRow> rowFactory = (rawRow, schema) -> new PullQueryRow(rawRow, schema, Optional.ofNullable(routingOptions.getIsDebugRequest() ? node : null), Optional.empty());
    if (node.isLocal()) {
        try {
            LOG.debug("Query {} executed locally at host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordLocalRequests(1));
            synchronized (pullPhysicalPlan) {
                pullPhysicalPlan.execute(ImmutableList.of(location), pullQueryQueue, rowFactory);
                return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
            }
        } catch (StandbyFallbackException | NotUpToBoundException e) {
            LOG.warn("Error executing query locally at node {}. Falling back to standby state which " + "may return stale results. Cause {}", node, e.getMessage());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error executing query locally at node %s: %s", node.location(), e.getMessage()), e);
        }
    } else {
        try {
            LOG.debug("Query {} routed to host {} at timestamp {}.", statement.getStatementText(), node.location(), System.currentTimeMillis());
            pullQueryMetrics.ifPresent(queryExecutorMetrics -> queryExecutorMetrics.recordRemoteRequests(1));
            forwardTo(node, ImmutableList.of(location), statement, serviceContext, pullQueryQueue, rowFactory, outputSchema, shouldCancelRequests, consistencyOffsetVector);
            return new PartitionFetchResult(RoutingResult.SUCCESS, location, Optional.empty());
        } catch (StandbyFallbackException e) {
            LOG.warn("Error forwarding query to node {}. Falling back to standby state which may " + "return stale results", node.location(), e.getCause());
            return new PartitionFetchResult(RoutingResult.STANDBY_FALLBACK, location, Optional.of(e));
        } catch (Exception e) {
            throw new KsqlException(String.format("Error forwarding query to node %s: %s", node.location(), e.getMessage()), e);
        }
    }
}

Example 4 with Host

use of io.confluent.ksql.execution.streams.RoutingFilter.Host in project ksql by confluentinc.

the class HARouting method handlePullQuery.

public CompletableFuture<Void> handlePullQuery(final ServiceContext serviceContext, final PullPhysicalPlan pullPhysicalPlan, final ConfiguredStatement<Query> statement, final RoutingOptions routingOptions, final LogicalSchema outputSchema, final QueryId queryId, final PullQueryQueue pullQueryQueue, final CompletableFuture<Void> shouldCancelRequests, final Optional<ConsistencyOffsetVector> consistencyOffsetVector) {
    final List<KsqlPartitionLocation> allLocations = pullPhysicalPlan.getMaterialization().locator().locate(pullPhysicalPlan.getKeys(), routingOptions, routingFilterFactory, pullPhysicalPlan.getPlanType() == PullPhysicalPlanType.RANGE_SCAN);
    final Map<Integer, List<Host>> emptyPartitions = allLocations.stream().filter(loc -> loc.getNodes().stream().noneMatch(node -> node.getHost().isSelected())).collect(Collectors.toMap(KsqlPartitionLocation::getPartition, loc -> loc.getNodes().stream().map(KsqlNode::getHost).collect(Collectors.toList())));
    if (!emptyPartitions.isEmpty()) {
        final MaterializationException materializationException = new MaterializationException("Unable to execute pull query. " + emptyPartitions.entrySet().stream().map(kv -> String.format("Partition %s failed to find valid host. Hosts scanned: %s", kv.getKey(), kv.getValue())).collect(Collectors.joining(", ", "[", "]")));
        LOG.debug(materializationException.getMessage());
        throw materializationException;
    }
    // at this point we should filter out the hosts that we should not route to
    final List<KsqlPartitionLocation> locations = allLocations.stream().map(KsqlPartitionLocation::removeFilteredHosts).collect(Collectors.toList());
    final CompletableFuture<Void> completableFuture = new CompletableFuture<>();
    coordinatorExecutorService.submit(() -> {
        try {
            executeRounds(serviceContext, pullPhysicalPlan, statement, routingOptions, outputSchema, queryId, locations, pullQueryQueue, shouldCancelRequests, consistencyOffsetVector);
            completableFuture.complete(null);
        } catch (Throwable t) {
            completableFuture.completeExceptionally(t);
        }
    });
    return completableFuture;
}

Example 5 with Host

use of io.confluent.ksql.execution.streams.RoutingFilter.Host in project ksql by confluentinc.

the class MaximumLagFilterTest method filter_shouldRemoveWhenNoLag.

@Test
public void filter_shouldRemoveWhenNoLag() {
    // Given:
    when(lagReportingAgent.getLagInfoForHost(eq(HOST1), eq(QueryStateStoreId.of(APPLICATION_ID, STATE_STORE)), eq(PARTITION))).thenReturn(Optional.empty());
    when(routingOptions.getMaxOffsetLagAllowed()).thenReturn(13L);
    // When:
    filter = MaximumLagFilter.create(Optional.of(lagReportingAgent), routingOptions, HOSTS, APPLICATION_ID, STATE_STORE, PARTITION).get();
    // Then:
    final Host host = filter.filter(HOST1);
    assertFalse(host.isSelected());
    assertEquals(host.getReasonNotSelected(), "Lag information is not present for host.");
}