Examples with LagInfoEntity io.confluent.ksql.rest.entity.LagInfoEntity used on opensource projects

Example 1 with LagInfoEntity

use of io.confluent.ksql.rest.entity.LagInfoEntity in project ksql by confluentinc.

the class MaximumLagFilter method create.

/**
 * Creates a FreshnessFilter
 * @param lagReportingAgent The optional lag reporting agent.
 * @param routingOptions The routing options
 * @param hosts The set of all hosts that have the store, including actives and standbys
 * @param applicationQueryId The query id of the persistent query that materialized the table
 * @param storeName The state store name of the materialized table
 * @param partition The partition of the topic
 * @return a new FreshnessFilter, unless lag reporting is disabled.
 */
public static Optional<MaximumLagFilter> create(final Optional<LagReportingAgent> lagReportingAgent, final RoutingOptions routingOptions, final List<KsqlHostInfo> hosts, final String applicationQueryId, final String storeName, final int partition) {
    if (!lagReportingAgent.isPresent()) {
        return Optional.empty();
    }
    final QueryStateStoreId queryStateStoreId = QueryStateStoreId.of(applicationQueryId, storeName);
    final ImmutableMap<KsqlHostInfo, Optional<LagInfoEntity>> lagByHost = hosts.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), host -> lagReportingAgent.get().getLagInfoForHost(host, queryStateStoreId, partition)));
    final OptionalLong maxEndOffset = lagByHost.values().stream().filter(Optional::isPresent).map(Optional::get).mapToLong(LagInfoEntity::getEndOffsetPosition).max();
    return Optional.of(new MaximumLagFilter(routingOptions, lagByHost, maxEndOffset));
}

Example 2 with LagInfoEntity

use of io.confluent.ksql.rest.entity.LagInfoEntity in project ksql by confluentinc.

the class LagReportingAgentTest method shouldReceiveLags_listAllCurrentPositions.

@Test
public void shouldReceiveLags_listAllCurrentPositions() {
    // When:
    lagReportingAgent.receiveHostLag(hostLag(HOST_ENTITY1, LAG_MAP1, 100));
    lagReportingAgent.receiveHostLag(hostLag(HOST_ENTITY2, LAG_MAP2, 200));
    lagReportingAgent.onHostStatusUpdated(HOSTS_ALIVE);
    // Then:
    ImmutableMap<KsqlHostInfoEntity, HostStoreLags> allLags = lagReportingAgent.getAllLags();
    LagInfoEntity lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(1)).get();
    assertEquals(M1_A1_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_A1_END, lag.getEndOffsetPosition());
    assertEquals(M1_A1_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(3)).get();
    assertEquals(M1_A3_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_A3_END, lag.getEndOffsetPosition());
    assertEquals(M1_A3_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_B).flatMap(s -> s.getLagByPartition(4)).get();
    assertEquals(M1_B4_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_B4_END, lag.getEndOffsetPosition());
    assertEquals(M1_B4_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY2).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(1)).get();
    assertEquals(M2_A1_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M2_A1_END, lag.getEndOffsetPosition());
    assertEquals(M2_A1_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY2).getStateStoreLags(QUERY_STORE_B).flatMap(s -> s.getLagByPartition(4)).get();
    assertEquals(M2_B4_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M2_B4_END, lag.getEndOffsetPosition());
    assertEquals(M2_B4_LAG, lag.getOffsetLag());
}

Example 3 with LagInfoEntity

use of io.confluent.ksql.rest.entity.LagInfoEntity in project ksql by confluentinc.

the class HighAvailabilityTestUtil method getOffsets.

// Gets (current, end) offsets for the given host.  Makes the simplified assumption that there's
// just one state store.
public static Pair<Long, Long> getOffsets(final KsqlHostInfoEntity server, final Map<KsqlHostInfoEntity, HostStatusEntity> clusterStatus) {
    HostStatusEntity hostStatusEntity = clusterStatus.get(server);
    long end = hostStatusEntity.getHostStoreLags().getStateStoreLags().values().stream().flatMap(stateStoreLags -> stateStoreLags.getLagByPartition().values().stream()).mapToLong(LagInfoEntity::getEndOffsetPosition).max().orElse(0);
    long current = hostStatusEntity.getHostStoreLags().getStateStoreLags().values().stream().flatMap(stateStoreLags -> stateStoreLags.getLagByPartition().values().stream()).mapToLong(LagInfoEntity::getCurrentOffsetPosition).max().orElse(0);
    return Pair.of(current, end);
}

Example 4 with LagInfoEntity

use of io.confluent.ksql.rest.entity.LagInfoEntity in project ksql by confluentinc.

the class LagReportingAgentFunctionalTest method shouldExchangeLags.

@Test(timeout = 60000)
public void shouldExchangeLags() {
    // Given:
    waitForClusterCondition(LagReportingAgentFunctionalTest::allServersDiscovered);
    // When:
    ClusterStatusResponse resp = waitForClusterCondition(LagReportingAgentFunctionalTest::allLagsReported);
    StateStoreLags stateStoreLags = resp.getClusterStatus().entrySet().iterator().next().getValue().getHostStoreLags().getStateStoreLags(STORE_0).get();
    // Then:
    // Read the raw Kafka data from the topic to verify the reported lags
    final List<ConsumerRecord<byte[], byte[]>> records = TEST_HARNESS.verifyAvailableRecords("_confluent-ksql-default_query_CTAS_USER_VIEWS_3-" + "Aggregate-Aggregate-Materialize-changelog", NUM_ROWS);
    Map<Integer, Optional<ConsumerRecord<byte[], byte[]>>> partitionToMaxOffset = records.stream().collect(Collectors.groupingBy(ConsumerRecord::partition, Collectors.maxBy(Comparator.comparingLong(ConsumerRecord::offset))));
    Assert.assertEquals(2, partitionToMaxOffset.size());
    Optional<LagInfoEntity> lagInfoEntity0 = stateStoreLags.getLagByPartition(0);
    Optional<LagInfoEntity> lagInfoEntity1 = stateStoreLags.getLagByPartition(1);
    long partition0Offset = lagInfoEntity0.get().getCurrentOffsetPosition();
    long partition1Offset = lagInfoEntity1.get().getCurrentOffsetPosition();
    Assert.assertEquals(partition0Offset, partitionToMaxOffset.get(0).get().offset() + 1);
    Assert.assertEquals(partition1Offset, partitionToMaxOffset.get(1).get().offset() + 1);
}

Example 5 with LagInfoEntity

use of io.confluent.ksql.rest.entity.LagInfoEntity in project ksql by confluentinc.

the class MaximumLagFilter method filter.

@Override
public Host filter(final KsqlHostInfo hostInfo) {
    final long allowedOffsetLag = routingOptions.getMaxOffsetLagAllowed();
    final Optional<LagInfoEntity> lagInfoEntity = lagByHost.get(hostInfo);
    if (!lagInfoEntity.isPresent()) {
        // available to promote HA.
        return Host.exclude(hostInfo, "Lag information is not present for host.");
    }
    final LagInfoEntity hostLag = lagInfoEntity.get();
    Preconditions.checkState(maxEndOffset.isPresent(), "Should have a maxEndOffset");
    // Compute the lag from the maximum end offset reported by all hosts.  This is so that
    // hosts that have fallen behind are held to the same end offset when computing lag.
    final long endOffset = maxEndOffset.getAsLong();
    final long offsetLag = Math.max(endOffset - hostLag.getCurrentOffsetPosition(), 0);
    if (offsetLag <= allowedOffsetLag) {
        return Host.include(hostInfo);
    } else {
        return Host.exclude(hostInfo, String.format("Host excluded because lag %s exceeds maximum allowed lag %s.", offsetLag, allowedOffsetLag));
    }
}