Search in sources :

Example 1 with KsqlHostInfoEntity

use of io.confluent.ksql.rest.entity.KsqlHostInfoEntity in project ksql by confluentinc.

the class HeartbeatResource method handleHeartbeat.

private void handleHeartbeat(final HeartbeatMessage request) {
    final KsqlHostInfoEntity ksqlHostInfoEntity = request.getHostInfo();
    final KsqlHostInfo ksqlHostInfo = new KsqlHostInfo(ksqlHostInfoEntity.getHost(), ksqlHostInfoEntity.getPort());
    final long timestamp = request.getTimestamp();
    heartbeatAgent.receiveHeartbeat(ksqlHostInfo, timestamp);
}
Also used : KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity)

Example 2 with KsqlHostInfoEntity

use of io.confluent.ksql.rest.entity.KsqlHostInfoEntity in project ksql by confluentinc.

the class HeartbeatResourceTest method shouldSendHeartbeat.

@Test
public void shouldSendHeartbeat() {
    // When:
    final HeartbeatMessage request = new HeartbeatMessage(new KsqlHostInfoEntity("localhost", 8080), 1);
    final EndpointResponse response = heartbeatResource.registerHeartbeat(request);
    // Then:
    assertThat(response.getStatus(), is(200));
    assertThat(response.getEntity(), instanceOf(HeartbeatResponse.class));
}
Also used : HeartbeatResponse(io.confluent.ksql.rest.entity.HeartbeatResponse) EndpointResponse(io.confluent.ksql.rest.EndpointResponse) HeartbeatMessage(io.confluent.ksql.rest.entity.HeartbeatMessage) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) Test(org.junit.Test)

Example 3 with KsqlHostInfoEntity

use of io.confluent.ksql.rest.entity.KsqlHostInfoEntity in project ksql by confluentinc.

the class PullQueryRoutingFunctionalTest method shouldQueryStandbyWhenActiveDeadStandbyAliveQueryIssuedToRouter_multipleKeys.

@SuppressWarnings("unchecked")
@Test
public void shouldQueryStandbyWhenActiveDeadStandbyAliveQueryIssuedToRouter_multipleKeys() throws Exception {
    // Given:
    ClusterFormation clusterFormation = findClusterFormation(TEST_APP_0, TEST_APP_1, TEST_APP_2);
    waitForClusterToBeDiscovered(clusterFormation.router.getApp(), 3, USER_CREDS);
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.router.getHost(), HighAvailabilityTestUtil.lagsReported(3), USER_CREDS);
    // Partition off the active
    clusterFormation.active.getShutoffs().shutOffAll();
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.standBy.getHost(), HighAvailabilityTestUtil::remoteServerIsUp, USER_CREDS);
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.active.getHost(), HighAvailabilityTestUtil::remoteServerIsDown, USER_CREDS);
    // When:
    final List<StreamedRow> rows_0 = makePullQueryRequest(clusterFormation.router.getApp(), sqlMultipleKeys, null, USER_CREDS);
    // Then:
    assertThat(rows_0, hasSize(HEADER + 2));
    KsqlHostInfoEntity host = rows_0.get(1).getSourceHost().get();
    assertThat(host.getHost(), is(clusterFormation.standBy.getHost().getHost()));
    assertThat(host.getPort(), is(clusterFormation.standBy.getHost().getPort()));
    host = rows_0.get(2).getSourceHost().get();
    assertThat(host.getHost(), is(clusterFormation.standBy.getHost().getHost()));
    assertThat(host.getPort(), is(clusterFormation.standBy.getHost().getPort()));
    List<List<?>> values = rows_0.stream().skip(HEADER).map(sr -> {
        assertThat(sr.getRow(), is(not(Optional.empty())));
        return sr.getRow().get().getColumns();
    }).collect(Collectors.toList());
    assertThat(values, containsInAnyOrder(ImmutableList.of(KEY, 1), ImmutableList.of(KEY1, 1)));
}
Also used : HighAvailabilityTestUtil.waitForRemoteServerToChangeStatus(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.waitForRemoteServerToChangeStatus) PhysicalSchema(io.confluent.ksql.schema.ksql.PhysicalSchema) HighAvailabilityTestUtil.makePullQueryRequest(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.makePullQueryRequest) ColumnName(io.confluent.ksql.name.ColumnName) Matchers.not(org.hamcrest.Matchers.not) LoggerFactory(org.slf4j.LoggerFactory) RoutingContext(io.vertx.ext.web.RoutingContext) ClusterStatusResponse(io.confluent.ksql.rest.entity.ClusterStatusResponse) KsqlErrorMessage(io.confluent.ksql.rest.entity.KsqlErrorMessage) WorkerExecutor(io.vertx.core.WorkerExecutor) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) HighAvailabilityTestUtil.makeAdminRequestWithResponse(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.makeAdminRequestWithResponse) Map(java.util.Map) After(org.junit.After) FaultyKafkaConsumer0(io.confluent.ksql.rest.integration.FaultyKafkaConsumer.FaultyKafkaConsumer0) FaultyKafkaConsumer2(io.confluent.ksql.rest.integration.FaultyKafkaConsumer.FaultyKafkaConsumer2) FaultyKafkaConsumer1(io.confluent.ksql.rest.integration.FaultyKafkaConsumer.FaultyKafkaConsumer1) ClassRule(org.junit.ClassRule) ZooKeeperClientException(kafka.zookeeper.ZooKeeperClientException) ImmutableMap(com.google.common.collect.ImmutableMap) TestKsqlRestApp(io.confluent.ksql.rest.server.TestKsqlRestApp) Set(java.util.Set) ConsumerConfig(org.apache.kafka.clients.consumer.ConsumerConfig) KsqlConfig(io.confluent.ksql.util.KsqlConfig) UUID(java.util.UUID) RecordMetadata(org.apache.kafka.clients.producer.RecordMetadata) Category(org.junit.experimental.categories.Category) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Collectors(java.util.stream.Collectors) KsqlRestConfig(io.confluent.ksql.rest.server.KsqlRestConfig) List(java.util.List) Principal(java.security.Principal) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Entry(java.util.Map.Entry) Optional(java.util.Optional) Matchers.is(org.hamcrest.Matchers.is) UserDataProvider(io.confluent.ksql.util.UserDataProvider) HighAvailabilityTestUtil.makeAdminRequest(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.makeAdminRequest) Matchers.containsString(org.hamcrest.Matchers.containsString) Retry(io.confluent.ksql.integration.Retry) StreamsConfig(org.apache.kafka.streams.StreamsConfig) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) TaskId(org.apache.kafka.streams.processor.TaskId) BeforeClass(org.junit.BeforeClass) HighAvailabilityTestUtil.waitForStreamsMetadataToInitialize(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.waitForStreamsMetadataToInitialize) FormatFactory(io.confluent.ksql.serde.FormatFactory) AssignmentConfigs(org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs) CompletableFuture(java.util.concurrent.CompletableFuture) KsqlIdentifierTestUtil(io.confluent.ksql.test.util.KsqlIdentifierTestUtil) Multimap(com.google.common.collect.Multimap) HighAvailabilityTestUtil.extractQueryId(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.extractQueryId) KsqlEntity(io.confluent.ksql.rest.entity.KsqlEntity) ImmutableList(com.google.common.collect.ImmutableList) Shutoffs(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.Shutoffs) KsqlTestFolder(io.confluent.ksql.test.util.KsqlTestFolder) Timeout(org.junit.rules.Timeout) Matchers.hasSize(org.hamcrest.Matchers.hasSize) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) SerdeFeatures(io.confluent.ksql.serde.SerdeFeatures) Before(org.junit.Before) AuthenticationPlugin(io.confluent.ksql.api.auth.AuthenticationPlugin) Logger(org.slf4j.Logger) IntegrationTest(io.confluent.common.utils.IntegrationTest) HighAvailabilityTestUtil.waitForClusterToBeDiscovered(io.confluent.ksql.rest.integration.HighAvailabilityTestUtil.waitForClusterToBeDiscovered) CONSUMER_PREFIX(org.apache.kafka.streams.StreamsConfig.CONSUMER_PREFIX) KSQL_STREAMS_PREFIX(io.confluent.ksql.util.KsqlConfig.KSQL_STREAMS_PREFIX) TestUtils(io.confluent.ksql.rest.server.utils.TestUtils) BasicCredentials(io.confluent.ksql.rest.client.BasicCredentials) IOException(java.io.IOException) Test(org.junit.Test) IntegrationTestHarness(io.confluent.ksql.integration.IntegrationTestHarness) TestBasicJaasConfig(io.confluent.ksql.test.util.TestBasicJaasConfig) TimeUnit(java.util.concurrent.TimeUnit) RuleChain(org.junit.rules.RuleChain) ActiveStandbyEntity(io.confluent.ksql.rest.entity.ActiveStandbyEntity) AtomicLong(java.util.concurrent.atomic.AtomicLong) Rule(org.junit.Rule) ClientState(org.apache.kafka.streams.processor.internals.assignment.ClientState) TaskAssignor(org.apache.kafka.streams.processor.internals.assignment.TaskAssignor) Preconditions(com.google.common.base.Preconditions) GenericKey(io.confluent.ksql.GenericKey) Comparator(java.util.Comparator) Assert(org.junit.Assert) SqlTypes(io.confluent.ksql.schema.ksql.types.SqlTypes) TemporaryFolder(org.junit.rules.TemporaryFolder) StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) IntegrationTest(io.confluent.common.utils.IntegrationTest) Test(org.junit.Test)

Example 4 with KsqlHostInfoEntity

use of io.confluent.ksql.rest.entity.KsqlHostInfoEntity in project ksql by confluentinc.

the class PullQueryRoutingFunctionalTest method shouldQueryStandbyWhenActiveDeadStandbyAliveQueryIssuedToRouter.

@Test
public void shouldQueryStandbyWhenActiveDeadStandbyAliveQueryIssuedToRouter() throws Exception {
    // Given:
    ClusterFormation clusterFormation = findClusterFormation(TEST_APP_0, TEST_APP_1, TEST_APP_2);
    waitForClusterToBeDiscovered(clusterFormation.router.getApp(), 3, USER_CREDS);
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.router.getHost(), HighAvailabilityTestUtil.lagsReported(3), USER_CREDS);
    // Partition off the active
    clusterFormation.active.getShutoffs().shutOffAll();
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.standBy.getHost(), HighAvailabilityTestUtil::remoteServerIsUp, USER_CREDS);
    waitForRemoteServerToChangeStatus(clusterFormation.router.getApp(), clusterFormation.active.getHost(), HighAvailabilityTestUtil::remoteServerIsDown, USER_CREDS);
    // When:
    final List<StreamedRow> rows_0 = makePullQueryRequest(clusterFormation.router.getApp(), sql, null, USER_CREDS);
    // Then:
    assertThat(rows_0, hasSize(HEADER + 1));
    KsqlHostInfoEntity host = rows_0.get(1).getSourceHost().get();
    assertThat(host.getHost(), is(clusterFormation.standBy.getHost().getHost()));
    assertThat(host.getPort(), is(clusterFormation.standBy.getHost().getPort()));
    assertThat(rows_0.get(1).getRow(), is(not(Optional.empty())));
    assertThat(rows_0.get(1).getRow().get().getColumns(), is(ImmutableList.of(KEY, 1)));
}
Also used : StreamedRow(io.confluent.ksql.rest.entity.StreamedRow) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) IntegrationTest(io.confluent.common.utils.IntegrationTest) Test(org.junit.Test)

Example 5 with KsqlHostInfoEntity

use of io.confluent.ksql.rest.entity.KsqlHostInfoEntity in project ksql by confluentinc.

the class LagReportingAgentTest method shouldReceiveLags_listAllCurrentPositions.

@Test
public void shouldReceiveLags_listAllCurrentPositions() {
    // When:
    lagReportingAgent.receiveHostLag(hostLag(HOST_ENTITY1, LAG_MAP1, 100));
    lagReportingAgent.receiveHostLag(hostLag(HOST_ENTITY2, LAG_MAP2, 200));
    lagReportingAgent.onHostStatusUpdated(HOSTS_ALIVE);
    // Then:
    ImmutableMap<KsqlHostInfoEntity, HostStoreLags> allLags = lagReportingAgent.getAllLags();
    LagInfoEntity lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(1)).get();
    assertEquals(M1_A1_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_A1_END, lag.getEndOffsetPosition());
    assertEquals(M1_A1_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(3)).get();
    assertEquals(M1_A3_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_A3_END, lag.getEndOffsetPosition());
    assertEquals(M1_A3_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY1).getStateStoreLags(QUERY_STORE_B).flatMap(s -> s.getLagByPartition(4)).get();
    assertEquals(M1_B4_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M1_B4_END, lag.getEndOffsetPosition());
    assertEquals(M1_B4_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY2).getStateStoreLags(QUERY_STORE_A).flatMap(s -> s.getLagByPartition(1)).get();
    assertEquals(M2_A1_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M2_A1_END, lag.getEndOffsetPosition());
    assertEquals(M2_A1_LAG, lag.getOffsetLag());
    lag = allLags.get(HOST_ENTITY2).getStateStoreLags(QUERY_STORE_B).flatMap(s -> s.getLagByPartition(4)).get();
    assertEquals(M2_B4_CUR, lag.getCurrentOffsetPosition());
    assertEquals(M2_B4_END, lag.getEndOffsetPosition());
    assertEquals(M2_B4_LAG, lag.getOffsetLag());
}
Also used : StateStoreLags(io.confluent.ksql.rest.entity.StateStoreLags) Mock(org.mockito.Mock) HostStoreLags(io.confluent.ksql.rest.entity.HostStoreLags) ServiceContext(io.confluent.ksql.services.ServiceContext) RunWith(org.junit.runner.RunWith) LagInfoEntity(io.confluent.ksql.rest.entity.LagInfoEntity) KsqlHostInfo(io.confluent.ksql.util.KsqlHostInfo) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) LagReportingMessage(io.confluent.ksql.rest.entity.LagReportingMessage) URI(java.net.URI) PersistentQueryMetadata(io.confluent.ksql.util.PersistentQueryMetadata) Before(org.junit.Before) ImmutableMap(com.google.common.collect.ImmutableMap) KsqlEngine(io.confluent.ksql.engine.KsqlEngine) Builder(io.confluent.ksql.rest.server.LagReportingAgent.Builder) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) SimpleKsqlClient(io.confluent.ksql.services.SimpleKsqlClient) Mockito.verify(org.mockito.Mockito.verify) SendLagService(io.confluent.ksql.rest.server.LagReportingAgent.SendLagService) LagInfo(org.apache.kafka.streams.LagInfo) Assert.assertFalse(org.junit.Assert.assertFalse) HostStatus(io.confluent.ksql.util.HostStatus) Clock(java.time.Clock) Optional(java.util.Optional) QueryStateStoreId(io.confluent.ksql.rest.entity.QueryStateStoreId) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) Assert.assertEquals(org.junit.Assert.assertEquals) Mockito.eq(org.mockito.Mockito.eq) HostStoreLags(io.confluent.ksql.rest.entity.HostStoreLags) LagInfoEntity(io.confluent.ksql.rest.entity.LagInfoEntity) KsqlHostInfoEntity(io.confluent.ksql.rest.entity.KsqlHostInfoEntity) Test(org.junit.Test)

Aggregations

KsqlHostInfoEntity (io.confluent.ksql.rest.entity.KsqlHostInfoEntity)23 Test (org.junit.Test)17 StreamedRow (io.confluent.ksql.rest.entity.StreamedRow)9 PersistentQueryMetadata (io.confluent.ksql.util.PersistentQueryMetadata)7 IntegrationTest (io.confluent.common.utils.IntegrationTest)6 KsqlEngine (io.confluent.ksql.engine.KsqlEngine)6 QueryDescriptionList (io.confluent.ksql.rest.entity.QueryDescriptionList)6 KsqlQueryStatus (io.confluent.ksql.util.KsqlConstants.KsqlQueryStatus)6 ImmutableMap (com.google.common.collect.ImmutableMap)5 QueryDescription (io.confluent.ksql.rest.entity.QueryDescription)5 ConfiguredStatement (io.confluent.ksql.statement.ConfiguredStatement)5 Map (java.util.Map)5 Optional (java.util.Optional)5 ListQueries (io.confluent.ksql.parser.tree.ListQueries)4 ClusterStatusResponse (io.confluent.ksql.rest.entity.ClusterStatusResponse)4 HeartbeatResponse (io.confluent.ksql.rest.entity.HeartbeatResponse)4 KsqlEntity (io.confluent.ksql.rest.entity.KsqlEntity)4 List (java.util.List)4 Collectors (java.util.stream.Collectors)4 ImmutableList (com.google.common.collect.ImmutableList)3