Search in sources :

Example 6 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistryTest method setUp.

@Before
public void setUp() {
    when(ksqlTopic.getKafkaTopicName()).thenReturn(TOPIC);
    when(kafkaConsumerFactory.create(any(), any(), any(), any(), any(), any())).thenReturn(kafkaConsumer);
    catchupCoordinator = new TestCatchupCoordinator();
    latestConsumer = new TestLatestConsumer(TOPIC, false, SCHEMA, kafkaConsumer, catchupCoordinator, assignment -> {
    }, ksqlConfig, Clock.systemUTC());
    latestConsumer2 = new TestLatestConsumer(TOPIC, false, SCHEMA, kafkaConsumer, catchupCoordinator, assignment -> {
    }, ksqlConfig, Clock.systemUTC());
    catchupConsumer = new TestCatchupConsumer(TOPIC, false, SCHEMA, kafkaConsumer, () -> latestConsumer, catchupCoordinator, pushOffsetRange, Clock.systemUTC(), pq -> {
    });
    when(latestConsumerFactory.create(any(), anyBoolean(), any(), any(), any(), any(), any(), any())).thenReturn(latestConsumer, latestConsumer2);
    when(catchupConsumerFactory.create(any(), anyBoolean(), any(), any(), any(), any(), any(), any(), anyLong(), any())).thenReturn(catchupConsumer);
    when(ksqlTopic.getKeyFormat()).thenReturn(keyFormat);
    when(keyFormat.isWindowed()).thenReturn(false);
    realExecutorService = Executors.newFixedThreadPool(2);
    doAnswer(a -> {
        final Runnable runnable = a.getArgument(0);
        startLatestRunnable.set(runnable);
        realExecutorService.submit(runnable);
        return null;
    }).when(executorService).submit(any(Runnable.class));
    doAnswer(a -> {
        final Runnable runnable = a.getArgument(0);
        realExecutorService.submit(runnable);
        return null;
    }).when(catchupService).submit(any(Runnable.class));
    when(processingQueue.getQueryId()).thenReturn(new QueryId("q1"));
    when(processingQueue2.getQueryId()).thenReturn(new QueryId("q2"));
    registry = new ScalablePushRegistry(locator, SCHEMA, false, ImmutableMap.of(), ksqlTopic, serviceContext, ksqlConfig, SOURCE_APP_ID, kafkaConsumerFactory, latestConsumerFactory, catchupConsumerFactory, executorService, catchupService);
    when(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_CATCHUP_CONSUMERS)).thenReturn(10);
}
Also used : CatchupCoordinator(io.confluent.ksql.physical.scalablepush.consumer.CatchupCoordinator) ColumnName(io.confluent.ksql.name.ColumnName) AssertEventually.assertThatEventually(io.confluent.ksql.test.util.AssertEventually.assertThatEventually) ServiceContext(io.confluent.ksql.services.ServiceContext) ArgumentMatchers.contains(org.mockito.ArgumentMatchers.contains) PushLocator(io.confluent.ksql.physical.scalablepush.locator.PushLocator) Mockito.doThrow(org.mockito.Mockito.doThrow) CatchupConsumer(io.confluent.ksql.physical.scalablepush.consumer.CatchupConsumer) Mockito.doAnswer(org.mockito.Mockito.doAnswer) After(org.junit.After) QueryId(io.confluent.ksql.query.QueryId) Mockito.doReturn(org.mockito.Mockito.doReturn) TopicPartition(org.apache.kafka.common.TopicPartition) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) KsqlConfig(io.confluent.ksql.util.KsqlConfig) LogicalSchema(io.confluent.ksql.schema.ksql.LogicalSchema) Executors(java.util.concurrent.Executors) CatchupMetadata(io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata) Optional(java.util.Optional) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) KafkaConsumer(org.apache.kafka.clients.consumer.KafkaConsumer) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) StreamsConfig(org.apache.kafka.streams.StreamsConfig) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) KeyFormat(io.confluent.ksql.serde.KeyFormat) Mock(org.mockito.Mock) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) Supplier(java.util.function.Supplier) CatchupConsumerFactory(io.confluent.ksql.physical.scalablepush.consumer.CatchupConsumer.CatchupConsumerFactory) PushOffsetRange(io.confluent.ksql.util.PushOffsetRange) KafkaConsumerFactoryInterface(io.confluent.ksql.physical.scalablepush.consumer.KafkaConsumerFactory.KafkaConsumerFactoryInterface) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) LatestConsumer(io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer) LatestConsumerFactory(io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer.LatestConsumerFactory) Mockito.times(org.mockito.Mockito.times) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Mockito.never(org.mockito.Mockito.never) GenericRow(io.confluent.ksql.GenericRow) KsqlTopic(io.confluent.ksql.execution.ddl.commands.KsqlTopic) Clock(java.time.Clock) SqlTypes(io.confluent.ksql.schema.ksql.types.SqlTypes) Collections(java.util.Collections) QueryId(io.confluent.ksql.query.QueryId) Before(org.junit.Before)

Aggregations

LatestConsumer (io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer)6 GenericRow (io.confluent.ksql.GenericRow)2 CatchupConsumer (io.confluent.ksql.physical.scalablepush.consumer.CatchupConsumer)2 KsqlException (io.confluent.ksql.util.KsqlException)2 ImmutableMap (com.google.common.collect.ImmutableMap)1 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)1 KsqlTopic (io.confluent.ksql.execution.ddl.commands.KsqlTopic)1 ColumnName (io.confluent.ksql.name.ColumnName)1 CatchupMetadata (io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata)1 CatchupConsumerFactory (io.confluent.ksql.physical.scalablepush.consumer.CatchupConsumer.CatchupConsumerFactory)1 CatchupCoordinator (io.confluent.ksql.physical.scalablepush.consumer.CatchupCoordinator)1 KafkaConsumerFactoryInterface (io.confluent.ksql.physical.scalablepush.consumer.KafkaConsumerFactory.KafkaConsumerFactoryInterface)1 LatestConsumerFactory (io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer.LatestConsumerFactory)1 PushLocator (io.confluent.ksql.physical.scalablepush.locator.PushLocator)1 QueryId (io.confluent.ksql.query.QueryId)1 LogicalSchema (io.confluent.ksql.schema.ksql.LogicalSchema)1 SqlTypes (io.confluent.ksql.schema.ksql.types.SqlTypes)1 KeyFormat (io.confluent.ksql.serde.KeyFormat)1 ServiceContext (io.confluent.ksql.services.ServiceContext)1 AssertEventually.assertThatEventually (io.confluent.ksql.test.util.AssertEventually.assertThatEventually)1