Examples with CatchupMetadata io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata used on opensource projects

Example 1 with CatchupMetadata

use of io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata in project ksql by confluentinc.

the class ScalablePushRegistryTest method shouldCatchExceptionCatchup_onRun.

@Test
public void shouldCatchExceptionCatchup_onRun() {
    // Given:
    catchupConsumer.setErrorOnRun(true);
    AtomicBoolean isErrorQueue = new AtomicBoolean(false);
    doAnswer(a -> {
        isErrorQueue.set(true);
        return null;
    }).when(processingQueue).onError();
    // When:
    registry.register(processingQueue, Optional.of(new CatchupMetadata(pushOffsetRange, CATCHUP_CONSUMER_GROUP)));
    // Then:
    assertThatEventually(isErrorQueue::get, is(true));
    assertThatEventually(registry::isLatestRunning, is(false));
    assertThatEventually(registry::latestNumRegistered, is(0));
    assertThatEventually(registry::catchupNumRegistered, is(0));
}

Example 2 with CatchupMetadata

use of io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata in project ksql by confluentinc.

the class ScalablePushRegistryTest method shouldCatchExceptionCatchup_onCreationFailure_catchupConsumer.

@Test
public void shouldCatchExceptionCatchup_onCreationFailure_catchupConsumer() {
    // Given:
    doThrow(new RuntimeException("Error!")).when(catchupConsumerFactory).create(any(), anyBoolean(), any(), any(), any(), any(), any(), any(), anyLong(), any());
    AtomicBoolean isErrorQueue = new AtomicBoolean(false);
    doAnswer(a -> {
        isErrorQueue.set(true);
        return null;
    }).when(processingQueue).onError();
    // When:
    final Exception e = assertThrows(RuntimeException.class, () -> registry.register(processingQueue, Optional.of(new CatchupMetadata(pushOffsetRange, CATCHUP_CONSUMER_GROUP))));
    // Then:
    assertThat(e.getMessage(), is("Error!"));
    assertThat(isErrorQueue.get(), is(true));
    assertThat(registry.isLatestRunning(), is(false));
    assertThat(registry.anyCatchupsRunning(), is(false));
}

Example 3 with CatchupMetadata

use of io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata in project ksql by confluentinc.

the class ScalablePushRegistryTest method shouldThrowErrorOnTooManyCatchups.

@Test
public void shouldThrowErrorOnTooManyCatchups() {
    // Given:
    when(ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_CATCHUP_CONSUMERS)).thenReturn(1);
    // When:
    registry.register(processingQueue, Optional.of(new CatchupMetadata(pushOffsetRange, CATCHUP_CONSUMER_GROUP)));
    assertThat(registry.isLatestRunning(), is(true));
    assertThatEventually(registry::latestNumRegistered, is(0));
    assertThatEventually(registry::catchupNumRegistered, is(1));
    final Exception e = assertThrows(RuntimeException.class, () -> registry.register(processingQueue2, Optional.of(new CatchupMetadata(pushOffsetRange, CATCHUP_CONSUMER_GROUP))));
    registry.unregister(processingQueue);
    // Then:
    assertThat(e.getMessage(), containsString("Too many catchups registered"));
    verify(processingQueue, never()).onError();
    verify(processingQueue2).onError();
    assertThat(registry.isLatestRunning(), is(false));
    assertThat(registry.latestNumRegistered(), is(0));
    assertThat(registry.catchupNumRegistered(), is(0));
}

Example 4 with CatchupMetadata

use of io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata in project ksql by confluentinc.

the class PushPhysicalPlanBuilder method translateDataSourceNode.

private AbstractPhysicalOperator translateDataSourceNode(final DataSourceNode logicalNode, final Optional<PushOffsetRange> offsetRange, final String catchupConsumerGroupId) {
    final ScalablePushRegistry scalablePushRegistry = persistentQueryMetadata.getScalablePushRegistry().orElseThrow(() -> new IllegalStateException("Scalable push registry cannot be found"));
    querySourceType = logicalNode.isWindowed() ? QuerySourceType.WINDOWED : QuerySourceType.NON_WINDOWED;
    final Optional<CatchupMetadata> catchupMetadata = offsetRange.map(or -> new CatchupMetadata(or, catchupConsumerGroupId));
    return new PeekStreamOperator(scalablePushRegistry, logicalNode, queryId, catchupMetadata);
}

Example 5 with CatchupMetadata

use of io.confluent.ksql.physical.scalablepush.ScalablePushRegistry.CatchupMetadata in project ksql by confluentinc.

the class ScalablePushRegistryTest method shouldCatchExceptionCatchup_onCreationFailure_kafkaConsumer.

@Test
public void shouldCatchExceptionCatchup_onCreationFailure_kafkaConsumer() {
    // Given:
    doReturn(kafkaConsumer).when(kafkaConsumerFactory).create(any(), any(), any(), any(), any(), contains("latest"));
    doThrow(new RuntimeException("Error!")).when(kafkaConsumerFactory).create(any(), any(), any(), any(), any(), contains("catchup"));
    AtomicBoolean isErrorQueue = new AtomicBoolean(false);
    doAnswer(a -> {
        isErrorQueue.set(true);
        return null;
    }).when(processingQueue).onError();
    // When:
    final Exception e = assertThrows(RuntimeException.class, () -> registry.register(processingQueue, Optional.of(new CatchupMetadata(pushOffsetRange, CATCHUP_CONSUMER_GROUP))));
    // Then:
    assertThat(e.getMessage(), is("Error!"));
    assertThat(isErrorQueue.get(), is(true));
    assertThat(registry.isLatestRunning(), is(false));
    assertThat(registry.anyCatchupsRunning(), is(false));
}