Examples with OffsetSpecification com.rabbitmq.stream.OffsetSpecification used on opensource projects

Example 1 with OffsetSpecification

use of com.rabbitmq.stream.OffsetSpecification in project rabbitmq-stream-java-client by rabbitmq.

the class StreamConsumerTest method useSubscriptionListenerToRestartExactlyWhereDesired.

@Test
@DisabledIfRabbitMqCtlNotSet
void useSubscriptionListenerToRestartExactlyWhereDesired() throws Exception {
    AtomicInteger subscriptionListenerCallCount = new AtomicInteger(0);
    AtomicInteger receivedMessages = new AtomicInteger(0);
    AtomicLong offsetTracking = new AtomicLong(0);
    AtomicBoolean started = new AtomicBoolean(false);
    int storeEvery = 10_000;
    String reference = "ref-1";
    CountDownLatch poisonLatch = new CountDownLatch(1);
    environment.consumerBuilder().name(reference).stream(stream).offset(OffsetSpecification.first()).subscriptionListener(subscriptionContext -> {
        subscriptionListenerCallCount.getAndIncrement();
        OffsetSpecification offsetSpecification = started.get() ? OffsetSpecification.offset(offsetTracking.get() + 1) : subscriptionContext.offsetSpecification();
        subscriptionContext.offsetSpecification(offsetSpecification);
    }).messageHandler((context, message) -> {
        receivedMessages.incrementAndGet();
        offsetTracking.set(context.offset());
        started.set(true);
        if ("poison".equals(new String(message.getBodyAsBinary()))) {
            poisonLatch.countDown();
        }
    }).autoTrackingStrategy().flushInterval(// long flush interval
    Duration.ofMinutes(60)).messageCountBeforeStorage(storeEvery).builder().build();
    AtomicInteger publishedMessages = new AtomicInteger(0);
    Producer producer = environment.producerBuilder().stream(stream).build();
    IntConsumer publish = messagesToPublish -> {
        publishedMessages.addAndGet(messagesToPublish);
        IntStream.range(0, messagesToPublish).forEach(i -> producer.send(producer.messageBuilder().addData("".getBytes()).build(), confirmationStatus -> {
        }));
    };
    publish.accept(storeEvery * 2 - 100);
    waitAtMost(5, () -> receivedMessages.get() == publishedMessages.get());
    Host.killConnection("rabbitmq-stream-consumer-0");
    publish.accept(storeEvery * 2);
    producer.send(producer.messageBuilder().addData("poison".getBytes()).build(), confirmationStatus -> {
    });
    latchAssert(poisonLatch).completes();
    // no duplicates because the custom offset tracking overrides the stored offset in the
    // subscription listener
    assertThat(receivedMessages).hasValue(publishedMessages.get() + 1);
}

Example 2 with OffsetSpecification

use of com.rabbitmq.stream.OffsetSpecification in project rabbitmq-stream-java-client by rabbitmq.

the class StreamConsumerTest method externalOffsetTrackingWithSubscriptionListener.

@Test
@DisabledIfRabbitMqCtlNotSet
void externalOffsetTrackingWithSubscriptionListener() throws Exception {
    AtomicInteger subscriptionListenerCallCount = new AtomicInteger(0);
    AtomicInteger receivedMessages = new AtomicInteger(0);
    AtomicLong offsetTracking = new AtomicLong(0);
    AtomicBoolean started = new AtomicBoolean(false);
    environment.consumerBuilder().stream(stream).offset(OffsetSpecification.first()).subscriptionListener(subscriptionContext -> {
        subscriptionListenerCallCount.incrementAndGet();
        OffsetSpecification offsetSpecification = started.get() ? OffsetSpecification.offset(offsetTracking.get() + 1) : subscriptionContext.offsetSpecification();
        subscriptionContext.offsetSpecification(offsetSpecification);
    }).messageHandler((context, message) -> {
        receivedMessages.incrementAndGet();
        offsetTracking.set(context.offset());
        started.set(true);
    }).build();
    int messageCount = 10_000;
    Producer producer = environment.producerBuilder().stream(stream).build();
    Runnable publish = () -> IntStream.range(0, messageCount).forEach(i -> producer.send(producer.messageBuilder().addData("".getBytes()).build(), confirmationStatus -> {
    }));
    publish.run();
    waitAtMost(5, () -> receivedMessages.get() == messageCount);
    assertThat(offsetTracking.get()).isGreaterThanOrEqualTo(messageCount - 1);
    Host.killConnection("rabbitmq-stream-consumer-0");
    waitAtMost(recoveryInitialDelay.multipliedBy(2), () -> subscriptionListenerCallCount.get() == 2);
    publish.run();
    waitAtMost(5, () -> receivedMessages.get() == messageCount * 2);
    assertThat(offsetTracking.get()).isGreaterThanOrEqualTo(messageCount * 2 - 1);
}

Example 3 with OffsetSpecification

use of com.rabbitmq.stream.OffsetSpecification in project rabbitmq-stream-java-client by rabbitmq.

the class ConsumersCoordinatorTest method shouldRestartWhereItLeftOffAfterDisruption.

@ParameterizedTest
@MethodSource("disruptionArguments")
void shouldRestartWhereItLeftOffAfterDisruption(Consumer<ConsumersCoordinatorTest> configurator) throws Exception {
    scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
    when(environment.scheduledExecutorService()).thenReturn(scheduledExecutorService);
    Duration retryDelay = Duration.ofMillis(100);
    when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(environment.topologyUpdateBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(consumer.isOpen()).thenReturn(true);
    when(locator.metadata("stream")).thenReturn(metadata(null, replicas())).thenReturn(metadata(null, Collections.emptyList())).thenReturn(metadata(null, replicas()));
    ArgumentCaptor<OffsetSpecification> offsetSpecificationArgumentCaptor = ArgumentCaptor.forClass(OffsetSpecification.class);
    when(clientFactory.client(any())).thenReturn(client);
    when(client.subscribe(subscriptionIdCaptor.capture(), anyString(), offsetSpecificationArgumentCaptor.capture(), anyInt(), anyMap())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    Runnable closingRunnable = coordinator.subscribe(consumer, "stream", OffsetSpecification.first(), null, NO_OP_SUBSCRIPTION_LISTENER, (offset, message) -> {
    });
    verify(clientFactory, times(1)).client(any());
    verify(client, times(1)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(0).isEqualTo(OffsetSpecification.first());
    long lastReceivedOffset = 10;
    messageListener.handle(subscriptionIdCaptor.getValue(), lastReceivedOffset, 0, new WrapperMessageBuilder().build());
    configurator.accept(this);
    Thread.sleep(retryDelay.toMillis() * 5);
    verify(client, times(2)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(1).isEqualTo(OffsetSpecification.offset(lastReceivedOffset));
    when(client.unsubscribe(subscriptionIdCaptor.getValue())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    closingRunnable.run();
    verify(client, times(1)).unsubscribe(subscriptionIdCaptor.getValue());
}

Example 4 with OffsetSpecification

use of com.rabbitmq.stream.OffsetSpecification in project rabbitmq-stream-java-client by rabbitmq.

the class ConsumersCoordinatorTest method shouldUseStoredOffsetOnRecovery.

@ParameterizedTest
@MethodSource("disruptionArguments")
@SuppressWarnings("unchecked")
void shouldUseStoredOffsetOnRecovery(Consumer<ConsumersCoordinatorTest> configurator) throws Exception {
    scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
    when(environment.scheduledExecutorService()).thenReturn(scheduledExecutorService);
    Duration retryDelay = Duration.ofMillis(100);
    when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(environment.topologyUpdateBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(consumer.isOpen()).thenReturn(true);
    when(locator.metadata("stream")).thenReturn(metadata(null, replicas())).thenReturn(metadata(null, Collections.emptyList())).thenReturn(metadata(null, replicas()));
    when(clientFactory.client(any())).thenReturn(client);
    String consumerName = "consumer-name";
    long lastStoredOffset = 5;
    long lastReceivedOffset = 10;
    when(client.queryOffset(consumerName, "stream")).thenReturn(new QueryOffsetResponse(Constants.RESPONSE_CODE_OK, 0L)).thenReturn(new QueryOffsetResponse(Constants.RESPONSE_CODE_OK, lastStoredOffset));
    ArgumentCaptor<OffsetSpecification> offsetSpecificationArgumentCaptor = ArgumentCaptor.forClass(OffsetSpecification.class);
    ArgumentCaptor<Map<String, String>> subscriptionPropertiesArgumentCaptor = ArgumentCaptor.forClass(Map.class);
    when(client.subscribe(subscriptionIdCaptor.capture(), anyString(), offsetSpecificationArgumentCaptor.capture(), anyInt(), subscriptionPropertiesArgumentCaptor.capture())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    Runnable closingRunnable = coordinator.subscribe(consumer, "stream", null, consumerName, NO_OP_SUBSCRIPTION_LISTENER, (offset, message) -> {
    });
    verify(clientFactory, times(1)).client(any());
    verify(client, times(1)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(0).isEqualTo(OffsetSpecification.next());
    assertThat(subscriptionPropertiesArgumentCaptor.getAllValues()).element(0).isEqualTo(Collections.singletonMap("name", "consumer-name"));
    messageListener.handle(subscriptionIdCaptor.getValue(), lastReceivedOffset, 0, new WrapperMessageBuilder().build());
    configurator.accept(this);
    Thread.sleep(retryDelay.toMillis() * 5);
    verify(client, times(2)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(1).isEqualTo(OffsetSpecification.offset(lastStoredOffset + 1)).isNotEqualTo(OffsetSpecification.offset(lastReceivedOffset));
    assertThat(subscriptionPropertiesArgumentCaptor.getAllValues()).element(1).isEqualTo(Collections.singletonMap("name", "consumer-name"));
    when(client.unsubscribe(subscriptionIdCaptor.getValue())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    closingRunnable.run();
    verify(client, times(1)).unsubscribe(subscriptionIdCaptor.getValue());
}

Example 5 with OffsetSpecification

use of com.rabbitmq.stream.OffsetSpecification in project rabbitmq-stream-java-client by rabbitmq.

the class ConsumersCoordinatorTest method shouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesReceived.

@ParameterizedTest
@MethodSource("disruptionArguments")
void shouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesReceived(Consumer<ConsumersCoordinatorTest> configurator) throws Exception {
    scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
    when(environment.scheduledExecutorService()).thenReturn(scheduledExecutorService);
    Duration retryDelay = Duration.ofMillis(100);
    when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(environment.topologyUpdateBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay));
    when(consumer.isOpen()).thenReturn(true);
    when(locator.metadata("stream")).thenReturn(metadata(null, replicas())).thenReturn(metadata(null, Collections.emptyList())).thenReturn(metadata(null, replicas()));
    ArgumentCaptor<OffsetSpecification> offsetSpecificationArgumentCaptor = ArgumentCaptor.forClass(OffsetSpecification.class);
    when(clientFactory.client(any())).thenReturn(client);
    when(client.subscribe(subscriptionIdCaptor.capture(), anyString(), offsetSpecificationArgumentCaptor.capture(), anyInt(), anyMap())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    Runnable closingRunnable = coordinator.subscribe(consumer, "stream", OffsetSpecification.next(), null, NO_OP_SUBSCRIPTION_LISTENER, (offset, message) -> {
    });
    verify(clientFactory, times(1)).client(any());
    verify(client, times(1)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(0).isEqualTo(OffsetSpecification.next());
    configurator.accept(this);
    Thread.sleep(retryDelay.toMillis() * 5);
    verify(client, times(2)).subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap());
    assertThat(offsetSpecificationArgumentCaptor.getAllValues()).element(1).isEqualTo(OffsetSpecification.next());
    when(client.unsubscribe(subscriptionIdCaptor.getValue())).thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK));
    closingRunnable.run();
    verify(client, times(1)).unsubscribe(subscriptionIdCaptor.getValue());
}