Examples with StartpointUpcoming org.apache.samza.startpoint.StartpointUpcoming used on opensource projects

Example 1 with StartpointUpcoming

use of org.apache.samza.startpoint.StartpointUpcoming in project samza by apache.

the class TestKafkaSystemAdminJava method testStartpointUpcomingVisitorShouldResolveToCorrectOffset.

@Test
public void testStartpointUpcomingVisitorShouldResolveToCorrectOffset() {
    // Define dummy variables for testing.
    final KafkaConsumer consumer = Mockito.mock(KafkaConsumer.class);
    final KafkaStartpointToOffsetResolver kafkaStartpointToOffsetResolver = new KafkaStartpointToOffsetResolver(consumer);
    final StartpointUpcoming testStartpointSpecific = new StartpointUpcoming();
    // Mock the consumer interactions.
    Mockito.when(consumer.endOffsets(ImmutableSet.of(TEST_TOPIC_PARTITION))).thenReturn(ImmutableMap.of(TEST_TOPIC_PARTITION, 10L));
    // Invoke the consumer with startpoint.
    String resolvedOffset = kafkaStartpointToOffsetResolver.visit(TEST_SYSTEM_STREAM_PARTITION, testStartpointSpecific);
    Assert.assertEquals(TEST_OFFSET, resolvedOffset);
}

Example 2 with StartpointUpcoming

use of org.apache.samza.startpoint.StartpointUpcoming in project samza by apache.

the class TestStartpoint method testStartpointUpcoming.

@Test
public void testStartpointUpcoming() throws InterruptedException {
    publishKafkaEventsWithDelayPerEvent(inputKafkaTopic4, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[3], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointUpcoming = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointUpcoming startpointUpcoming = new StartpointUpcoming();
    writeStartpoints(startpointManager, inputKafkaTopic4, ZK_TEST_PARTITION_COUNT, startpointUpcoming);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic4.equals(streamName)) {
                recvEventsInputStartpointUpcoming.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Expecting none, so just attempt a small number of fetches.
    CountDownLatch processedMessagesLatchStartpointUpcoming = new CountDownLatch(5);
    CountDownLatch shutdownLatchStartpointUpcoming = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointUpcoming = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic4, outputKafkaTopic, processedMessagesLatchStartpointUpcoming, shutdownLatchStartpointUpcoming, Optional.of(processedCallback));
    // Startpoint upcoming
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointUpcoming, applicationConfig4);
    executeRun(appRunner, applicationConfig4);
    assertFalse("Expecting to timeout and not process any old messages.", processedMessagesLatchStartpointUpcoming.await(15, TimeUnit.SECONDS));
    assertEquals("Expecting not to process any old messages.", 0, recvEventsInputStartpointUpcoming.size());
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointUpcoming.await(1, TimeUnit.MINUTES));
}

Example 3 with StartpointUpcoming

use of org.apache.samza.startpoint.StartpointUpcoming in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointUpcomingToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointUpcomingToCorrectOffset() {
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    Assert.assertEquals(EventHubSystemConsumer.END_OF_STREAM, resolver.visit(systemStreamPartition, new StartpointUpcoming()));
}