Examples with StartpointOldest org.apache.samza.startpoint.StartpointOldest used on opensource projects

Example 1 with StartpointOldest

use of org.apache.samza.startpoint.StartpointOldest in project samza by apache.

the class TestStartpoint method testStartpointOldest.

@Test
public void testStartpointOldest() throws InterruptedException {
    publishKafkaEventsWithDelayPerEvent(inputKafkaTopic3, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[2], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointOldest = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointOldest startpointOldest = new StartpointOldest();
    writeStartpoints(startpointManager, inputKafkaTopic3, ZK_TEST_PARTITION_COUNT, startpointOldest);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic3.equals(streamName)) {
                recvEventsInputStartpointOldest.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Fetch all since consuming from oldest
    CountDownLatch processedMessagesLatchStartpointOldest = new CountDownLatch(NUM_KAFKA_EVENTS);
    CountDownLatch shutdownLatchStartpointOldest = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointOldest = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic3, outputKafkaTopic, processedMessagesLatchStartpointOldest, shutdownLatchStartpointOldest, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointOldest, applicationConfig3);
    executeRun(appRunner, applicationConfig3);
    assertTrue(processedMessagesLatchStartpointOldest.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointOldest.await(1, TimeUnit.MINUTES));
    assertEquals("Expecting to have processed all the events", NUM_KAFKA_EVENTS, recvEventsInputStartpointOldest.size());
}

Example 2 with StartpointOldest

use of org.apache.samza.startpoint.StartpointOldest in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointOldestToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointOldestToCorrectOffset() {
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, resolver.visit(systemStreamPartition, new StartpointOldest()));
}

Example 3 with StartpointOldest

use of org.apache.samza.startpoint.StartpointOldest in project samza by apache.

the class TestKafkaSystemAdminJava method testStartpointOldestVisitorShouldResolveToCorrectOffset.

@Test
public void testStartpointOldestVisitorShouldResolveToCorrectOffset() {
    // Define dummy variables for testing.
    final KafkaConsumer consumer = Mockito.mock(KafkaConsumer.class);
    final KafkaStartpointToOffsetResolver kafkaStartpointToOffsetResolver = new KafkaStartpointToOffsetResolver(consumer);
    final StartpointOldest testStartpointSpecific = new StartpointOldest();
    // Mock the consumer interactions.
    Mockito.when(consumer.beginningOffsets(ImmutableSet.of(TEST_TOPIC_PARTITION))).thenReturn(ImmutableMap.of(TEST_TOPIC_PARTITION, 10L));
    // Invoke the consumer with startpoint.
    String resolvedOffset = kafkaStartpointToOffsetResolver.visit(TEST_SYSTEM_STREAM_PARTITION, testStartpointSpecific);
    Assert.assertEquals(TEST_OFFSET, resolvedOffset);
}