Examples with StartpointSpecific org.apache.samza.startpoint.StartpointSpecific used on opensource projects

Example 1 with StartpointSpecific

use of org.apache.samza.startpoint.StartpointSpecific in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointSpecificToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointSpecificToCorrectOffset() {
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    Assert.assertEquals("100", resolver.visit(systemStreamPartition, new StartpointSpecific("100")));
}

Example 2 with StartpointSpecific

use of org.apache.samza.startpoint.StartpointSpecific in project samza by apache.

the class TestStartpoint method testStartpointSpecific.

@Test
public void testStartpointSpecific() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents1 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic1, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointSpecific = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointSpecific startpointSpecific = new StartpointSpecific(String.valueOf(sentEvents1.get(100).offset()));
    writeStartpoints(startpointManager, inputKafkaTopic1, ZK_TEST_PARTITION_COUNT, startpointSpecific);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic1.equals(streamName)) {
                recvEventsInputStartpointSpecific.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointSpecific = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointSpecific = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointSpecific = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic1, outputKafkaTopic, processedMessagesLatchStartpointSpecific, shutdownLatchStartpointSpecific, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointSpecific, applicationConfig1);
    executeRun(appRunner, applicationConfig1);
    assertTrue(processedMessagesLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    Integer startpointSpecificOffset = Integer.valueOf(startpointSpecific.getSpecificOffset());
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointSpecific.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        String assertMsg = String.format("Expecting message offset: %d >= Startpoint specific offset: %d", eventOffset, startpointSpecificOffset);
        assertTrue(assertMsg, eventOffset >= startpointSpecificOffset);
    }
}

Example 3 with StartpointSpecific

use of org.apache.samza.startpoint.StartpointSpecific in project samza by apache.

the class TestKafkaSystemAdminJava method testStartpointSpecificOffsetVisitorShouldResolveToCorrectOffset.

@Test
public void testStartpointSpecificOffsetVisitorShouldResolveToCorrectOffset() {
    final KafkaConsumer consumer = Mockito.mock(KafkaConsumer.class);
    final KafkaStartpointToOffsetResolver kafkaStartpointToOffsetResolver = new KafkaStartpointToOffsetResolver(consumer);
    final StartpointSpecific testStartpointSpecific = new StartpointSpecific(TEST_OFFSET);
    // Invoke the consumer with startpoint.
    String resolvedOffset = kafkaStartpointToOffsetResolver.visit(TEST_SYSTEM_STREAM_PARTITION, testStartpointSpecific);
    Assert.assertEquals(TEST_OFFSET, resolvedOffset);
}