Examples with StartpointTimestamp org.apache.samza.startpoint.StartpointTimestamp used on opensource projects

Example 1 with StartpointTimestamp

use of org.apache.samza.startpoint.StartpointTimestamp in project samza by apache.

the class TestEventHubSystemAdmin method testStartpointResolverShouldResolveTheStartpointTimestampToCorrectOffset.

@Test
public void testStartpointResolverShouldResolveTheStartpointTimestampToCorrectOffset() throws EventHubException {
    // Initialize variables required for testing.
    EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
    EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
    String mockedOffsetToReturn = "100";
    // Setup the mock variables.
    EventHubClientManager mockEventHubClientManager = Mockito.mock(EventHubClientManager.class);
    EventHubClient mockEventHubClient = Mockito.mock(EventHubClient.class);
    PartitionReceiver mockPartitionReceiver = Mockito.mock(PartitionReceiver.class);
    EventData mockEventData = Mockito.mock(EventData.class);
    EventData.SystemProperties mockSystemProperties = Mockito.mock(EventData.SystemProperties.class);
    // Configure the mock variables to return the appropriate values.
    Mockito.when(mockEventHubSystemAdmin.getOrCreateStreamEventHubClient("test-stream")).thenReturn(mockEventHubClientManager);
    Mockito.when(mockEventHubClientManager.getEventHubClient()).thenReturn(mockEventHubClient);
    Mockito.when(mockEventHubClient.createReceiverSync(Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(mockPartitionReceiver);
    Mockito.when(mockPartitionReceiver.receiveSync(1)).thenReturn(Arrays.asList(mockEventData));
    Mockito.when(mockEventData.getSystemProperties()).thenReturn(mockSystemProperties);
    Mockito.when(mockSystemProperties.getOffset()).thenReturn(mockedOffsetToReturn);
    // Test the Offset resolver.
    EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
    String resolvedOffset = resolver.visit(systemStreamPartition, new StartpointTimestamp(100L));
    Assert.assertEquals(mockedOffsetToReturn, resolvedOffset);
}

Example 2 with StartpointTimestamp

use of org.apache.samza.startpoint.StartpointTimestamp in project samza by apache.

the class TestStartpoint method testStartpointTimestamp.

@Test
public void testStartpointTimestamp() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents2 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic2, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[1], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointTimestamp = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointTimestamp startpointTimestamp = new StartpointTimestamp(sentEvents2.get(150).timestamp());
    writeStartpoints(startpointManager, inputKafkaTopic2, ZK_TEST_PARTITION_COUNT, startpointTimestamp);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic2.equals(streamName)) {
                recvEventsInputStartpointTimestamp.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointTimestamp = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointTimestamp = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointTimestamp = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic2, outputKafkaTopic, processedMessagesLatchStartpointTimestamp, shutdownLatchStartpointTimestamp, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointTimestamp, applicationConfig2);
    executeRun(appRunner, applicationConfig2);
    assertTrue(processedMessagesLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointTimestamp.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        // sanity check
        assertNotEquals(0, ime.getEventTime());
        String assertMsg = String.format("Expecting message timestamp: %d >= Startpoint timestamp: %d", ime.getEventTime(), startpointTimestamp.getTimestampOffset());
        assertTrue(assertMsg, ime.getEventTime() >= startpointTimestamp.getTimestampOffset());
    }
}

Example 3 with StartpointTimestamp

use of org.apache.samza.startpoint.StartpointTimestamp in project samza by apache.

the class TestKafkaSystemAdminJava method testStartpointTimestampVisitorShouldResolveToCorrectOffsetWhenTimestampDoesNotExist.

@Test
public void testStartpointTimestampVisitorShouldResolveToCorrectOffsetWhenTimestampDoesNotExist() {
    final KafkaConsumer consumer = Mockito.mock(KafkaConsumer.class);
    final KafkaStartpointToOffsetResolver kafkaStartpointToOffsetResolver = new KafkaStartpointToOffsetResolver(consumer);
    final StartpointTimestamp startpointTimestamp = new StartpointTimestamp(0L);
    final Map<TopicPartition, OffsetAndTimestamp> offsetForTimesResult = new HashMap<>();
    offsetForTimesResult.put(TEST_TOPIC_PARTITION, null);
    // Mock the consumer interactions.
    Mockito.when(consumer.offsetsForTimes(ImmutableMap.of(TEST_TOPIC_PARTITION, 0L))).thenReturn(offsetForTimesResult);
    Mockito.when(consumer.endOffsets(ImmutableSet.of(TEST_TOPIC_PARTITION))).thenReturn(ImmutableMap.of(TEST_TOPIC_PARTITION, 10L));
    String resolvedOffset = kafkaStartpointToOffsetResolver.visit(TEST_SYSTEM_STREAM_PARTITION, startpointTimestamp);
    Assert.assertEquals(TEST_OFFSET, resolvedOffset);
    // Mock verifications.
    Mockito.verify(consumer).offsetsForTimes(ImmutableMap.of(TEST_TOPIC_PARTITION, 0L));
}

Example 4 with StartpointTimestamp

use of org.apache.samza.startpoint.StartpointTimestamp in project samza by apache.

the class TestKafkaSystemAdminJava method testStartpointTimestampVisitorShouldResolveToCorrectOffset.

@Test
public void testStartpointTimestampVisitorShouldResolveToCorrectOffset() {
    // Define dummy variables for testing.
    final Long testTimeStamp = 10L;
    final KafkaConsumer consumer = Mockito.mock(KafkaConsumer.class);
    final KafkaStartpointToOffsetResolver kafkaStartpointToOffsetResolver = new KafkaStartpointToOffsetResolver(consumer);
    final StartpointTimestamp startpointTimestamp = new StartpointTimestamp(testTimeStamp);
    final Map<TopicPartition, OffsetAndTimestamp> offsetForTimesResult = ImmutableMap.of(TEST_TOPIC_PARTITION, new OffsetAndTimestamp(Long.valueOf(TEST_OFFSET), testTimeStamp));
    // Mock the consumer interactions.
    Mockito.when(consumer.offsetsForTimes(ImmutableMap.of(TEST_TOPIC_PARTITION, testTimeStamp))).thenReturn(offsetForTimesResult);
    Mockito.when(consumer.position(TEST_TOPIC_PARTITION)).thenReturn(Long.valueOf(TEST_OFFSET));
    String resolvedOffset = kafkaStartpointToOffsetResolver.visit(TEST_SYSTEM_STREAM_PARTITION, startpointTimestamp);
    Assert.assertEquals(TEST_OFFSET, resolvedOffset);
}