Search in sources :

Example 11 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamOffsetManagement.

// TODO: Add assertions.
//@Test
public void testEndOfStreamOffsetManagement() throws Exception {
    //explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
    TestTask mockStreamTask1 = new TestTask(true, false, false, null);
    TestTask mockStreamTask2 = new TestTask(true, false, false, null);
    Partition p1 = new Partition(1);
    Partition p2 = new Partition(2);
    SystemStreamPartition ssp1 = new SystemStreamPartition("system1", "stream1", p1);
    SystemStreamPartition ssp2 = new SystemStreamPartition("system1", "stream2", p2);
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp2, "1", "key1", "message1");
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp2, "2", "key1", "message1");
    IncomingMessageEnvelope envelope3 = IncomingMessageEnvelope.buildEndOfStreamEnvelope(ssp2);
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspMap = new HashMap<>();
    List<IncomingMessageEnvelope> messageList = new ArrayList<>();
    messageList.add(envelope1);
    messageList.add(envelope2);
    messageList.add(envelope3);
    sspMap.put(ssp2, messageList);
    SystemConsumer mockConsumer = mock(SystemConsumer.class);
    when(mockConsumer.poll(anyObject(), anyLong())).thenReturn(sspMap);
    HashMap<String, SystemConsumer> systemConsumerMap = new HashMap<>();
    systemConsumerMap.put("system1", mockConsumer);
    SystemConsumers consumers = TestSystemConsumers.getSystemConsumers(systemConsumerMap);
    TaskName taskName1 = new TaskName("task1");
    TaskName taskName2 = new TaskName("task2");
    Set<TaskName> taskNames = new HashSet<>();
    taskNames.add(taskName1);
    taskNames.add(taskName2);
    OffsetManager offsetManager = mock(OffsetManager.class);
    when(offsetManager.getLastProcessedOffset(taskName1, ssp1)).thenReturn(Option.apply("3"));
    when(offsetManager.getLastProcessedOffset(taskName2, ssp2)).thenReturn(Option.apply("0"));
    when(offsetManager.getStartingOffset(taskName1, ssp1)).thenReturn(Option.apply(IncomingMessageEnvelope.END_OF_STREAM_OFFSET));
    when(offsetManager.getStartingOffset(taskName2, ssp2)).thenReturn(Option.apply("1"));
    TaskInstance taskInstance1 = createTaskInstance(mockStreamTask1, taskName1, ssp1, offsetManager, consumers);
    TaskInstance taskInstance2 = createTaskInstance(mockStreamTask2, taskName2, ssp2, offsetManager, consumers);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName1, taskInstance1);
    tasks.put(taskName2, taskInstance2);
    taskInstance1.registerConsumers();
    taskInstance2.registerConsumers();
    consumers.start();
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumers, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    runLoop.run();
}
Also used : SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Partition(org.apache.samza.Partition) SystemConsumer(org.apache.samza.system.SystemConsumer) TaskInstance(org.apache.samza.container.TaskInstance) SystemConsumers(org.apache.samza.system.SystemConsumers) TestSystemConsumers(org.apache.samza.system.TestSystemConsumers) HashMap(java.util.HashMap) IncomingMessageEnvelope(org.apache.samza.system.IncomingMessageEnvelope) OffsetManager(org.apache.samza.checkpoint.OffsetManager) ArrayList(java.util.ArrayList) TaskName(org.apache.samza.container.TaskName) ArrayList(java.util.ArrayList) List(java.util.List) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) HashSet(java.util.HashSet)

Example 12 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testCommitAllTasks.

//@Test
public void testCommitAllTasks() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
    task0.setCommitRequest(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setCommitRequest(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    //have a null message in between to make sure task0 finishes processing and invoke the commit
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(null).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager).checkpoint(taskName1);
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 13 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method createTaskInstance.

TaskInstance createTaskInstance(AsyncStreamTask task, TaskName taskName, SystemStreamPartition ssp, OffsetManager manager, SystemConsumers consumers) {
    TaskInstanceMetrics taskInstanceMetrics = new TaskInstanceMetrics("task", new MetricsRegistryMap());
    scala.collection.immutable.Set<SystemStreamPartition> sspSet = JavaConverters.asScalaSetConverter(Collections.singleton(ssp)).asScala().toSet();
    return new TaskInstance(task, taskName, mock(Config.class), taskInstanceMetrics, null, consumers, mock(TaskInstanceCollector.class), mock(SamzaContainerContext.class), manager, null, null, sspSet, new TaskInstanceExceptionHandler(taskInstanceMetrics, new scala.collection.immutable.HashSet<String>()));
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) SamzaContainerContext(org.apache.samza.container.SamzaContainerContext) Config(org.apache.samza.config.Config) TaskInstanceMetrics(org.apache.samza.container.TaskInstanceMetrics) TaskInstanceExceptionHandler(org.apache.samza.container.TaskInstanceExceptionHandler) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) HashSet(java.util.HashSet)

Example 14 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamWithOutOfOrderProcess.

//@Test
public void testEndOfStreamWithOutOfOrderProcess() throws Exception {
    int maxMessagesInFlight = 2;
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight);
    TestTask task1 = new TestTask(true, true, false, task1ProcessedMessagesLatch, maxMessagesInFlight);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    task0.callbackHandler = buildOutofOrderCallback(task0);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(2, task0.processed);
    assertEquals(2, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(5L, containerMetrics.envelopes().getCount());
    assertEquals(3L, containerMetrics.processes().getCount());
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 15 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testProcessBehaviourWhenAsyncCommitIsEnabled.

//@Test
public void testProcessBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
    int maxMessagesInFlight = 2;
    TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
    TestTask task1 = new TestTask(true, false, true, null);
    CountDownLatch commitLatch = new CountDownLatch(1);
    task0.commitHandler = callback -> {
        TaskCallbackImpl taskCallback = (TaskCallbackImpl) callback;
        if (taskCallback.envelope.equals(envelope3)) {
            try {
                commitLatch.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    };
    task0.callbackHandler = callback -> {
        TaskCallbackImpl taskCallback = (TaskCallbackImpl) callback;
        if (taskCallback.envelope.equals(envelope0)) {
            assertEquals(2, containerMetrics.processes().getCount());
            assertEquals(0, containerMetrics.commits().getCount());
            commitLatch.countDown();
        }
    };
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    when(consumerMultiplexer.choose(false)).thenReturn(envelope3).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
    runLoop.run();
    commitLatch.await();
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Aggregations

TaskInstance (org.apache.samza.container.TaskInstance)15 HashMap (java.util.HashMap)14 TaskName (org.apache.samza.container.TaskName)13 CountDownLatch (java.util.concurrent.CountDownLatch)11 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)3 ArrayList (java.util.ArrayList)2 HashSet (java.util.HashSet)2 List (java.util.List)2 IncomingMessageEnvelope (org.apache.samza.system.IncomingMessageEnvelope)2 Partition (org.apache.samza.Partition)1 OffsetManager (org.apache.samza.checkpoint.OffsetManager)1 Config (org.apache.samza.config.Config)1 SamzaContainerContext (org.apache.samza.container.SamzaContainerContext)1 TaskInstanceExceptionHandler (org.apache.samza.container.TaskInstanceExceptionHandler)1 TaskInstanceMetrics (org.apache.samza.container.TaskInstanceMetrics)1 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)1 SystemConsumer (org.apache.samza.system.SystemConsumer)1 SystemConsumers (org.apache.samza.system.SystemConsumers)1 TestSystemConsumers (org.apache.samza.system.TestSystemConsumers)1