Examples with TaskInstance org.apache.samza.container.TaskInstance used on opensource projects

Example 6 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testProcessMultipleTasks.

//@Test
public void testProcessMultipleTasks() throws Exception {
    CountDownLatch task0ProcessedMessages = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessages = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessages);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessages);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessages.await();
    task1ProcessedMessages.await();
    assertEquals(1, task0.processed);
    assertEquals(1, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(2L, containerMetrics.envelopes().getCount());
    assertEquals(2L, containerMetrics.processes().getCount());
}

Example 7 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamWithMultipleTasks.

//@Test
public void testEndOfStreamWithMultipleTasks() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
    TestTask task1 = new TestTask(true, true, false, task1ProcessedMessagesLatch);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(1, task0.processed);
    assertEquals(1, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(4L, containerMetrics.envelopes().getCount());
    assertEquals(2L, containerMetrics.processes().getCount());
}

Example 8 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testWindow.

//@Test
public void testWindow() throws Exception {
    TestTask task0 = new TestTask(true, true, false, null);
    TestTask task1 = new TestTask(true, false, true, null);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    long windowMs = 1;
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(null);
    runLoop.run();
    assertEquals(4, task1.windowCount);
}

Example 9 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testShutdownOnConsensus.

//@Test
public void testShutdownOnConsensus() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, true, task0ProcessedMessagesLatch);
    task0.setShutdownRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setShutdownRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    // consensus is reached after envelope1 is processed.
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(1, task0.processed);
    assertEquals(1, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(2L, containerMetrics.envelopes().getCount());
    assertEquals(2L, containerMetrics.processes().getCount());
}

Example 10 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testProcessInOrder.

//@Test
public void testProcessInOrder() throws Exception {
    CountDownLatch task0ProcessedMessages = new CountDownLatch(2);
    CountDownLatch task1ProcessedMessages = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessages);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessages);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    // Wait till the tasks completes processing all the messages.
    task0ProcessedMessages.await();
    task1ProcessedMessages.await();
    assertEquals(2, task0.processed);
    assertEquals(2, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(3L, containerMetrics.envelopes().getCount());
    assertEquals(3L, containerMetrics.processes().getCount());
}