Search in sources :

Example 1 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class AsyncRunLoop method getSspToAsyncTaskWorkerMap.

/**
   * Returns mapping of the SystemStreamPartition to the AsyncTaskWorkers to efficiently route the envelopes
   */
private static Map<SystemStreamPartition, List<AsyncTaskWorker>> getSspToAsyncTaskWorkerMap(Map<TaskName, TaskInstance> taskInstances, Map<TaskName, AsyncTaskWorker> taskWorkers) {
    Map<SystemStreamPartition, List<AsyncTaskWorker>> sspToWorkerMap = new HashMap<>();
    for (TaskInstance task : taskInstances.values()) {
        Set<SystemStreamPartition> ssps = JavaConverters.setAsJavaSetConverter(task.systemStreamPartitions()).asJava();
        for (SystemStreamPartition ssp : ssps) {
            sspToWorkerMap.putIfAbsent(ssp, new ArrayList<>());
            sspToWorkerMap.get(ssp).add(taskWorkers.get(task.taskName()));
        }
    }
    return sspToWorkerMap;
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) List(java.util.List) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition)

Example 2 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testCommitBehaviourWhenAsyncCommitIsEnabled.

//@Test
public void testCommitBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
    int maxMessagesInFlight = 3;
    TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
    task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, null);
    task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    IncomingMessageEnvelope firstMsg = new IncomingMessageEnvelope(ssp0, "0", "key0", "value0");
    IncomingMessageEnvelope secondMsg = new IncomingMessageEnvelope(ssp0, "1", "key1", "value1");
    IncomingMessageEnvelope thirdMsg = new IncomingMessageEnvelope(ssp0, "2", "key0", "value0");
    final CountDownLatch firstMsgCompletionLatch = new CountDownLatch(1);
    final CountDownLatch secondMsgCompletionLatch = new CountDownLatch(1);
    task0.callbackHandler = callback -> {
        IncomingMessageEnvelope envelope = ((TaskCallbackImpl) callback).envelope;
        try {
            if (envelope.equals(firstMsg)) {
                firstMsgCompletionLatch.await();
            } else if (envelope.equals(secondMsg)) {
                firstMsgCompletionLatch.countDown();
                secondMsgCompletionLatch.await();
            } else if (envelope.equals(thirdMsg)) {
                secondMsgCompletionLatch.countDown();
                verify(offsetManager).update(taskName0, firstMsg.getSystemStreamPartition(), firstMsg.getOffset());
                verify(offsetManager, atLeastOnce()).checkpoint(taskName0);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    };
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    when(consumerMultiplexer.choose(false)).thenReturn(firstMsg).thenReturn(secondMsg).thenReturn(thirdMsg).thenReturn(envelope1).thenReturn(null);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
    runLoop.run();
    firstMsgCompletionLatch.await();
    secondMsgCompletionLatch.await();
    assertEquals(3, task0.processed);
    assertEquals(3, task0.committed);
    assertEquals(1, task1.processed);
    assertEquals(0, task1.committed);
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) IncomingMessageEnvelope(org.apache.samza.system.IncomingMessageEnvelope) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 3 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testCommitSingleTask.

// TODO fix in SAMZA-1183
// @Test
public void testCommitSingleTask() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
    task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    //have a null message in between to make sure task0 finishes processing and invoke the commit
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(null).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager, never()).checkpoint(taskName1);
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 4 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamCommitBehavior.

//@Test
public void testEndOfStreamCommitBehavior() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    //explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
    TestTask task0 = new TestTask(true, false, false, task0ProcessedMessagesLatch);
    TestTask task1 = new TestTask(true, false, false, task1ProcessedMessagesLatch);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager).checkpoint(taskName1);
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 5 with TaskInstance

use of org.apache.samza.container.TaskInstance in project samza by apache.

the class TestAsyncRunLoop method testProcessOutOfOrder.

//@Test
public void testProcessOutOfOrder() throws Exception {
    int maxMessagesInFlight = 2;
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch, maxMessagesInFlight);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    task0.callbackHandler = buildOutofOrderCallback(task0);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(2, task0.processed);
    assertEquals(2, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(3L, containerMetrics.envelopes().getCount());
    assertEquals(3L, containerMetrics.processes().getCount());
}
Also used : TaskInstance(org.apache.samza.container.TaskInstance) HashMap(java.util.HashMap) TaskName(org.apache.samza.container.TaskName) CountDownLatch(java.util.concurrent.CountDownLatch)

Aggregations

TaskInstance (org.apache.samza.container.TaskInstance)17 HashMap (java.util.HashMap)15 TaskName (org.apache.samza.container.TaskName)15 CountDownLatch (java.util.concurrent.CountDownLatch)11 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)4 HashSet (java.util.HashSet)3 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Partition (org.apache.samza.Partition)2 TaskInstanceMetrics (org.apache.samza.container.TaskInstanceMetrics)2 IncomingMessageEnvelope (org.apache.samza.system.IncomingMessageEnvelope)2 Map (java.util.Map)1 OffsetManager (org.apache.samza.checkpoint.OffsetManager)1 Config (org.apache.samza.config.Config)1 SamzaContainer (org.apache.samza.container.SamzaContainer)1 SamzaContainerContext (org.apache.samza.container.SamzaContainerContext)1 TaskInstanceExceptionHandler (org.apache.samza.container.TaskInstanceExceptionHandler)1 TaskModel (org.apache.samza.job.model.TaskModel)1 Gauge (org.apache.samza.metrics.Gauge)1 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)1