Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 26 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestGroupByPartition method testLocalStreamsGroupedCorrectly.

@Test
public void testLocalStreamsGroupedCorrectly() {
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    GroupByPartition grouper = new GroupByPartition();
    Map<TaskName, Set<SystemStreamPartition>> emptyResult = grouper.group(allSSPs);
    // empty SSP set gets empty groups
    assertTrue(emptyResult.isEmpty());
    Collections.addAll(allSSPs, aa0, aa1, aa2, ab1, ab2, ac0);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition0 = new HashSet<SystemStreamPartition>();
    partition0.add(aa0);
    partition0.add(ac0);
    expectedResult.put(new TaskName("Partition 0"), partition0);
    HashSet<SystemStreamPartition> partition1 = new HashSet<SystemStreamPartition>();
    partition1.add(aa1);
    partition1.add(ab1);
    expectedResult.put(new TaskName("Partition 1"), partition1);
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    partition2.add(aa2);
    partition2.add(ab2);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}

Example 27 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testCommitAllTasks.

//@Test
public void testCommitAllTasks() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
    task0.setCommitRequest(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setCommitRequest(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    //have a null message in between to make sure task0 finishes processing and invoke the commit
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(null).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager).checkpoint(taskName1);
}

Example 28 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamWithOutOfOrderProcess.

//@Test
public void testEndOfStreamWithOutOfOrderProcess() throws Exception {
    int maxMessagesInFlight = 2;
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight);
    TestTask task1 = new TestTask(true, true, false, task1ProcessedMessagesLatch, maxMessagesInFlight);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    task0.callbackHandler = buildOutofOrderCallback(task0);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(2, task0.processed);
    assertEquals(2, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(5L, containerMetrics.envelopes().getCount());
    assertEquals(3L, containerMetrics.processes().getCount());
}

Example 29 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testProcessBehaviourWhenAsyncCommitIsEnabled.

//@Test
public void testProcessBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
    int maxMessagesInFlight = 2;
    TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
    TestTask task1 = new TestTask(true, false, true, null);
    CountDownLatch commitLatch = new CountDownLatch(1);
    task0.commitHandler = callback -> {
        TaskCallbackImpl taskCallback = (TaskCallbackImpl) callback;
        if (taskCallback.envelope.equals(envelope3)) {
            try {
                commitLatch.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    };
    task0.callbackHandler = callback -> {
        TaskCallbackImpl taskCallback = (TaskCallbackImpl) callback;
        if (taskCallback.envelope.equals(envelope0)) {
            assertEquals(2, containerMetrics.processes().getCount());
            assertEquals(0, containerMetrics.commits().getCount());
            commitLatch.countDown();
        }
    };
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    when(consumerMultiplexer.choose(false)).thenReturn(envelope3).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
    runLoop.run();
    commitLatch.await();
}

Example 30 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testCreateCallback.

@Test
public void testCreateCallback() {
    TaskCallbackImpl callback = callbackManager.createCallback(new TaskName("Partition 0"), null, null);
    assertTrue(callback.matchSeqNum(0));
    callback = callbackManager.createCallback(new TaskName("Partition 0"), null, null);
    assertTrue(callback.matchSeqNum(1));
}