Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 16 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestGroupBySystemStreamPartition method testLocalStreamGroupedCorrectly.

@Test
public void testLocalStreamGroupedCorrectly() {
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    HashMap<String, String> configMap = new HashMap<String, String>();
    Config config = new MapConfig(configMap);
    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
    Map<TaskName, Set<SystemStreamPartition>> emptyResult = grouper.group(allSSPs);
    assertTrue(emptyResult.isEmpty());
    Collections.addAll(allSSPs, aa0, aa1, aa2, ac0);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partitionaa0 = new HashSet<SystemStreamPartition>();
    partitionaa0.add(aa0);
    expectedResult.put(new TaskName(aa0.toString()), partitionaa0);
    HashSet<SystemStreamPartition> partitionaa1 = new HashSet<SystemStreamPartition>();
    partitionaa1.add(aa1);
    expectedResult.put(new TaskName(aa1.toString()), partitionaa1);
    HashSet<SystemStreamPartition> partitionaa2 = new HashSet<SystemStreamPartition>();
    partitionaa2.add(aa2);
    expectedResult.put(new TaskName(aa2.toString()), partitionaa2);
    HashSet<SystemStreamPartition> partitionac0 = new HashSet<SystemStreamPartition>();
    partitionac0.add(ac0);
    expectedResult.put(new TaskName(ac0.toString()), partitionac0);
    assertEquals(expectedResult, result);
}

Example 17 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateCallbackWithCoordinatorRequests.

@Test
public void testUpdateCallbackWithCoordinatorRequests() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "topic", new Partition(0));
    // simulate out of order
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp, "2", null, null);
    ReadableCoordinator coordinator2 = new ReadableCoordinator(taskName);
    coordinator2.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskCallbackImpl callback2 = new TaskCallbackImpl(listener, taskName, envelope2, coordinator2, 2, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback2);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp, "1", null, null);
    ReadableCoordinator coordinator1 = new ReadableCoordinator(taskName);
    coordinator1.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator1, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp, "0", null, null);
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(2, callbacksToUpdate.size());
    //Check for envelope0
    TaskCallbackImpl taskCallback = callbacksToUpdate.get(0);
    assertTrue(taskCallback.matchSeqNum(0));
    assertEquals(ssp, taskCallback.envelope.getSystemStreamPartition());
    assertEquals("0", taskCallback.envelope.getOffset());
    //Check for envelope1
    taskCallback = callbacksToUpdate.get(1);
    assertTrue(taskCallback.matchSeqNum(1));
    assertEquals(ssp, taskCallback.envelope.getSystemStreamPartition());
    assertEquals("1", taskCallback.envelope.getOffset());
}

Example 18 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateShouldReturnAllCompletedCallbacksTillTheCommitRequestDefined.

@Test
public void testUpdateShouldReturnAllCompletedCallbacksTillTheCommitRequestDefined() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp1 = new SystemStreamPartition("kafka", "topic", new Partition(0));
    SystemStreamPartition ssp2 = new SystemStreamPartition("kafka", "topic", new Partition(0));
    // Callback for Envelope3 contains commit request.
    IncomingMessageEnvelope envelope3 = new IncomingMessageEnvelope(ssp2, "0", null, null);
    ReadableCoordinator coordinator3 = new ReadableCoordinator(taskName);
    coordinator3.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback3 = new TaskCallbackImpl(listener, taskName, envelope3, coordinator3, 3, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback3);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp1, "2", null, null);
    ReadableCoordinator coordinator2 = new ReadableCoordinator(taskName);
    coordinator2.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskCallbackImpl callback2 = new TaskCallbackImpl(listener, taskName, envelope2, coordinator2, 2, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback2);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp1, "1", null, null);
    ReadableCoordinator coordinator1 = new ReadableCoordinator(taskName);
    coordinator1.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator1, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertTrue(callbacksToUpdate.isEmpty());
    // Callback for Envelope0 contains commit request.
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp1, "0", null, null);
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    // Check for both Envelope1, Envelope2, Envelope3 in callbacks to commit.
    // Two callbacks belonging to different system partition and has commitRequest defined is returned.
    callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(2, callbacksToUpdate.size());
    TaskCallbackImpl callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(0));
    assertEquals(envelope0.getSystemStreamPartition(), callback.envelope.getSystemStreamPartition());
    assertEquals(envelope0.getOffset(), callback.envelope.getOffset());
    callback = callbacksToUpdate.get(1);
    assertTrue(callback.matchSeqNum(1));
    assertEquals(envelope1.getSystemStreamPartition(), callback.envelope.getSystemStreamPartition());
    assertEquals(envelope1.getOffset(), callback.envelope.getOffset());
}

Example 19 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateCallbackInOrder.

@Test
public void testUpdateCallbackInOrder() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "topic", new Partition(0));
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp, "0", null, null);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(1, callbacksToUpdate.size());
    TaskCallbackImpl callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(0));
    assertEquals(ssp, callback.envelope.getSystemStreamPartition());
    assertEquals("0", callback.envelope.getOffset());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp, "1", null, null);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertEquals(1, callbacksToUpdate.size());
    callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(1));
    assertEquals(ssp, callback.envelope.getSystemStreamPartition());
    assertEquals("1", callback.envelope.getOffset());
}

Example 20 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testProcessInOrder.

//@Test
public void testProcessInOrder() throws Exception {
    CountDownLatch task0ProcessedMessages = new CountDownLatch(2);
    CountDownLatch task1ProcessedMessages = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessages);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessages);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    // Wait till the tasks completes processing all the messages.
    task0ProcessedMessages.await();
    task1ProcessedMessages.await();
    assertEquals(2, task0.processed);
    assertEquals(2, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(3L, containerMetrics.envelopes().getCount());
    assertEquals(3L, containerMetrics.processes().getCount());
}