Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 81 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateCallbackWithCoordinatorRequests.

@Test
public void testUpdateCallbackWithCoordinatorRequests() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "topic", new Partition(0));
    // simulate out of order
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp, "2", null, null);
    ReadableCoordinator coordinator2 = new ReadableCoordinator(taskName);
    coordinator2.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskCallbackImpl callback2 = new TaskCallbackImpl(listener, taskName, envelope2, coordinator2, 2, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback2);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp, "1", null, null);
    ReadableCoordinator coordinator1 = new ReadableCoordinator(taskName);
    coordinator1.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator1, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp, "0", null, null);
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(2, callbacksToUpdate.size());
    // Check for envelope0
    TaskCallbackImpl taskCallback = callbacksToUpdate.get(0);
    assertTrue(taskCallback.matchSeqNum(0));
    assertEquals(ssp, taskCallback.getSystemStreamPartition());
    assertEquals("0", taskCallback.getOffset());
    // Check for envelope1
    taskCallback = callbacksToUpdate.get(1);
    assertTrue(taskCallback.matchSeqNum(1));
    assertEquals(ssp, taskCallback.getSystemStreamPartition());
    assertEquals("1", taskCallback.getOffset());
}

Example 82 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateCallbackInOrder.

@Test
public void testUpdateCallbackInOrder() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp = new SystemStreamPartition("kafka", "topic", new Partition(0));
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp, "0", null, null);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(1, callbacksToUpdate.size());
    TaskCallbackImpl callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(0));
    assertEquals(ssp, callback.getSystemStreamPartition());
    assertEquals("0", callback.getOffset());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp, "1", null, null);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertEquals(1, callbacksToUpdate.size());
    callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(1));
    assertEquals(ssp, callback.getSystemStreamPartition());
    assertEquals("1", callback.getOffset());
}

Example 83 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestTaskCallbackManager method testUpdateShouldReturnAllCompletedCallbacksTillTheCommitRequestDefined.

@Test
public void testUpdateShouldReturnAllCompletedCallbacksTillTheCommitRequestDefined() {
    TaskName taskName = new TaskName("Partition 0");
    SystemStreamPartition ssp1 = new SystemStreamPartition("kafka", "topic", new Partition(0));
    SystemStreamPartition ssp2 = new SystemStreamPartition("kafka", "topic", new Partition(0));
    // Callback for Envelope3 contains commit request.
    IncomingMessageEnvelope envelope3 = new IncomingMessageEnvelope(ssp2, "0", null, null);
    ReadableCoordinator coordinator3 = new ReadableCoordinator(taskName);
    coordinator3.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback3 = new TaskCallbackImpl(listener, taskName, envelope3, coordinator3, 3, 0);
    List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callback3);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp1, "2", null, null);
    ReadableCoordinator coordinator2 = new ReadableCoordinator(taskName);
    coordinator2.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
    TaskCallbackImpl callback2 = new TaskCallbackImpl(listener, taskName, envelope2, coordinator2, 2, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback2);
    assertTrue(callbacksToUpdate.isEmpty());
    IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp1, "1", null, null);
    ReadableCoordinator coordinator1 = new ReadableCoordinator(taskName);
    coordinator1.commit(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskCallbackImpl callback1 = new TaskCallbackImpl(listener, taskName, envelope1, coordinator1, 1, 0);
    callbacksToUpdate = callbackManager.updateCallback(callback1);
    assertTrue(callbacksToUpdate.isEmpty());
    // Callback for Envelope0 contains commit request.
    IncomingMessageEnvelope envelope0 = new IncomingMessageEnvelope(ssp1, "0", null, null);
    ReadableCoordinator coordinator = new ReadableCoordinator(taskName);
    TaskCallbackImpl callback0 = new TaskCallbackImpl(listener, taskName, envelope0, coordinator, 0, 0);
    // Check for both Envelope1, Envelope2, Envelope3 in callbacks to commit.
    // Two callbacks belonging to different system partition and has commitRequest defined is returned.
    callbacksToUpdate = callbackManager.updateCallback(callback0);
    assertEquals(2, callbacksToUpdate.size());
    TaskCallbackImpl callback = callbacksToUpdate.get(0);
    assertTrue(callback.matchSeqNum(0));
    assertEquals(envelope0.getSystemStreamPartition(), callback.getSystemStreamPartition());
    assertEquals(envelope0.getOffset(), callback.getOffset());
    callback = callbacksToUpdate.get(1);
    assertTrue(callback.matchSeqNum(1));
    assertEquals(envelope1.getSystemStreamPartition(), callback.getSystemStreamPartition());
    assertEquals(envelope1.getOffset(), callback.getOffset());
}

Example 84 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestDiagnosticsStreamMessage method getSampleContainerModels.

public static Map<String, ContainerModel> getSampleContainerModels() {
    Map<String, ContainerModel> containerModels = new HashMap<>();
    Map<TaskName, TaskModel> tasks = new HashMap<>();
    Set<SystemStreamPartition> sspsForTask1 = new HashSet<>();
    sspsForTask1.add(new SystemStreamPartition("kafka", "test-stream", new Partition(0)));
    tasks.put(new TaskName("Partition 0"), new TaskModel(new TaskName("Partition 0"), sspsForTask1, new Partition(0)));
    Set<SystemStreamPartition> sspsForTask2 = new HashSet<>();
    sspsForTask2.add(new SystemStreamPartition("kafka", "test-stream", new Partition(1)));
    tasks.put(new TaskName("Partition 1"), new TaskModel(new TaskName("Partition 1"), sspsForTask2, new Partition(1)));
    containerModels.put("0", new ContainerModel("0", tasks));
    return containerModels;
}

Example 85 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestJobModelCalculator method testPreviousChangelogPartitionsMaintained.

@Test
public void testPreviousChangelogPartitionsMaintained() {
    // existing changelog mapping has 2 tasks, but the job model ultimately will need 4 tasks
    // intentionally using an "out-of-order" changelog mapping to make sure it gets maintained
    Map<TaskName, Integer> changelogPartitionMapping = ImmutableMap.of(taskName(0), 1, taskName(1), 0);
    Config config = config(ImmutableList.of(SYSTEM_STREAM0, SYSTEM_STREAM1), ImmutableMap.of());
    // these task models have special changelog partitions from the previous mapping
    TaskModel taskModel0 = new TaskModel(taskName(0), ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(0)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(0))), new Partition(1));
    TaskModel taskModel1 = new TaskModel(taskName(1), ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(1)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(1))), new Partition(0));
    // tasks 2 and 3 will get assigned new changelog partitions
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel0, taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel1, taskName(3), taskModel(3, 3))));
    JobModel expected = new JobModel(config, containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}