Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 1 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestHostAwareContainerAllocator method getJobModelManager.

private static JobModelManager getJobModelManager(int containerCount) {
    //Ideally, the JobModelReader should be constructed independent of HttpServer.
    //That way it becomes easier to mock objects. Save it for later.
    HttpServer server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
    Map<String, ContainerModel> containers = new java.util.HashMap<>();
    for (int i = 0; i < containerCount; i++) {
        ContainerModel container = new ContainerModel(String.valueOf(i), i, new HashMap<TaskName, TaskModel>());
        containers.put(String.valueOf(i), container);
    }
    JobModel jobModel = new JobModel(getConfig(), containers);
    return new JobModelManager(jobModel, server, null);
}

Example 2 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class ChangelogPartitionManager method readChangeLogPartitionMapping.

/**
   * Read the taskName to partition mapping that is being maintained by this ChangelogManager
   * @return TaskName to change log partition mapping, or an empty map if there were no messages.
   */
public Map<TaskName, Integer> readChangeLogPartitionMapping() {
    log.debug("Reading changelog partition information");
    final HashMap<TaskName, Integer> changelogMapping = new HashMap<TaskName, Integer>();
    for (CoordinatorStreamMessage coordinatorStreamMessage : getBootstrappedStream(SetChangelogMapping.TYPE)) {
        SetChangelogMapping changelogMapEntry = new SetChangelogMapping(coordinatorStreamMessage);
        changelogMapping.put(new TaskName(changelogMapEntry.getTaskName()), changelogMapEntry.getPartition());
        log.debug("TaskName: {} is mapped to {}", changelogMapEntry.getTaskName(), changelogMapEntry.getPartition());
    }
    return changelogMapping;
}

Example 3 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testCommitBehaviourWhenAsyncCommitIsEnabled.

//@Test
public void testCommitBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
    int maxMessagesInFlight = 3;
    TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
    task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, null);
    task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    IncomingMessageEnvelope firstMsg = new IncomingMessageEnvelope(ssp0, "0", "key0", "value0");
    IncomingMessageEnvelope secondMsg = new IncomingMessageEnvelope(ssp0, "1", "key1", "value1");
    IncomingMessageEnvelope thirdMsg = new IncomingMessageEnvelope(ssp0, "2", "key0", "value0");
    final CountDownLatch firstMsgCompletionLatch = new CountDownLatch(1);
    final CountDownLatch secondMsgCompletionLatch = new CountDownLatch(1);
    task0.callbackHandler = callback -> {
        IncomingMessageEnvelope envelope = ((TaskCallbackImpl) callback).envelope;
        try {
            if (envelope.equals(firstMsg)) {
                firstMsgCompletionLatch.await();
            } else if (envelope.equals(secondMsg)) {
                firstMsgCompletionLatch.countDown();
                secondMsgCompletionLatch.await();
            } else if (envelope.equals(thirdMsg)) {
                secondMsgCompletionLatch.countDown();
                verify(offsetManager).update(taskName0, firstMsg.getSystemStreamPartition(), firstMsg.getOffset());
                verify(offsetManager, atLeastOnce()).checkpoint(taskName0);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    };
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    when(consumerMultiplexer.choose(false)).thenReturn(firstMsg).thenReturn(secondMsg).thenReturn(thirdMsg).thenReturn(envelope1).thenReturn(null);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
    runLoop.run();
    firstMsgCompletionLatch.await();
    secondMsgCompletionLatch.await();
    assertEquals(3, task0.processed);
    assertEquals(3, task0.committed);
    assertEquals(1, task1.processed);
    assertEquals(0, task1.committed);
}

Example 4 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testCommitSingleTask.

// TODO fix in SAMZA-1183
// @Test
public void testCommitSingleTask() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
    task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    //have a null message in between to make sure task0 finishes processing and invoke the commit
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(null).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager, never()).checkpoint(taskName1);
}

Example 5 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testEndOfStreamCommitBehavior.

//@Test
public void testEndOfStreamCommitBehavior() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    //explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
    TestTask task0 = new TestTask(true, false, false, task0ProcessedMessagesLatch);
    TestTask task1 = new TestTask(true, false, false, task1ProcessedMessagesLatch);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    verify(offsetManager).checkpoint(taskName0);
    verify(offsetManager).checkpoint(taskName1);
}