use of org.apache.samza.container.TaskInstance in project samza by apache.
the class AsyncRunLoop method getSspToAsyncTaskWorkerMap.
/**
* Returns mapping of the SystemStreamPartition to the AsyncTaskWorkers to efficiently route the envelopes
*/
private static Map<SystemStreamPartition, List<AsyncTaskWorker>> getSspToAsyncTaskWorkerMap(Map<TaskName, TaskInstance> taskInstances, Map<TaskName, AsyncTaskWorker> taskWorkers) {
Map<SystemStreamPartition, List<AsyncTaskWorker>> sspToWorkerMap = new HashMap<>();
for (TaskInstance task : taskInstances.values()) {
Set<SystemStreamPartition> ssps = JavaConverters.setAsJavaSetConverter(task.systemStreamPartitions()).asJava();
for (SystemStreamPartition ssp : ssps) {
sspToWorkerMap.putIfAbsent(ssp, new ArrayList<>());
sspToWorkerMap.get(ssp).add(taskWorkers.get(task.taskName()));
}
}
return sspToWorkerMap;
}
use of org.apache.samza.container.TaskInstance in project samza by apache.
the class TestAsyncRunLoop method testCommitBehaviourWhenAsyncCommitIsEnabled.
//@Test
public void testCommitBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
int maxMessagesInFlight = 3;
TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
TestTask task1 = new TestTask(true, false, true, null);
task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
IncomingMessageEnvelope firstMsg = new IncomingMessageEnvelope(ssp0, "0", "key0", "value0");
IncomingMessageEnvelope secondMsg = new IncomingMessageEnvelope(ssp0, "1", "key1", "value1");
IncomingMessageEnvelope thirdMsg = new IncomingMessageEnvelope(ssp0, "2", "key0", "value0");
final CountDownLatch firstMsgCompletionLatch = new CountDownLatch(1);
final CountDownLatch secondMsgCompletionLatch = new CountDownLatch(1);
task0.callbackHandler = callback -> {
IncomingMessageEnvelope envelope = ((TaskCallbackImpl) callback).envelope;
try {
if (envelope.equals(firstMsg)) {
firstMsgCompletionLatch.await();
} else if (envelope.equals(secondMsg)) {
firstMsgCompletionLatch.countDown();
secondMsgCompletionLatch.await();
} else if (envelope.equals(thirdMsg)) {
secondMsgCompletionLatch.countDown();
verify(offsetManager).update(taskName0, firstMsg.getSystemStreamPartition(), firstMsg.getOffset());
verify(offsetManager, atLeastOnce()).checkpoint(taskName0);
}
} catch (Exception e) {
e.printStackTrace();
}
};
Map<TaskName, TaskInstance> tasks = new HashMap<>();
tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
when(consumerMultiplexer.choose(false)).thenReturn(firstMsg).thenReturn(secondMsg).thenReturn(thirdMsg).thenReturn(envelope1).thenReturn(null);
AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
runLoop.run();
firstMsgCompletionLatch.await();
secondMsgCompletionLatch.await();
assertEquals(3, task0.processed);
assertEquals(3, task0.committed);
assertEquals(1, task1.processed);
assertEquals(0, task1.committed);
}
use of org.apache.samza.container.TaskInstance in project samza by apache.
the class TestAsyncRunLoop method testCommitSingleTask.
// TODO fix in SAMZA-1183
// @Test
public void testCommitSingleTask() throws Exception {
CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch);
task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
Map<TaskName, TaskInstance> tasks = new HashMap<>();
tasks.put(taskName0, t0);
tasks.put(taskName1, t1);
int maxMessagesInFlight = 1;
AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
//have a null message in between to make sure task0 finishes processing and invoke the commit
when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(null).thenReturn(envelope1).thenReturn(null);
runLoop.run();
task0ProcessedMessagesLatch.await();
task1ProcessedMessagesLatch.await();
verify(offsetManager).checkpoint(taskName0);
verify(offsetManager, never()).checkpoint(taskName1);
}
use of org.apache.samza.container.TaskInstance in project samza by apache.
the class TestAsyncRunLoop method testEndOfStreamCommitBehavior.
//@Test
public void testEndOfStreamCommitBehavior() throws Exception {
CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
//explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
TestTask task0 = new TestTask(true, false, false, task0ProcessedMessagesLatch);
TestTask task1 = new TestTask(true, false, false, task1ProcessedMessagesLatch);
TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
Map<TaskName, TaskInstance> tasks = new HashMap<>();
tasks.put(taskName0, t0);
tasks.put(taskName1, t1);
int maxMessagesInFlight = 1;
AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null);
runLoop.run();
task0ProcessedMessagesLatch.await();
task1ProcessedMessagesLatch.await();
verify(offsetManager).checkpoint(taskName0);
verify(offsetManager).checkpoint(taskName1);
}
use of org.apache.samza.container.TaskInstance in project samza by apache.
the class TestAsyncRunLoop method testProcessOutOfOrder.
//@Test
public void testProcessOutOfOrder() throws Exception {
int maxMessagesInFlight = 2;
CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2);
CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight);
TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch, maxMessagesInFlight);
TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
Map<TaskName, TaskInstance> tasks = new HashMap<>();
tasks.put(taskName0, t0);
tasks.put(taskName1, t1);
task0.callbackHandler = buildOutofOrderCallback(task0);
AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null);
runLoop.run();
task0ProcessedMessagesLatch.await();
task1ProcessedMessagesLatch.await();
assertEquals(2, task0.processed);
assertEquals(2, task0.completed.get());
assertEquals(1, task1.processed);
assertEquals(1, task1.completed.get());
assertEquals(3L, containerMetrics.envelopes().getCount());
assertEquals(3L, containerMetrics.processes().getCount());
}
Aggregations