Examples with TaskName org.apache.samza.container.TaskName used on opensource projects

Example 11 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testWindow.

//@Test
public void testWindow() throws Exception {
    TestTask task0 = new TestTask(true, true, false, null);
    TestTask task1 = new TestTask(true, false, true, null);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    long windowMs = 1;
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    when(consumerMultiplexer.choose(false)).thenReturn(null);
    runLoop.run();
    assertEquals(4, task1.windowCount);
}

Example 12 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestAsyncRunLoop method testShutdownOnConsensus.

//@Test
public void testShutdownOnConsensus() throws Exception {
    CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(1);
    CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1);
    TestTask task0 = new TestTask(true, true, true, task0ProcessedMessagesLatch);
    task0.setShutdownRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, task1ProcessedMessagesLatch);
    task1.setShutdownRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0);
    TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1);
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, t0);
    tasks.put(taskName1, t1);
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    int maxMessagesInFlight = 1;
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
    // consensus is reached after envelope1 is processed.
    when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope1).thenReturn(null);
    runLoop.run();
    task0ProcessedMessagesLatch.await();
    task1ProcessedMessagesLatch.await();
    assertEquals(1, task0.processed);
    assertEquals(1, task0.completed.get());
    assertEquals(1, task1.processed);
    assertEquals(1, task1.completed.get());
    assertEquals(2L, containerMetrics.envelopes().getCount());
    assertEquals(2L, containerMetrics.processes().getCount());
}

Example 13 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestHostAwareContainerAllocator method getJobModelManager.

private static JobModelManager getJobModelManager(int containerCount) {
    //Ideally, the JobModelReader should be constructed independent of HttpServer.
    //That way it becomes easier to mock objects. Save it for later.
    HttpServer server = new MockHttpServer("/", 7777, null, new ServletHolder(DefaultServlet.class));
    Map<String, ContainerModel> containers = new java.util.HashMap<>();
    for (int i = 0; i < containerCount; i++) {
        ContainerModel container = new ContainerModel(String.valueOf(i), i, new HashMap<TaskName, TaskModel>());
        containers.put(String.valueOf(i), container);
    }
    JobModel jobModel = new JobModel(getConfig(), containers);
    return new JobModelManager(jobModel, server, null);
}

Example 14 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestGroupByPartition method testNoTaskOnlyContainsBroadcastStreams.

@Test
public void testNoTaskOnlyContainsBroadcastStreams() {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put("task.broadcast.inputs", "SystemA.StreamA#0, SystemA.StreamB#1");
    Config config = new MapConfig(configMap);
    GroupByPartition grouper = new GroupByPartition(config);
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    Collections.addAll(allSSPs, aa0, ab1, ab2);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    // broadcast stream
    partition2.add(aa0);
    partition2.add(ab1);
    // broadcast stream
    partition2.add(ab2);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}

Example 15 with TaskName

use of org.apache.samza.container.TaskName in project samza by apache.

the class TestGroupBySystemStreamPartition method testBroadcastStreamGroupedCorrectly.

@Test
public void testBroadcastStreamGroupedCorrectly() {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put("task.broadcast.inputs", "SystemA.StreamA#0");
    Config config = new MapConfig(configMap);
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    Collections.addAll(allSSPs, aa0, aa1, aa2, ac0);
    GroupBySystemStreamPartitionFactory grouperFactory = new GroupBySystemStreamPartitionFactory();
    SystemStreamPartitionGrouper grouper = grouperFactory.getSystemStreamPartitionGrouper(config);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partitionaa1 = new HashSet<SystemStreamPartition>();
    partitionaa1.add(aa1);
    partitionaa1.add(aa0);
    expectedResult.put(new TaskName(aa1.toString()), partitionaa1);
    HashSet<SystemStreamPartition> partitionaa2 = new HashSet<SystemStreamPartition>();
    partitionaa2.add(aa2);
    partitionaa2.add(aa0);
    expectedResult.put(new TaskName(aa2.toString()), partitionaa2);
    HashSet<SystemStreamPartition> partitionac0 = new HashSet<SystemStreamPartition>();
    partitionac0.add(ac0);
    partitionac0.add(aa0);
    expectedResult.put(new TaskName(ac0.toString()), partitionac0);
    assertEquals(expectedResult, result);
}