Examples with SystemStreamPartition org.apache.samza.system.SystemStreamPartition used on opensource projects

Example 1 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class AsyncRunLoop method getSspToAsyncTaskWorkerMap.

/**
   * Returns mapping of the SystemStreamPartition to the AsyncTaskWorkers to efficiently route the envelopes
   */
private static Map<SystemStreamPartition, List<AsyncTaskWorker>> getSspToAsyncTaskWorkerMap(Map<TaskName, TaskInstance> taskInstances, Map<TaskName, AsyncTaskWorker> taskWorkers) {
    Map<SystemStreamPartition, List<AsyncTaskWorker>> sspToWorkerMap = new HashMap<>();
    for (TaskInstance task : taskInstances.values()) {
        Set<SystemStreamPartition> ssps = JavaConverters.setAsJavaSetConverter(task.systemStreamPartitions()).asJava();
        for (SystemStreamPartition ssp : ssps) {
            sspToWorkerMap.putIfAbsent(ssp, new ArrayList<>());
            sspToWorkerMap.get(ssp).add(taskWorkers.get(task.taskName()));
        }
    }
    return sspToWorkerMap;
}

Example 2 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class GroupBySystemStreamPartition method group.

@Override
public Map<TaskName, Set<SystemStreamPartition>> group(Set<SystemStreamPartition> ssps) {
    Map<TaskName, Set<SystemStreamPartition>> groupedMap = new HashMap<TaskName, Set<SystemStreamPartition>>();
    for (SystemStreamPartition ssp : ssps) {
        if (broadcastStreams.contains(ssp)) {
            continue;
        }
        HashSet<SystemStreamPartition> sspSet = new HashSet<SystemStreamPartition>();
        sspSet.add(ssp);
        groupedMap.put(new TaskName(ssp.toString()), sspSet);
    }
    // assign the broadcast streams to all the taskNames
    if (!broadcastStreams.isEmpty()) {
        for (Set<SystemStreamPartition> value : groupedMap.values()) {
            for (SystemStreamPartition ssp : broadcastStreams) {
                value.add(ssp);
            }
        }
    }
    return groupedMap;
}

Example 3 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestJoinOperator method createStreamOperatorTask.

private StreamOperatorTask createStreamOperatorTask(Clock clock, StreamApplication app) throws Exception {
    ApplicationRunner runner = mock(ApplicationRunner.class);
    when(runner.getStreamSpec("instream")).thenReturn(new StreamSpec("instream", "instream", "insystem"));
    when(runner.getStreamSpec("instream2")).thenReturn(new StreamSpec("instream2", "instream2", "insystem2"));
    TaskContext taskContext = mock(TaskContext.class);
    when(taskContext.getSystemStreamPartitions()).thenReturn(ImmutableSet.of(new SystemStreamPartition("insystem", "instream", new Partition(0)), new SystemStreamPartition("insystem2", "instream2", new Partition(0))));
    when(taskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    Config config = mock(Config.class);
    StreamOperatorTask sot = new StreamOperatorTask(app, runner, clock);
    sot.init(config, taskContext);
    return sot;
}

Example 4 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestWindowOperator method setup.

@Before
public void setup() throws Exception {
    config = mock(Config.class);
    taskContext = mock(TaskContext.class);
    runner = mock(ApplicationRunner.class);
    when(taskContext.getSystemStreamPartitions()).thenReturn(ImmutableSet.of(new SystemStreamPartition("kafka", "integers", new Partition(0))));
    when(taskContext.getMetricsRegistry()).thenReturn(new MetricsRegistryMap());
    when(runner.getStreamSpec("integers")).thenReturn(new StreamSpec("integers", "integers", "kafka"));
}

Example 5 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestSamzaObjectMapper method setup.

@Before
public void setup() throws IOException {
    Map<String, String> configMap = new HashMap<String, String>();
    Set<SystemStreamPartition> ssp = new HashSet<>();
    configMap.put("a", "b");
    Config config = new MapConfig(configMap);
    TaskName taskName = new TaskName("test");
    ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
    TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
    Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
    tasks.put(taskName, taskModel);
    ContainerModel containerModel = new ContainerModel("1", 1, tasks);
    Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
    containerMap.put("1", containerModel);
    jobModel = new JobModel(config, containerMap);
}