Examples with SystemStreamPartition org.apache.samza.system.SystemStreamPartition used on opensource projects

Example 26 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestTaskConfigJava method testGetBroadcastSystemStreamPartitions.

@Test
public void testGetBroadcastSystemStreamPartitions() {
    HashMap<String, String> map = new HashMap<String, String>();
    map.put("task.broadcast.inputs", "kafka.foo#4, kafka.boo#5, kafka.z-o-o#[12-14], kafka.foo.bar#[3-4]");
    Config config = new MapConfig(map);
    TaskConfigJava taskConfig = new TaskConfigJava(config);
    Set<SystemStreamPartition> systemStreamPartitionSet = taskConfig.getBroadcastSystemStreamPartitions();
    HashSet<SystemStreamPartition> expected = new HashSet<SystemStreamPartition>();
    expected.add(new SystemStreamPartition("kafka", "foo", new Partition(4)));
    expected.add(new SystemStreamPartition("kafka", "boo", new Partition(5)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(12)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(13)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(14)));
    expected.add(new SystemStreamPartition("kafka", "foo.bar", new Partition(3)));
    expected.add(new SystemStreamPartition("kafka", "foo.bar", new Partition(4)));
    assertEquals(expected, systemStreamPartitionSet);
    map.put("task.broadcast.inputs", "kafka.foo");
    taskConfig = new TaskConfigJava(new MapConfig(map));
    boolean catchCorrectException = false;
    try {
        taskConfig.getBroadcastSystemStreamPartitions();
    } catch (IllegalArgumentException e) {
        catchCorrectException = true;
    }
    assertTrue(catchCorrectException);
    map.put("task.broadcast.inputs", "kafka.org.apache.events.WhitelistedIps#1-2");
    taskConfig = new TaskConfigJava(new MapConfig(map));
    boolean invalidFormatException = false;
    try {
        taskConfig.getBroadcastSystemStreamPartitions();
    } catch (IllegalArgumentException e) {
        invalidFormatException = true;
    }
    assertTrue(invalidFormatException);
}

Example 27 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestGroupByPartition method testBroadcastStreamsGroupedCorrectly.

@Test
public void testBroadcastStreamsGroupedCorrectly() {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put("task.broadcast.inputs", "SystemA.StreamA#0, SystemA.StreamB#1");
    Config config = new MapConfig(configMap);
    GroupByPartition grouper = new GroupByPartition(config);
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    Collections.addAll(allSSPs, aa0, aa1, aa2, ab1, ab2, ac0);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition0 = new HashSet<SystemStreamPartition>();
    // broadcast stream
    partition0.add(aa0);
    partition0.add(ac0);
    // broadcast stream
    partition0.add(ab1);
    expectedResult.put(new TaskName("Partition 0"), partition0);
    HashSet<SystemStreamPartition> partition1 = new HashSet<SystemStreamPartition>();
    partition1.add(aa1);
    // broadcast stream
    partition1.add(ab1);
    // broadcast stream
    partition1.add(aa0);
    expectedResult.put(new TaskName("Partition 1"), partition1);
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    partition2.add(aa2);
    partition2.add(ab2);
    // broadcast stream
    partition2.add(aa0);
    // broadcast stream
    partition2.add(ab1);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}

Example 28 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestGroupByPartition method testLocalStreamsGroupedCorrectly.

@Test
public void testLocalStreamsGroupedCorrectly() {
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    GroupByPartition grouper = new GroupByPartition();
    Map<TaskName, Set<SystemStreamPartition>> emptyResult = grouper.group(allSSPs);
    // empty SSP set gets empty groups
    assertTrue(emptyResult.isEmpty());
    Collections.addAll(allSSPs, aa0, aa1, aa2, ab1, ab2, ac0);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition0 = new HashSet<SystemStreamPartition>();
    partition0.add(aa0);
    partition0.add(ac0);
    expectedResult.put(new TaskName("Partition 0"), partition0);
    HashSet<SystemStreamPartition> partition1 = new HashSet<SystemStreamPartition>();
    partition1.add(aa1);
    partition1.add(ab1);
    expectedResult.put(new TaskName("Partition 1"), partition1);
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    partition2.add(aa2);
    partition2.add(ab2);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}

Example 29 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class BlockingEnvelopeMap method poll.

/**
   * {@inheritDoc}
   */
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException {
    long stopTime = clock.currentTimeMillis() + timeout;
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messagesToReturn = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>();
    metrics.incPoll();
    for (SystemStreamPartition systemStreamPartition : systemStreamPartitions) {
        BlockingQueue<IncomingMessageEnvelope> queue = bufferedMessages.get(systemStreamPartition);
        List<IncomingMessageEnvelope> outgoingList = new ArrayList<IncomingMessageEnvelope>(queue.size());
        if (queue.size() > 0) {
            queue.drainTo(outgoingList);
        } else if (timeout != 0) {
            IncomingMessageEnvelope envelope = null;
            // How long we can legally block (if timeout > 0)
            long timeRemaining = stopTime - clock.currentTimeMillis();
            if (timeout == SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES) {
                // the head of the stream.
                while (envelope == null && !isAtHead(systemStreamPartition)) {
                    metrics.incBlockingPoll(systemStreamPartition);
                    envelope = queue.poll(1000, TimeUnit.MILLISECONDS);
                }
            } else if (timeout > 0 && timeRemaining > 0) {
                // Block until we get at least one message.
                metrics.incBlockingTimeoutPoll(systemStreamPartition);
                envelope = queue.poll(timeRemaining, TimeUnit.MILLISECONDS);
            }
            // If we got a message, add it.
            if (envelope != null) {
                outgoingList.add(envelope);
                // Drain any remaining messages without blocking.
                queue.drainTo(outgoingList);
            }
        }
        if (outgoingList.size() > 0) {
            messagesToReturn.put(systemStreamPartition, outgoingList);
            subtractSizeOnQDrain(systemStreamPartition, outgoingList);
        }
    }
    return messagesToReturn;
}

Example 30 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestAsyncRunLoop method createTaskInstance.

TaskInstance createTaskInstance(AsyncStreamTask task, TaskName taskName, SystemStreamPartition ssp, OffsetManager manager, SystemConsumers consumers) {
    TaskInstanceMetrics taskInstanceMetrics = new TaskInstanceMetrics("task", new MetricsRegistryMap());
    scala.collection.immutable.Set<SystemStreamPartition> sspSet = JavaConverters.asScalaSetConverter(Collections.singleton(ssp)).asScala().toSet();
    return new TaskInstance(task, taskName, mock(Config.class), taskInstanceMetrics, null, consumers, mock(TaskInstanceCollector.class), mock(SamzaContainerContext.class), manager, null, null, sspSet, new TaskInstanceExceptionHandler(taskInstanceMetrics, new scala.collection.immutable.HashSet<String>()));
}