Examples with Partition org.apache.samza.Partition used on opensource projects

Example 41 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TestMultiFileHdfsReader method testReachingMaxReconnect.

@Test(expected = SamzaException.class)
public void testReachingMaxReconnect() {
    int numMaxRetries = 3;
    SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
    MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0", numMaxRetries);
    // first read a few events, and then reconnect
    for (int i = 0; i < NUM_EVENTS / 2; i++) {
        multiReader.readNext();
    }
    for (int i = 0; i < numMaxRetries; i++) {
        IncomingMessageEnvelope envelope = multiReader.readNext();
        multiReader.reconnect();
        IncomingMessageEnvelope envelopeAfterReconnect = multiReader.readNext();
        Assert.assertEquals(envelope, envelopeAfterReconnect);
    }
    multiReader.readNext();
    multiReader.reconnect();
    Assert.fail();
}

Example 42 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TaskConfigJava method getBroadcastSystemStreamPartitions.

/**
   * Get the systemStreamPartitions of the broadcast stream. Specifying
   * one partition for one stream or a range of the partitions for one
   * stream is allowed.
   *
   * @return a Set of SystemStreamPartitions
   */
public Set<SystemStreamPartition> getBroadcastSystemStreamPartitions() {
    HashSet<SystemStreamPartition> systemStreamPartitionSet = new HashSet<SystemStreamPartition>();
    List<String> systemStreamPartitions = getList(BROADCAST_INPUT_STREAMS, Collections.<String>emptyList());
    for (String systemStreamPartition : systemStreamPartitions) {
        int hashPosition = systemStreamPartition.indexOf("#");
        if (hashPosition == -1) {
            throw new IllegalArgumentException("incorrect format in " + systemStreamPartition + ". Broadcast stream names should be in the form 'system.stream#partitionId' or 'system.stream#[partitionN-partitionM]'");
        } else {
            String systemStreamName = systemStreamPartition.substring(0, hashPosition);
            String partitionSegment = systemStreamPartition.substring(hashPosition + 1);
            SystemStream systemStream = Util.getSystemStreamFromNames(systemStreamName);
            if (Pattern.matches(BROADCAST_STREAM_PATTERN, partitionSegment)) {
                systemStreamPartitionSet.add(new SystemStreamPartition(systemStream, new Partition(Integer.valueOf(partitionSegment))));
            } else {
                if (Pattern.matches(BROADCAST_STREAM_RANGE_PATTERN, partitionSegment)) {
                    int partitionStart = Integer.valueOf(partitionSegment.substring(1, partitionSegment.lastIndexOf("-")));
                    int partitionEnd = Integer.valueOf(partitionSegment.substring(partitionSegment.lastIndexOf("-") + 1, partitionSegment.indexOf("]")));
                    if (partitionStart > partitionEnd) {
                        LOGGER.warn("The starting partition in stream " + systemStream.toString() + " is bigger than the ending Partition. No partition is added");
                    }
                    for (int i = partitionStart; i <= partitionEnd; i++) {
                        systemStreamPartitionSet.add(new SystemStreamPartition(systemStream, new Partition(i)));
                    }
                } else {
                    throw new IllegalArgumentException("incorrect format in " + systemStreamPartition + ". Broadcast stream names should be in the form 'system.stream#partitionId' or 'system.stream#[partitionN-partitionM]'");
                }
            }
        }
    }
    return systemStreamPartitionSet;
}