Examples with SystemStreamPartition org.apache.samza.system.SystemStreamPartition used on opensource projects

Example 6 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestSinglePartitionWithoutOffsetsSystemAdmin method testShouldGetASinglePartition.

@Test
public void testShouldGetASinglePartition() {
    SinglePartitionWithoutOffsetsSystemAdmin admin = new SinglePartitionWithoutOffsetsSystemAdmin();
    Set<String> streamNames = new HashSet<String>();
    streamNames.add("a");
    streamNames.add("b");
    Map<String, SystemStreamMetadata> metadata = admin.getSystemStreamMetadata(streamNames);
    assertEquals(2, metadata.size());
    SystemStreamMetadata metadata1 = metadata.get("a");
    SystemStreamMetadata metadata2 = metadata.get("b");
    assertEquals(1, metadata1.getSystemStreamPartitionMetadata().size());
    assertEquals(1, metadata2.getSystemStreamPartitionMetadata().size());
    assertNull(metadata.get(new SystemStreamPartition("test-system", "c", new Partition(0))));
}

Example 7 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestExecutionPlanner method createSystemAdmin.

static SystemAdmin createSystemAdmin(Map<String, Integer> streamToPartitions) {
    return new SystemAdmin() {

        @Override
        public Map<SystemStreamPartition, String> getOffsetsAfter(Map<SystemStreamPartition, String> offsets) {
            return null;
        }

        @Override
        public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
            Map<String, SystemStreamMetadata> map = new HashMap<>();
            for (String stream : streamNames) {
                Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> m = new HashMap<>();
                for (int i = 0; i < streamToPartitions.get(stream); i++) {
                    m.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", ""));
                }
                map.put(stream, new SystemStreamMetadata(stream, m));
            }
            return map;
        }

        @Override
        public void createChangelogStream(String streamName, int numOfPartitions) {
        }

        @Override
        public void validateChangelogStream(String streamName, int numOfPartitions) {
        }

        @Override
        public void createCoordinatorStream(String streamName) {
        }

        @Override
        public Integer offsetComparator(String offset1, String offset2) {
            return null;
        }
    };
}

Example 8 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testOrderKeyRewrite.

/**
   * Verify that if a particular key-value is written, then another, then the original again,
   * that the original occurs last in the set.
   */
@Test
public void testOrderKeyRewrite() throws InterruptedException {
    final SystemStream systemStream = new SystemStream("system", "stream");
    final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0));
    final SystemConsumer systemConsumer = mock(SystemConsumer.class);
    final List<IncomingMessageEnvelope> list = new ArrayList<>();
    SetConfig setConfig1 = new SetConfig("source", "key1", "value1");
    SetConfig setConfig2 = new SetConfig("source", "key1", "value2");
    SetConfig setConfig3 = new SetConfig("source", "key1", "value1");
    list.add(createIncomingMessageEnvelope(setConfig1, ssp));
    list.add(createIncomingMessageEnvelope(setConfig2, ssp));
    list.add(createIncomingMessageEnvelope(setConfig3, ssp));
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() {

        {
            put(ssp, list);
        }
    };
    when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap());
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    consumer.bootstrap();
    Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream();
    // First message should have been removed as a duplicate
    assertEquals(2, bootstrappedMessages.size());
    CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]);
    assertEquals(setConfig2, coordinatorStreamMessages[0]);
    //Config 3 MUST be the last message, not config 2
    assertEquals(setConfig3, coordinatorStreamMessages[1]);
}

Example 9 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class MockCoordinatorStreamSystemFactory method getConsumer.

/**
   * Returns a consumer that sends all configs to the coordinator stream.
   *
   * @param config Along with the configs, you can pass checkpoints and changelog stream messages into the stream.
   *               The expected pattern is cp:source:taskname -> ssp,offset for checkpoint (Use sspToString util)
   *               ch:source:taskname -> changelogPartition for changelog
   *               Everything else is processed as normal config
   */
public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) {
    if (useCachedConsumer && mockConsumer != null) {
        return mockConsumer;
    }
    String jobName = config.get("job.name");
    String jobId = config.get("job.id");
    if (jobName == null) {
        throw new ConfigException("Must define job.name.");
    }
    if (jobId == null) {
        jobId = "1";
    }
    String streamName = Util.getCoordinatorStreamName(jobName, jobId);
    SystemStreamPartition systemStreamPartition = new SystemStreamPartition(systemName, streamName, new Partition(0));
    mockConsumer = new MockCoordinatorStreamWrappedConsumer(systemStreamPartition, config);
    return mockConsumer;
}

Example 10 with SystemStreamPartition

use of org.apache.samza.system.SystemStreamPartition in project samza by apache.

the class TestGroupByPartition method testNoTaskOnlyContainsBroadcastStreams.

@Test
public void testNoTaskOnlyContainsBroadcastStreams() {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put("task.broadcast.inputs", "SystemA.StreamA#0, SystemA.StreamB#1");
    Config config = new MapConfig(configMap);
    GroupByPartition grouper = new GroupByPartition(config);
    HashSet<SystemStreamPartition> allSSPs = new HashSet<SystemStreamPartition>();
    Collections.addAll(allSSPs, aa0, ab1, ab2);
    Map<TaskName, Set<SystemStreamPartition>> result = grouper.group(allSSPs);
    Map<TaskName, Set<SystemStreamPartition>> expectedResult = new HashMap<TaskName, Set<SystemStreamPartition>>();
    HashSet<SystemStreamPartition> partition2 = new HashSet<SystemStreamPartition>();
    // broadcast stream
    partition2.add(aa0);
    partition2.add(ab1);
    // broadcast stream
    partition2.add(ab2);
    expectedResult.put(new TaskName("Partition 2"), partition2);
    assertEquals(expectedResult, result);
}