Examples with Partition org.apache.samza.Partition used on opensource projects

Example 21 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TestSamzaObjectMapper method setup.

@Before
public void setup() throws IOException {
    Map<String, String> configMap = new HashMap<String, String>();
    Set<SystemStreamPartition> ssp = new HashSet<>();
    configMap.put("a", "b");
    Config config = new MapConfig(configMap);
    TaskName taskName = new TaskName("test");
    ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
    TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
    Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
    tasks.put(taskName, taskModel);
    ContainerModel containerModel = new ContainerModel("1", 1, tasks);
    Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
    containerMap.put("1", containerModel);
    jobModel = new JobModel(config, containerMap);
}

Example 22 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumer.

@Test
public void testCoordinatorStreamSystemConsumer() {
    Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
    expectedConfig.put("job.id", "1234");
    SystemStream systemStream = new SystemStream("system", "stream");
    MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    assertEquals(0, systemConsumer.getRegisterCount());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
    assertFalse(systemConsumer.isStarted());
    consumer.start();
    assertTrue(systemConsumer.isStarted());
    try {
        consumer.getConfig();
        fail("Should have failed when retrieving config before bootstrapping.");
    } catch (SamzaException e) {
    // Expected.
    }
    consumer.bootstrap();
    assertEquals(expectedConfig, consumer.getConfig());
    assertFalse(systemConsumer.isStopped());
    consumer.stop();
    assertTrue(systemConsumer.isStopped());
}

Example 23 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumerRegisterOnceOnly.

@Test
public void testCoordinatorStreamSystemConsumerRegisterOnceOnly() throws Exception {
    Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
    expectedConfig.put("job.id", "1234");
    SystemStream systemStream = new SystemStream("system", "stream");
    MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    assertEquals(0, systemConsumer.getRegisterCount());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
    assertFalse(systemConsumer.isStarted());
    consumer.start();
    assertTrue(systemConsumer.isStarted());
    consumer.register();
    assertEquals(1, systemConsumer.getRegisterCount());
}

Example 24 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class TestTaskConfigJava method testGetBroadcastSystemStreamPartitions.

@Test
public void testGetBroadcastSystemStreamPartitions() {
    HashMap<String, String> map = new HashMap<String, String>();
    map.put("task.broadcast.inputs", "kafka.foo#4, kafka.boo#5, kafka.z-o-o#[12-14], kafka.foo.bar#[3-4]");
    Config config = new MapConfig(map);
    TaskConfigJava taskConfig = new TaskConfigJava(config);
    Set<SystemStreamPartition> systemStreamPartitionSet = taskConfig.getBroadcastSystemStreamPartitions();
    HashSet<SystemStreamPartition> expected = new HashSet<SystemStreamPartition>();
    expected.add(new SystemStreamPartition("kafka", "foo", new Partition(4)));
    expected.add(new SystemStreamPartition("kafka", "boo", new Partition(5)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(12)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(13)));
    expected.add(new SystemStreamPartition("kafka", "z-o-o", new Partition(14)));
    expected.add(new SystemStreamPartition("kafka", "foo.bar", new Partition(3)));
    expected.add(new SystemStreamPartition("kafka", "foo.bar", new Partition(4)));
    assertEquals(expected, systemStreamPartitionSet);
    map.put("task.broadcast.inputs", "kafka.foo");
    taskConfig = new TaskConfigJava(new MapConfig(map));
    boolean catchCorrectException = false;
    try {
        taskConfig.getBroadcastSystemStreamPartitions();
    } catch (IllegalArgumentException e) {
        catchCorrectException = true;
    }
    assertTrue(catchCorrectException);
    map.put("task.broadcast.inputs", "kafka.org.apache.events.WhitelistedIps#1-2");
    taskConfig = new TaskConfigJava(new MapConfig(map));
    boolean invalidFormatException = false;
    try {
        taskConfig.getBroadcastSystemStreamPartitions();
    } catch (IllegalArgumentException e) {
        invalidFormatException = true;
    }
    assertTrue(invalidFormatException);
}

Example 25 with Partition

use of org.apache.samza.Partition in project samza by apache.

the class PartitionDescriptorUtil method getDescriptorMapFromJson.

public static Map<Partition, List<String>> getDescriptorMapFromJson(String json) {
    try {
        @SuppressWarnings("unchecked") Map<String, String> rawMap = new ObjectMapper().readValue(json, HashMap.class);
        Map<Partition, List<String>> descriptorMap = new HashMap<>();
        rawMap.forEach((key, value) -> descriptorMap.put(new Partition(Integer.valueOf(key)), getPathsFromString(value)));
        return descriptorMap;
    } catch (IOException | NumberFormatException e) {
        throw new SamzaException("Failed to convert json: " + json, e);
    }
}