use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TestMultiFileHdfsReader method testReconnect.
@Test
public void testReconnect() {
SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0");
// first read a few events, and then reconnect
for (int i = 0; i < NUM_EVENTS / 2; i++) {
multiReader.readNext();
}
IncomingMessageEnvelope envelope = multiReader.readNext();
multiReader.reconnect();
IncomingMessageEnvelope envelopeAfterReconnect = multiReader.readNext();
Assert.assertEquals(envelope, envelopeAfterReconnect);
multiReader.close();
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TestMultiFileHdfsReader method testInvalidPartitionDescriptor.
@Test(expected = SamzaException.class)
public void testInvalidPartitionDescriptor() {
SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0));
new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, new ArrayList<>(), "0:0");
Assert.fail();
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TestAsyncRunLoop method testEndOfStreamOffsetManagement.
// TODO: Add assertions.
//@Test
public void testEndOfStreamOffsetManagement() throws Exception {
//explicitly configure to disable commits inside process or window calls and invoke commit from end of stream
TestTask mockStreamTask1 = new TestTask(true, false, false, null);
TestTask mockStreamTask2 = new TestTask(true, false, false, null);
Partition p1 = new Partition(1);
Partition p2 = new Partition(2);
SystemStreamPartition ssp1 = new SystemStreamPartition("system1", "stream1", p1);
SystemStreamPartition ssp2 = new SystemStreamPartition("system1", "stream2", p2);
IncomingMessageEnvelope envelope1 = new IncomingMessageEnvelope(ssp2, "1", "key1", "message1");
IncomingMessageEnvelope envelope2 = new IncomingMessageEnvelope(ssp2, "2", "key1", "message1");
IncomingMessageEnvelope envelope3 = IncomingMessageEnvelope.buildEndOfStreamEnvelope(ssp2);
Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspMap = new HashMap<>();
List<IncomingMessageEnvelope> messageList = new ArrayList<>();
messageList.add(envelope1);
messageList.add(envelope2);
messageList.add(envelope3);
sspMap.put(ssp2, messageList);
SystemConsumer mockConsumer = mock(SystemConsumer.class);
when(mockConsumer.poll(anyObject(), anyLong())).thenReturn(sspMap);
HashMap<String, SystemConsumer> systemConsumerMap = new HashMap<>();
systemConsumerMap.put("system1", mockConsumer);
SystemConsumers consumers = TestSystemConsumers.getSystemConsumers(systemConsumerMap);
TaskName taskName1 = new TaskName("task1");
TaskName taskName2 = new TaskName("task2");
Set<TaskName> taskNames = new HashSet<>();
taskNames.add(taskName1);
taskNames.add(taskName2);
OffsetManager offsetManager = mock(OffsetManager.class);
when(offsetManager.getLastProcessedOffset(taskName1, ssp1)).thenReturn(Option.apply("3"));
when(offsetManager.getLastProcessedOffset(taskName2, ssp2)).thenReturn(Option.apply("0"));
when(offsetManager.getStartingOffset(taskName1, ssp1)).thenReturn(Option.apply(IncomingMessageEnvelope.END_OF_STREAM_OFFSET));
when(offsetManager.getStartingOffset(taskName2, ssp2)).thenReturn(Option.apply("1"));
TaskInstance taskInstance1 = createTaskInstance(mockStreamTask1, taskName1, ssp1, offsetManager, consumers);
TaskInstance taskInstance2 = createTaskInstance(mockStreamTask2, taskName2, ssp2, offsetManager, consumers);
Map<TaskName, TaskInstance> tasks = new HashMap<>();
tasks.put(taskName1, taskInstance1);
tasks.put(taskName2, taskInstance2);
taskInstance1.registerConsumers();
taskInstance2.registerConsumers();
consumers.start();
int maxMessagesInFlight = 1;
AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumers, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, false);
runLoop.run();
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumer.
@Test
public void testCoordinatorStreamSystemConsumer() {
Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
expectedConfig.put("job.id", "1234");
SystemStream systemStream = new SystemStream("system", "stream");
MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
assertEquals(0, systemConsumer.getRegisterCount());
consumer.register();
assertEquals(1, systemConsumer.getRegisterCount());
assertFalse(systemConsumer.isStarted());
consumer.start();
assertTrue(systemConsumer.isStarted());
try {
consumer.getConfig();
fail("Should have failed when retrieving config before bootstrapping.");
} catch (SamzaException e) {
// Expected.
}
consumer.bootstrap();
assertEquals(expectedConfig, consumer.getConfig());
assertFalse(systemConsumer.isStopped());
consumer.stop();
assertTrue(systemConsumer.isStopped());
}
use of org.apache.samza.system.SystemStreamPartition in project samza by apache.
the class TestCoordinatorStreamSystemConsumer method testCoordinatorStreamSystemConsumerRegisterOnceOnly.
@Test
public void testCoordinatorStreamSystemConsumerRegisterOnceOnly() throws Exception {
Map<String, String> expectedConfig = new LinkedHashMap<String, String>();
expectedConfig.put("job.id", "1234");
SystemStream systemStream = new SystemStream("system", "stream");
MockSystemConsumer systemConsumer = new MockSystemConsumer(new SystemStreamPartition(systemStream, new Partition(0)));
CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
assertEquals(0, systemConsumer.getRegisterCount());
consumer.register();
assertEquals(1, systemConsumer.getRegisterCount());
assertFalse(systemConsumer.isStarted());
consumer.start();
assertTrue(systemConsumer.isStarted());
consumer.register();
assertEquals(1, systemConsumer.getRegisterCount());
}
Aggregations