use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.
the class CoordinatorStreamSystemConsumer method register.
/**
* Retrieves the oldest offset in the coordinator stream, and registers the
* coordinator stream with the SystemConsumer using the earliest offset.
*/
public void register() {
if (isStarted) {
log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition);
return;
}
log.debug("Attempting to register: {}", coordinatorSystemStreamPartition);
Set<String> streamNames = new HashSet<String>();
String streamName = coordinatorSystemStreamPartition.getStream();
streamNames.add(streamName);
Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString()));
if (systemStreamMetadataMap == null) {
throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal.");
}
SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName);
if (systemStreamMetadata == null) {
throw new SamzaException("Expected " + streamName + " to be in system stream metadata.");
}
SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition());
if (systemStreamPartitionMetadata == null) {
throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist.");
}
String startingOffset = systemStreamPartitionMetadata.getOldestOffset();
log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset);
systemConsumer.register(coordinatorSystemStreamPartition, startingOffset);
}
use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.
the class TestSinglePartitionWithoutOffsetsSystemAdmin method testShouldGetASinglePartition.
@Test
public void testShouldGetASinglePartition() {
SinglePartitionWithoutOffsetsSystemAdmin admin = new SinglePartitionWithoutOffsetsSystemAdmin();
Set<String> streamNames = new HashSet<String>();
streamNames.add("a");
streamNames.add("b");
Map<String, SystemStreamMetadata> metadata = admin.getSystemStreamMetadata(streamNames);
assertEquals(2, metadata.size());
SystemStreamMetadata metadata1 = metadata.get("a");
SystemStreamMetadata metadata2 = metadata.get("b");
assertEquals(1, metadata1.getSystemStreamPartitionMetadata().size());
assertEquals(1, metadata2.getSystemStreamPartitionMetadata().size());
assertNull(metadata.get(new SystemStreamPartition("test-system", "c", new Partition(0))));
}
use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.
the class TestExecutionPlanner method createSystemAdmin.
static SystemAdmin createSystemAdmin(Map<String, Integer> streamToPartitions) {
return new SystemAdmin() {
@Override
public Map<SystemStreamPartition, String> getOffsetsAfter(Map<SystemStreamPartition, String> offsets) {
return null;
}
@Override
public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
Map<String, SystemStreamMetadata> map = new HashMap<>();
for (String stream : streamNames) {
Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> m = new HashMap<>();
for (int i = 0; i < streamToPartitions.get(stream); i++) {
m.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", ""));
}
map.put(stream, new SystemStreamMetadata(stream, m));
}
return map;
}
@Override
public void createChangelogStream(String streamName, int numOfPartitions) {
}
@Override
public void validateChangelogStream(String streamName, int numOfPartitions) {
}
@Override
public void createCoordinatorStream(String streamName) {
}
@Override
public Integer offsetComparator(String offset1, String offset2) {
return null;
}
};
}
use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.
the class TestHdfsSystemConsumer method testHdfsSystemConsumerE2E.
/*
* A simple end to end test that covers the workflow from system admin to
* partitioner, system consumer, and so on, making sure the basic functionality
* works as expected.
*/
@Test
public void testHdfsSystemConsumerE2E() throws Exception {
Config config = generateDefaultConfig();
HdfsSystemFactory systemFactory = new HdfsSystemFactory();
// create admin and do partitioning
HdfsSystemAdmin systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config);
String streamName = WORKING_DIRECTORY;
Set<String> streamNames = new HashSet<>();
streamNames.add(streamName);
generateAvroDataFiles();
Map<String, SystemStreamMetadata> streamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
SystemStreamMetadata systemStreamMetadata = streamMetadataMap.get(streamName);
Assert.assertEquals(NUM_FILES, systemStreamMetadata.getSystemStreamPartitionMetadata().size());
// create consumer and read from files
HdfsSystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, new NoOpMetricsRegistry());
Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadataMap = systemStreamMetadata.getSystemStreamPartitionMetadata();
Set<SystemStreamPartition> systemStreamPartitionSet = new HashSet<>();
metadataMap.forEach((partition, metadata) -> {
SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, streamName, partition);
systemStreamPartitionSet.add(ssp);
String offset = metadata.getOldestOffset();
systemConsumer.register(ssp, offset);
});
systemConsumer.start();
// verify events read from consumer
int eventsReceived = 0;
// one "End of Stream" event in the end
int totalEvents = (NUM_EVENTS + 1) * NUM_FILES;
int remainingRetires = 100;
Map<SystemStreamPartition, List<IncomingMessageEnvelope>> overallResults = new HashMap<>();
while (eventsReceived < totalEvents && remainingRetires > 0) {
remainingRetires--;
Map<SystemStreamPartition, List<IncomingMessageEnvelope>> result = systemConsumer.poll(systemStreamPartitionSet, 200);
for (SystemStreamPartition ssp : result.keySet()) {
List<IncomingMessageEnvelope> messageEnvelopeList = result.get(ssp);
overallResults.putIfAbsent(ssp, new ArrayList<>());
overallResults.get(ssp).addAll(messageEnvelopeList);
if (overallResults.get(ssp).size() >= NUM_EVENTS + 1) {
systemStreamPartitionSet.remove(ssp);
}
eventsReceived += messageEnvelopeList.size();
}
}
Assert.assertEquals(eventsReceived, totalEvents);
Assert.assertEquals(NUM_FILES, overallResults.size());
overallResults.values().forEach(messages -> {
Assert.assertEquals(NUM_EVENTS + 1, messages.size());
for (int index = 0; index < NUM_EVENTS; index++) {
GenericRecord record = (GenericRecord) messages.get(index).getMessage();
Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1));
Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString());
}
Assert.assertEquals(messages.get(NUM_EVENTS).getOffset(), IncomingMessageEnvelope.END_OF_STREAM_OFFSET);
});
}
use of org.apache.samza.system.SystemStreamMetadata in project samza by apache.
the class TestStorageRecovery method putMetadata.
private void putMetadata() {
SystemStreamMetadata.SystemStreamPartitionMetadata sspm = new SystemStreamMetadata.SystemStreamPartitionMetadata("0", "1", "2");
HashMap<Partition, SystemStreamPartitionMetadata> map = new HashMap<Partition, SystemStreamPartitionMetadata>();
map.put(new Partition(0), sspm);
map.put(new Partition(1), sspm);
systemStreamMetadata = new SystemStreamMetadata(SYSTEM_STREAM_NAME, map);
HashMap<Partition, SystemStreamPartitionMetadata> map1 = new HashMap<Partition, SystemStreamPartitionMetadata>();
map1.put(new Partition(0), sspm);
map1.put(new Partition(1), sspm);
inputSystemStreamMetadata = new SystemStreamMetadata(INPUT_STREAM, map1);
}
Aggregations