Examples with IncomingMessageEnvelope org.apache.samza.system.IncomingMessageEnvelope used on opensource projects

Example 6 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class BlockingEnvelopeMap method subtractSizeOnQDrain.

private void subtractSizeOnQDrain(SystemStreamPartition systemStreamPartition, List<IncomingMessageEnvelope> outgoingList) {
    long outgoingListBytes = 0;
    for (IncomingMessageEnvelope envelope : outgoingList) {
        outgoingListBytes += envelope.getSize();
    }
    // subtract the size of the messages dequeued.
    bufferedMessagesSize.get(systemStreamPartition).addAndGet(-1 * outgoingListBytes);
}

Example 7 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class MockCoordinatorStreamWrappedConsumer method convertConfigToCoordinatorMessage.

private void convertConfigToCoordinatorMessage(Config config) {
    try {
        for (Map.Entry<String, String> configPair : config.entrySet()) {
            byte[] keyBytes = null;
            byte[] messgeBytes = null;
            if (configPair.getKey().startsWith(CHANGELOGPREFIX)) {
                String[] changelogInfo = configPair.getKey().split(":");
                String changeLogPartition = configPair.getValue();
                SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition));
                keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8");
                messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8");
            } else {
                SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue());
                keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8");
                messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8");
            }
            // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp.
            put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes));
        }
        setIsAtHead(systemStreamPartition, true);
    } catch (Exception e) {
        throw new SamzaException(e);
    }
}

Example 8 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testOrderKeyRewrite.

/**
   * Verify that if a particular key-value is written, then another, then the original again,
   * that the original occurs last in the set.
   */
@Test
public void testOrderKeyRewrite() throws InterruptedException {
    final SystemStream systemStream = new SystemStream("system", "stream");
    final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0));
    final SystemConsumer systemConsumer = mock(SystemConsumer.class);
    final List<IncomingMessageEnvelope> list = new ArrayList<>();
    SetConfig setConfig1 = new SetConfig("source", "key1", "value1");
    SetConfig setConfig2 = new SetConfig("source", "key1", "value2");
    SetConfig setConfig3 = new SetConfig("source", "key1", "value1");
    list.add(createIncomingMessageEnvelope(setConfig1, ssp));
    list.add(createIncomingMessageEnvelope(setConfig2, ssp));
    list.add(createIncomingMessageEnvelope(setConfig3, ssp));
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() {

        {
            put(ssp, list);
        }
    };
    when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap());
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    consumer.bootstrap();
    Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream();
    // First message should have been removed as a duplicate
    assertEquals(2, bootstrappedMessages.size());
    CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]);
    assertEquals(setConfig2, coordinatorStreamMessages[0]);
    //Config 3 MUST be the last message, not config 2
    assertEquals(setConfig3, coordinatorStreamMessages[1]);
}

Example 9 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class MultiFileHdfsReader method readNext.

public IncomingMessageEnvelope readNext() {
    if (!hasNext()) {
        LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition);
        return null;
    }
    // record the next offset before we read, so when the read fails and we reconnect,
    // we seek to the same offset that we try below
    curSingleFileOffset = curReader.nextOffset();
    IncomingMessageEnvelope messageEnvelope = curReader.readNext();
    // Copy everything except for the offset. Turn the single-file style offset into a multi-file one
    return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize());
}

Example 10 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestHdfsSystemConsumer method testHdfsSystemConsumerE2E.

/*
   * A simple end to end test that covers the workflow from system admin to
   * partitioner, system consumer, and so on, making sure the basic functionality
   * works as expected.
   */
@Test
public void testHdfsSystemConsumerE2E() throws Exception {
    Config config = generateDefaultConfig();
    HdfsSystemFactory systemFactory = new HdfsSystemFactory();
    // create admin and do partitioning
    HdfsSystemAdmin systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config);
    String streamName = WORKING_DIRECTORY;
    Set<String> streamNames = new HashSet<>();
    streamNames.add(streamName);
    generateAvroDataFiles();
    Map<String, SystemStreamMetadata> streamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
    SystemStreamMetadata systemStreamMetadata = streamMetadataMap.get(streamName);
    Assert.assertEquals(NUM_FILES, systemStreamMetadata.getSystemStreamPartitionMetadata().size());
    // create consumer and read from files
    HdfsSystemConsumer systemConsumer = systemFactory.getConsumer(SYSTEM_NAME, config, new NoOpMetricsRegistry());
    Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadataMap = systemStreamMetadata.getSystemStreamPartitionMetadata();
    Set<SystemStreamPartition> systemStreamPartitionSet = new HashSet<>();
    metadataMap.forEach((partition, metadata) -> {
        SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, streamName, partition);
        systemStreamPartitionSet.add(ssp);
        String offset = metadata.getOldestOffset();
        systemConsumer.register(ssp, offset);
    });
    systemConsumer.start();
    // verify events read from consumer
    int eventsReceived = 0;
    // one "End of Stream" event in the end
    int totalEvents = (NUM_EVENTS + 1) * NUM_FILES;
    int remainingRetires = 100;
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> overallResults = new HashMap<>();
    while (eventsReceived < totalEvents && remainingRetires > 0) {
        remainingRetires--;
        Map<SystemStreamPartition, List<IncomingMessageEnvelope>> result = systemConsumer.poll(systemStreamPartitionSet, 200);
        for (SystemStreamPartition ssp : result.keySet()) {
            List<IncomingMessageEnvelope> messageEnvelopeList = result.get(ssp);
            overallResults.putIfAbsent(ssp, new ArrayList<>());
            overallResults.get(ssp).addAll(messageEnvelopeList);
            if (overallResults.get(ssp).size() >= NUM_EVENTS + 1) {
                systemStreamPartitionSet.remove(ssp);
            }
            eventsReceived += messageEnvelopeList.size();
        }
    }
    Assert.assertEquals(eventsReceived, totalEvents);
    Assert.assertEquals(NUM_FILES, overallResults.size());
    overallResults.values().forEach(messages -> {
        Assert.assertEquals(NUM_EVENTS + 1, messages.size());
        for (int index = 0; index < NUM_EVENTS; index++) {
            GenericRecord record = (GenericRecord) messages.get(index).getMessage();
            Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1));
            Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString());
        }
        Assert.assertEquals(messages.get(NUM_EVENTS).getOffset(), IncomingMessageEnvelope.END_OF_STREAM_OFFSET);
    });
}