Examples with CoordinatorStreamMessage org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage used on opensource projects

Example 6 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class ConfigManager method processConfigMessages.

/**
   * This function reads all the messages with "set-config" type added to the coordinator stream since the last time the method was invoked
   *
   * @param keysToProcess a list of keys to process. Only messages with these keys will call their handler function,
   *                      and other messages will be skipped. If the list is empty all messages will be skipped.
   */
@SuppressWarnings("unchecked")
private void processConfigMessages(List<String> keysToProcess) {
    if (!coordinatorStreamConsumer.hasNewMessages(coordinatorStreamIterator)) {
        return;
    }
    if (keysToProcess == null) {
        throw new IllegalArgumentException("The keys to process list is null");
    }
    for (CoordinatorStreamMessage message : coordinatorStreamConsumer.getUnreadMessages(coordinatorStreamIterator, SetConfig.TYPE)) {
        String key = null;
        try {
            SetConfig setConfigMessage = new SetConfig(message);
            key = setConfigMessage.getKey();
            Map<String, String> valuesMap = (Map<String, String>) setConfigMessage.getMessageMap().get("values");
            String value = null;
            if (valuesMap != null) {
                value = valuesMap.get("value");
            }
            log.debug("Received set-config message with key: " + key + " and value: " + value);
            if (keysToProcess.contains(key)) {
                if (key.equals(YARN_CONTAINER_COUNT_OPT)) {
                    handleYarnContainerChange(value);
                } else if (key.equals(SERVER_URL_OPT)) {
                    handleServerURLChange(value);
                } else {
                    log.info("Setting the " + key + " configuration is currently not supported, skipping the message");
                }
            }
        //TODO: change the handlers to implement a common interface, to make them pluggable
        } catch (Exception e) {
            log.debug("Error in reading a message, skipping message with key " + key);
        }
    }
}

Example 7 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class TestCoordinatorStreamMessage method testCoordinatorStreamMessage.

@Test
public void testCoordinatorStreamMessage() {
    CoordinatorStreamMessage message = new CoordinatorStreamMessage("source");
    assertEquals("source", message.getSource());
    assertEquals(CoordinatorStreamMessage.VERSION, message.getVersion());
    assertNotNull(message.getUsername());
    assertTrue(message.getTimestamp() > 0);
    assertTrue(!message.isDelete());
    CoordinatorStreamMessage secondMessage = new CoordinatorStreamMessage(message.getKeyArray(), message.getMessageMap());
    assertEquals(secondMessage, message);
}

Example 8 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testOrderKeyRewrite.

/**
   * Verify that if a particular key-value is written, then another, then the original again,
   * that the original occurs last in the set.
   */
@Test
public void testOrderKeyRewrite() throws InterruptedException {
    final SystemStream systemStream = new SystemStream("system", "stream");
    final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0));
    final SystemConsumer systemConsumer = mock(SystemConsumer.class);
    final List<IncomingMessageEnvelope> list = new ArrayList<>();
    SetConfig setConfig1 = new SetConfig("source", "key1", "value1");
    SetConfig setConfig2 = new SetConfig("source", "key1", "value2");
    SetConfig setConfig3 = new SetConfig("source", "key1", "value1");
    list.add(createIncomingMessageEnvelope(setConfig1, ssp));
    list.add(createIncomingMessageEnvelope(setConfig2, ssp));
    list.add(createIncomingMessageEnvelope(setConfig3, ssp));
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() {

        {
            put(ssp, list);
        }
    };
    when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap());
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    consumer.bootstrap();
    Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream();
    // First message should have been removed as a duplicate
    assertEquals(2, bootstrappedMessages.size());
    CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]);
    assertEquals(setConfig2, coordinatorStreamMessages[0]);
    //Config 3 MUST be the last message, not config 2
    assertEquals(setConfig3, coordinatorStreamMessages[1]);
}

Example 9 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class TestCoordinatorStreamSystemProducer method testCoordinatorStreamSystemProducer.

@Test
public void testCoordinatorStreamSystemProducer() {
    String source = "source";
    SystemStream systemStream = new SystemStream("system", "stream");
    MockCoordinatorSystemProducer systemProducer = new MockCoordinatorSystemProducer(source);
    MockSystemAdmin systemAdmin = new MockSystemAdmin();
    CoordinatorStreamSystemProducer producer = new CoordinatorStreamSystemProducer(systemStream, systemProducer, systemAdmin);
    SetConfig setConfig1 = new SetConfig(source, "job.name", "my-job-name");
    SetConfig setConfig2 = new SetConfig(source, "job.id", "1234");
    Delete delete = new Delete(source, "job.name", SetConfig.TYPE);
    assertFalse(systemProducer.isRegistered());
    producer.register(source);
    assertTrue(systemProducer.isRegistered());
    assertFalse(systemProducer.isStarted());
    producer.start();
    assertTrue(systemProducer.isStarted());
    producer.send(setConfig1);
    producer.send(setConfig2);
    producer.send(delete);
    assertFalse(systemProducer.isStopped());
    producer.stop();
    assertTrue(systemProducer.isStopped());
    List<OutgoingMessageEnvelope> envelopes = systemProducer.getEnvelopes();
    OutgoingMessageEnvelope envelope0 = envelopes.get(0);
    OutgoingMessageEnvelope envelope1 = envelopes.get(1);
    OutgoingMessageEnvelope envelope2 = envelopes.get(2);
    TypeReference<Object[]> keyRef = new TypeReference<Object[]>() {
    };
    TypeReference<Map<String, Object>> msgRef = new TypeReference<Map<String, Object>>() {
    };
    assertEquals(3, envelopes.size());
    assertEquals(new CoordinatorStreamMessage(setConfig1), new CoordinatorStreamMessage(deserialize((byte[]) envelope0.getKey(), keyRef), deserialize((byte[]) envelope0.getMessage(), msgRef)));
    assertEquals(new CoordinatorStreamMessage(setConfig2), new CoordinatorStreamMessage(deserialize((byte[]) envelope1.getKey(), keyRef), deserialize((byte[]) envelope1.getMessage(), msgRef)));
    assertEquals(new CoordinatorStreamMessage(delete), new CoordinatorStreamMessage(deserialize((byte[]) envelope2.getKey(), keyRef), deserialize((byte[]) envelope2.getMessage(), msgRef)));
}

Example 10 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class TestCoordinatorStreamMessage method testCoordinatorStreamMessageIsDelete.

@Test
public void testCoordinatorStreamMessageIsDelete() {
    CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[] {}, null);
    assertTrue(message.isDelete());
    assertNull(message.getMessageMap());
}