Examples with SetConfig org.apache.samza.coordinator.stream.messages.SetConfig used on opensource projects

Example 1 with SetConfig

use of org.apache.samza.coordinator.stream.messages.SetConfig in project samza by apache.

the class CoordinatorStreamSystemConsumer method bootstrap.

/**
   * Read all messages from the earliest offset, all the way to the latest.
   * Currently, this method only pays attention to config messages.
   */
public void bootstrap() {
    synchronized (bootstrapLock) {
        // Make a copy so readers aren't affected while we modify the set.
        final LinkedHashSet<CoordinatorStreamMessage> bootstrappedMessages = new LinkedHashSet<>(bootstrappedStreamSet);
        log.info("Bootstrapping configuration from coordinator stream.");
        SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(systemConsumer, coordinatorSystemStreamPartition);
        try {
            while (iterator.hasNext()) {
                IncomingMessageEnvelope envelope = iterator.next();
                Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
                Map<String, Object> valueMap = null;
                if (envelope.getMessage() != null) {
                    valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
                }
                CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
                log.debug("Received coordinator stream message: {}", coordinatorStreamMessage);
                // Remove any existing entry. Set.add() does not add if the element already exists.
                if (bootstrappedMessages.remove(coordinatorStreamMessage)) {
                    log.debug("Removed duplicate message: {}", coordinatorStreamMessage);
                }
                bootstrappedMessages.add(coordinatorStreamMessage);
                if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) {
                    String configKey = coordinatorStreamMessage.getKey();
                    if (coordinatorStreamMessage.isDelete()) {
                        configMap.remove(configKey);
                    } else {
                        String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue();
                        configMap.put(configKey, configValue);
                    }
                }
            }
            bootstrappedStreamSet = Collections.unmodifiableSet(bootstrappedMessages);
            log.debug("Bootstrapped configuration: {}", configMap);
            isBootstrapped = true;
        } catch (Exception e) {
            throw new SamzaException(e);
        }
    }
}

Example 2 with SetConfig

use of org.apache.samza.coordinator.stream.messages.SetConfig in project samza by apache.

the class ConfigManager method processConfigMessages.

/**
   * This function reads all the messages with "set-config" type added to the coordinator stream since the last time the method was invoked
   *
   * @param keysToProcess a list of keys to process. Only messages with these keys will call their handler function,
   *                      and other messages will be skipped. If the list is empty all messages will be skipped.
   */
@SuppressWarnings("unchecked")
private void processConfigMessages(List<String> keysToProcess) {
    if (!coordinatorStreamConsumer.hasNewMessages(coordinatorStreamIterator)) {
        return;
    }
    if (keysToProcess == null) {
        throw new IllegalArgumentException("The keys to process list is null");
    }
    for (CoordinatorStreamMessage message : coordinatorStreamConsumer.getUnreadMessages(coordinatorStreamIterator, SetConfig.TYPE)) {
        String key = null;
        try {
            SetConfig setConfigMessage = new SetConfig(message);
            key = setConfigMessage.getKey();
            Map<String, String> valuesMap = (Map<String, String>) setConfigMessage.getMessageMap().get("values");
            String value = null;
            if (valuesMap != null) {
                value = valuesMap.get("value");
            }
            log.debug("Received set-config message with key: " + key + " and value: " + value);
            if (keysToProcess.contains(key)) {
                if (key.equals(YARN_CONTAINER_COUNT_OPT)) {
                    handleYarnContainerChange(value);
                } else if (key.equals(SERVER_URL_OPT)) {
                    handleServerURLChange(value);
                } else {
                    log.info("Setting the " + key + " configuration is currently not supported, skipping the message");
                }
            }
        //TODO: change the handlers to implement a common interface, to make them pluggable
        } catch (Exception e) {
            log.debug("Error in reading a message, skipping message with key " + key);
        }
    }
}

Example 3 with SetConfig

use of org.apache.samza.coordinator.stream.messages.SetConfig in project samza by apache.

the class MockCoordinatorStreamWrappedConsumer method convertConfigToCoordinatorMessage.

private void convertConfigToCoordinatorMessage(Config config) {
    try {
        for (Map.Entry<String, String> configPair : config.entrySet()) {
            byte[] keyBytes = null;
            byte[] messgeBytes = null;
            if (configPair.getKey().startsWith(CHANGELOGPREFIX)) {
                String[] changelogInfo = configPair.getKey().split(":");
                String changeLogPartition = configPair.getValue();
                SetChangelogMapping changelogMapping = new SetChangelogMapping(changelogInfo[1], changelogInfo[2], Integer.parseInt(changeLogPartition));
                keyBytes = MAPPER.writeValueAsString(changelogMapping.getKeyArray()).getBytes("UTF-8");
                messgeBytes = MAPPER.writeValueAsString(changelogMapping.getMessageMap()).getBytes("UTF-8");
            } else {
                SetConfig setConfig = new SetConfig("source", configPair.getKey(), configPair.getValue());
                keyBytes = MAPPER.writeValueAsString(setConfig.getKeyArray()).getBytes("UTF-8");
                messgeBytes = MAPPER.writeValueAsString(setConfig.getMessageMap()).getBytes("UTF-8");
            }
            // The ssp here is the coordinator ssp (which is always fixed) and not the task ssp.
            put(systemStreamPartition, new IncomingMessageEnvelope(systemStreamPartition, "", keyBytes, messgeBytes));
        }
        setIsAtHead(systemStreamPartition, true);
    } catch (Exception e) {
        throw new SamzaException(e);
    }
}

Example 4 with SetConfig

use of org.apache.samza.coordinator.stream.messages.SetConfig in project samza by apache.

the class TestCoordinatorStreamMessage method testHashCodeAndEquality.

@Test
public void testHashCodeAndEquality() {
    SetConfig message = new SetConfig("source", "key1", "value1");
    SetConfig message1 = new SetConfig("source", "key1", "value1");
    SetConfig message2 = new SetConfig("source", "key2", "value1");
    assertEquals(message.hashCode(), message1.hashCode());
    assertEquals(message, message1);
    assertTrue(!message.equals(message2));
}

Example 5 with SetConfig

use of org.apache.samza.coordinator.stream.messages.SetConfig in project samza by apache.

the class TestCoordinatorStreamSystemConsumer method testOrderKeyRewrite.

/**
   * Verify that if a particular key-value is written, then another, then the original again,
   * that the original occurs last in the set.
   */
@Test
public void testOrderKeyRewrite() throws InterruptedException {
    final SystemStream systemStream = new SystemStream("system", "stream");
    final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0));
    final SystemConsumer systemConsumer = mock(SystemConsumer.class);
    final List<IncomingMessageEnvelope> list = new ArrayList<>();
    SetConfig setConfig1 = new SetConfig("source", "key1", "value1");
    SetConfig setConfig2 = new SetConfig("source", "key1", "value2");
    SetConfig setConfig3 = new SetConfig("source", "key1", "value1");
    list.add(createIncomingMessageEnvelope(setConfig1, ssp));
    list.add(createIncomingMessageEnvelope(setConfig2, ssp));
    list.add(createIncomingMessageEnvelope(setConfig3, ssp));
    Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() {

        {
            put(ssp, list);
        }
    };
    when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap());
    CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin());
    consumer.bootstrap();
    Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream();
    // First message should have been removed as a duplicate
    assertEquals(2, bootstrappedMessages.size());
    CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]);
    assertEquals(setConfig2, coordinatorStreamMessages[0]);
    //Config 3 MUST be the last message, not config 2
    assertEquals(setConfig3, coordinatorStreamMessages[1]);
}