Examples with CoordinatorStreamMessage org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage used on opensource projects

Example 1 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class TaskAssignmentManager method readTaskAssignment.

/**
   * Method to allow read container task information from coordinator stream. This method is used
   * in {@link org.apache.samza.coordinator.JobModelManager}.
   *
   * @return the map of taskName: containerId
   */
public Map<String, String> readTaskAssignment() {
    taskNameToContainerId.clear();
    for (CoordinatorStreamMessage message : getBootstrappedStream(SetTaskContainerMapping.TYPE)) {
        if (message.isDelete()) {
            taskNameToContainerId.remove(message.getKey());
            log.debug("Got TaskContainerMapping delete message: {}", message);
        } else {
            SetTaskContainerMapping mapping = new SetTaskContainerMapping(message);
            taskNameToContainerId.put(mapping.getKey(), mapping.getTaskAssignment());
            log.debug("Got TaskContainerMapping message: {}", mapping);
        }
    }
    for (Map.Entry<String, String> entry : taskNameToContainerId.entrySet()) {
        log.debug("Assignment for task \"{}\": {}", entry.getKey(), entry.getValue());
    }
    return Collections.unmodifiableMap(new HashMap<>(taskNameToContainerId));
}

Example 2 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class CoordinatorStreamSystemConsumer method bootstrap.

/**
   * Read all messages from the earliest offset, all the way to the latest.
   * Currently, this method only pays attention to config messages.
   */
public void bootstrap() {
    synchronized (bootstrapLock) {
        // Make a copy so readers aren't affected while we modify the set.
        final LinkedHashSet<CoordinatorStreamMessage> bootstrappedMessages = new LinkedHashSet<>(bootstrappedStreamSet);
        log.info("Bootstrapping configuration from coordinator stream.");
        SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(systemConsumer, coordinatorSystemStreamPartition);
        try {
            while (iterator.hasNext()) {
                IncomingMessageEnvelope envelope = iterator.next();
                Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
                Map<String, Object> valueMap = null;
                if (envelope.getMessage() != null) {
                    valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
                }
                CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
                log.debug("Received coordinator stream message: {}", coordinatorStreamMessage);
                // Remove any existing entry. Set.add() does not add if the element already exists.
                if (bootstrappedMessages.remove(coordinatorStreamMessage)) {
                    log.debug("Removed duplicate message: {}", coordinatorStreamMessage);
                }
                bootstrappedMessages.add(coordinatorStreamMessage);
                if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) {
                    String configKey = coordinatorStreamMessage.getKey();
                    if (coordinatorStreamMessage.isDelete()) {
                        configMap.remove(configKey);
                    } else {
                        String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue();
                        configMap.put(configKey, configValue);
                    }
                }
            }
            bootstrappedStreamSet = Collections.unmodifiableSet(bootstrappedMessages);
            log.debug("Bootstrapped configuration: {}", configMap);
            isBootstrapped = true;
        } catch (Exception e) {
            throw new SamzaException(e);
        }
    }
}

Example 3 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class CoordinatorStreamSystemConsumer method getBootstrappedStream.

public Set<CoordinatorStreamMessage> getBootstrappedStream(String type) {
    log.debug("Bootstrapping coordinator stream for messages of type {}", type);
    bootstrap();
    LinkedHashSet<CoordinatorStreamMessage> bootstrappedStream = new LinkedHashSet<CoordinatorStreamMessage>();
    for (CoordinatorStreamMessage coordinatorStreamMessage : bootstrappedStreamSet) {
        log.trace("Considering message: {}", coordinatorStreamMessage);
        if (type.equalsIgnoreCase(coordinatorStreamMessage.getType())) {
            log.trace("Adding message: {}", coordinatorStreamMessage);
            bootstrappedStream.add(coordinatorStreamMessage);
        }
    }
    return bootstrappedStream;
}

Example 4 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class ChangelogPartitionManager method readChangeLogPartitionMapping.

/**
   * Read the taskName to partition mapping that is being maintained by this ChangelogManager
   * @return TaskName to change log partition mapping, or an empty map if there were no messages.
   */
public Map<TaskName, Integer> readChangeLogPartitionMapping() {
    log.debug("Reading changelog partition information");
    final HashMap<TaskName, Integer> changelogMapping = new HashMap<TaskName, Integer>();
    for (CoordinatorStreamMessage coordinatorStreamMessage : getBootstrappedStream(SetChangelogMapping.TYPE)) {
        SetChangelogMapping changelogMapEntry = new SetChangelogMapping(coordinatorStreamMessage);
        changelogMapping.put(new TaskName(changelogMapEntry.getTaskName()), changelogMapEntry.getPartition());
        log.debug("TaskName: {} is mapped to {}", changelogMapEntry.getTaskName(), changelogMapEntry.getPartition());
    }
    return changelogMapping;
}

Example 5 with CoordinatorStreamMessage

use of org.apache.samza.coordinator.stream.messages.CoordinatorStreamMessage in project samza by apache.

the class LocalityManager method readContainerLocality.

/**
   * Method to allow read container locality information from coordinator stream. This method is used
   * in {@link org.apache.samza.coordinator.JobModelManager}.
   *
   * @return the map of containerId: (hostname, jmxAddress, jmxTunnelAddress)
   */
public Map<String, Map<String, String>> readContainerLocality() {
    if (this.writeOnly) {
        throw new UnsupportedOperationException("Read container locality function is not supported in write-only LocalityManager");
    }
    Map<String, Map<String, String>> allMappings = new HashMap<>();
    for (CoordinatorStreamMessage message : getBootstrappedStream(SetContainerHostMapping.TYPE)) {
        SetContainerHostMapping mapping = new SetContainerHostMapping(message);
        Map<String, String> localityMappings = new HashMap<>();
        localityMappings.put(SetContainerHostMapping.HOST_KEY, mapping.getHostLocality());
        localityMappings.put(SetContainerHostMapping.JMX_URL_KEY, mapping.getJmxUrl());
        localityMappings.put(SetContainerHostMapping.JMX_TUNNELING_URL_KEY, mapping.getJmxTunnelingUrl());
        allMappings.put(mapping.getKey(), localityMappings);
    }
    containerToHostMapping = Collections.unmodifiableMap(allMappings);
    for (Map.Entry<String, Map<String, String>> entry : containerToHostMapping.entrySet()) {
        log.debug(String.format("Locality for container %s: %s", entry.getKey(), entry.getValue()));
    }
    return allMappings;
}