Examples with ProcessorNode org.apache.samza.zk.ZkUtils.ProcessorNode used on opensource projects

Example 1 with ProcessorNode

use of org.apache.samza.zk.ZkUtils.ProcessorNode in project samza by apache.

the class ZkJobCoordinator method getGrouperMetadata.

/**
 * Builds the {@link GrouperMetadataImpl} based upon provided {@param jobModelVersion}
 * and {@param processorNodes}.
 * @param jobModelVersion the most recent jobModelVersion available in the zookeeper.
 * @param processorNodes the list of live processors in the zookeeper.
 * @return the built grouper metadata.
 */
private GrouperMetadataImpl getGrouperMetadata(String jobModelVersion, List<ProcessorNode> processorNodes) {
    Map<TaskName, String> taskToProcessorId = new HashMap<>();
    Map<TaskName, List<SystemStreamPartition>> taskToSSPs = new HashMap<>();
    if (jobModelVersion != null) {
        JobModel jobModel = readJobModelFromMetadataStore(jobModelVersion);
        for (ContainerModel containerModel : jobModel.getContainers().values()) {
            for (TaskModel taskModel : containerModel.getTasks().values()) {
                taskToProcessorId.put(taskModel.getTaskName(), containerModel.getId());
                for (SystemStreamPartition partition : taskModel.getSystemStreamPartitions()) {
                    taskToSSPs.computeIfAbsent(taskModel.getTaskName(), k -> new ArrayList<>());
                    taskToSSPs.get(taskModel.getTaskName()).add(partition);
                }
            }
        }
    }
    Map<String, LocationId> processorLocality = new HashMap<>();
    for (ProcessorNode processorNode : processorNodes) {
        ProcessorData processorData = processorNode.getProcessorData();
        processorLocality.put(processorData.getProcessorId(), processorData.getLocationId());
    }
    Map<TaskName, LocationId> taskLocality = zkUtils.readTaskLocality();
    return new GrouperMetadataImpl(processorLocality, taskLocality, taskToSSPs, taskToProcessorId);
}

Example 2 with ProcessorNode

use of org.apache.samza.zk.ZkUtils.ProcessorNode in project samza by apache.

the class ZkJobCoordinator method doOnProcessorChange.

void doOnProcessorChange() {
    List<ProcessorNode> processorNodes = zkUtils.getAllProcessorNodes();
    List<String> currentProcessorIds = new ArrayList<>();
    for (ProcessorNode processorNode : processorNodes) {
        currentProcessorIds.add(processorNode.getProcessorData().getProcessorId());
    }
    Set<String> uniqueProcessorIds = new HashSet<>(currentProcessorIds);
    if (currentProcessorIds.size() != uniqueProcessorIds.size()) {
        LOG.info("Processors: {} has duplicates. Not generating JobModel.", currentProcessorIds);
        return;
    }
    // Generate the JobModel
    LOG.info("Generating new JobModel with processors: {}.", currentProcessorIds);
    JobModel newJobModel = generateNewJobModel(processorNodes);
    /*
     * Leader skips the rebalance even if there are changes in the quorum as long as the work assignment remains the same
     * across all the processors. The optimization is useful in the following scenarios
     *   1. The processor in the quorum restarts within the debounce window. Originally, this would trigger rebalance
     *      across the processors stopping and starting their work assignment which is detrimental to availability of
     *      the system. e.g. common scenario during rolling upgrades
     *   2. Processors in the quorum which don't have work assignment and their failures/restarts don't impact the
     *      quorum.
     */
    if (new ZkConfig(config).getEnableStartupWithActiveJobModel() && JobModelUtil.compareContainerModels(newJobModel, activeJobModel)) {
        LOG.info("Skipping rebalance since there are no changes in work assignment");
        return;
    }
    // Create checkpoint and changelog streams if they don't exist
    if (!hasLoadedMetadataResources) {
        loadMetadataResources(newJobModel);
        hasLoadedMetadataResources = true;
    }
    // Assign the next version of JobModel
    String currentJMVersion = zkUtils.getJobModelVersion();
    String nextJMVersion = zkUtils.getNextJobModelVersion(currentJMVersion);
    LOG.info("pid=" + processorId + "Generated new JobModel with version: " + nextJMVersion + " and processors: " + currentProcessorIds);
    // Publish the new job model
    publishJobModelToMetadataStore(newJobModel, nextJMVersion);
    // Start the barrier for the job model update
    barrier.create(nextJMVersion, currentProcessorIds);
    // Notify all processors about the new JobModel by updating JobModel Version number
    zkUtils.publishJobModelVersion(currentJMVersion, nextJMVersion);
    LOG.info("pid=" + processorId + "Published new Job Model. Version = " + nextJMVersion);
    debounceTimer.scheduleAfterDebounceTime(ON_ZK_CLEANUP, 0, () -> zkUtils.cleanupZK(NUM_VERSIONS_TO_LEAVE));
}