Examples with GrouperMetadata org.apache.samza.container.grouper.task.GrouperMetadata used on opensource projects

Example 6 with GrouperMetadata

use of org.apache.samza.container.grouper.task.GrouperMetadata in project samza by apache.

the class TestGroupByPartitionWithGrouperProxy method testSingleStreamRepartitioning.

@Test
public void testSingleStreamRepartitioning() {
    Map<TaskName, List<SystemStreamPartition>> prevGroupingWithSingleStream = ImmutableMap.<TaskName, List<SystemStreamPartition>>builder().put(new TaskName("Partition 0"), ImmutableList.of(new SystemStreamPartition("kafka", "PVE", new Partition(0)))).put(new TaskName("Partition 1"), ImmutableList.of(new SystemStreamPartition("kafka", "PVE", new Partition(1)))).put(new TaskName("Partition 2"), ImmutableList.of(new SystemStreamPartition("kafka", "PVE", new Partition(2)))).put(new TaskName("Partition 3"), ImmutableList.of(new SystemStreamPartition("kafka", "PVE", new Partition(3)))).build();
    Set<SystemStreamPartition> currSsps = IntStream.range(0, 8).mapToObj(partitionId -> new SystemStreamPartition("kafka", "PVE", new Partition(partitionId))).collect(Collectors.toSet());
    Map<TaskName, Set<SystemStreamPartition>> expectedGroupingForStateful = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder().put(new TaskName("Partition 1"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(1)), new SystemStreamPartition("kafka", "PVE", new Partition(5)))).put(new TaskName("Partition 0"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(0)), new SystemStreamPartition("kafka", "PVE", new Partition(4)))).put(new TaskName("Partition 3"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(7)), new SystemStreamPartition("kafka", "PVE", new Partition(3)))).put(new TaskName("Partition 2"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(2)), new SystemStreamPartition("kafka", "PVE", new Partition(6)))).build();
    Map<TaskName, Set<SystemStreamPartition>> expectedGroupingForStateless = ImmutableMap.<TaskName, Set<SystemStreamPartition>>builder().put(new TaskName("Partition 0"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(0)))).put(new TaskName("Partition 1"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(1)))).put(new TaskName("Partition 2"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(2)))).put(new TaskName("Partition 3"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(3)))).put(new TaskName("Partition 4"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(4)))).put(new TaskName("Partition 5"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(5)))).put(new TaskName("Partition 6"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(6)))).put(new TaskName("Partition 7"), ImmutableSet.of(new SystemStreamPartition("kafka", "PVE", new Partition(7)))).build();
    // SSPGrouperProxy for stateful job
    SSPGrouperProxy groupByPartition = buildSspGrouperProxy(true);
    GrouperMetadata grouperMetadata = new GrouperMetadataImpl(new HashMap<>(), new HashMap<>(), prevGroupingWithSingleStream, new HashMap<>());
    Map<TaskName, Set<SystemStreamPartition>> finalGrouping = groupByPartition.group(currSsps, grouperMetadata);
    Assert.assertEquals(expectedGroupingForStateful, finalGrouping);
    // SSPGrouperProxy for stateless job
    groupByPartition = buildSspGrouperProxy(false);
    finalGrouping = groupByPartition.group(currSsps, grouperMetadata);
    Assert.assertEquals(expectedGroupingForStateless, finalGrouping);
}

Example 7 with GrouperMetadata

use of org.apache.samza.container.grouper.task.GrouperMetadata in project samza by apache.

the class PassthroughJobCoordinator method getJobModel.

@Override
public JobModel getJobModel() {
    SystemAdmins systemAdmins = new SystemAdmins(config, this.getClass().getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, SystemClock.instance());
    systemAdmins.start();
    try {
        String containerId = Integer.toString(config.getInt(JobConfig.PROCESSOR_ID));
        GrouperMetadata grouperMetadata = new GrouperMetadataImpl(ImmutableMap.of(String.valueOf(containerId), locationId), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
        return JobModelCalculator.INSTANCE.calculateJobModel(this.config, Collections.emptyMap(), streamMetadataCache, grouperMetadata);
    } finally {
        systemAdmins.stop();
    }
}

Example 8 with GrouperMetadata

use of org.apache.samza.container.grouper.task.GrouperMetadata in project samza by apache.

the class TestJobModelHelper method verifyGrouperMetadata.

private void verifyGrouperMetadata(Map<String, LocationId> processorLocality, Map<TaskName, LocationId> taskLocality, Map<TaskName, List<SystemStreamPartition>> previousTaskToSSPAssignment, Map<TaskName, String> previousTaskToProcessorAssignment) {
    GrouperMetadata grouperMetadata = this.grouperMetadataArgumentCaptor.getValue();
    assertEquals(processorLocality, grouperMetadata.getProcessorLocality());
    assertEquals(taskLocality, grouperMetadata.getTaskLocality());
    assertPreviousTaskToSSPAssignment(previousTaskToSSPAssignment, grouperMetadata.getPreviousTaskToSSPAssignment());
    assertEquals(previousTaskToProcessorAssignment, grouperMetadata.getPreviousTaskToProcessorAssignment());
}

Example 9 with GrouperMetadata

use of org.apache.samza.container.grouper.task.GrouperMetadata in project samza by apache.

the class JobModelHelper method updateTaskAssignments.

/**
 * This method does the following:
 * 1. Deletes the existing task assignments if the partition-task grouping has changed from the previous run of the job.
 * 2. Saves the newly generated task assignments to the storage layer through the {@param TaskAssignementManager}.
 *
 * @param jobModel              represents the {@see JobModel} of the samza job.
 * @param taskAssignmentManager required to persist the processor to task assignments to the metadata store.
 * @param taskPartitionAssignmentManager required to persist the task to partition assignments to the metadata store.
 * @param grouperMetadata       provides the historical metadata of the samza application.
 */
private void updateTaskAssignments(JobModel jobModel, TaskAssignmentManager taskAssignmentManager, TaskPartitionAssignmentManager taskPartitionAssignmentManager, GrouperMetadata grouperMetadata) {
    LOG.info("Storing the task assignments into metadata store.");
    Set<String> activeTaskNames = new HashSet<>();
    Set<String> standbyTaskNames = new HashSet<>();
    Set<SystemStreamPartition> systemStreamPartitions = new HashSet<>();
    for (ContainerModel containerModel : jobModel.getContainers().values()) {
        for (TaskModel taskModel : containerModel.getTasks().values()) {
            if (TaskMode.Active.equals(taskModel.getTaskMode())) {
                activeTaskNames.add(taskModel.getTaskName().getTaskName());
            }
            if (TaskMode.Standby.equals(taskModel.getTaskMode())) {
                standbyTaskNames.add(taskModel.getTaskName().getTaskName());
            }
            systemStreamPartitions.addAll(taskModel.getSystemStreamPartitions());
        }
    }
    Map<TaskName, String> previousTaskToContainerId = grouperMetadata.getPreviousTaskToProcessorAssignment();
    if (activeTaskNames.size() != previousTaskToContainerId.size()) {
        LOG.warn(String.format("Current task count %s does not match saved task count %s. Stateful jobs may observe misalignment of keys!", activeTaskNames.size(), previousTaskToContainerId.size()));
        // If the tasks changed, then the partition-task grouping is also likely changed and we can't handle that
        // without a much more complicated mapping. Further, the partition count may have changed, which means
        // input message keys are likely reshuffled w.r.t. partitions, so the local state may not contain necessary
        // data associated with the incoming keys. Warn the user and default to grouper
        // In this scenario the tasks may have been reduced, so we need to delete all the existing messages
        taskAssignmentManager.deleteTaskContainerMappings(previousTaskToContainerId.keySet().stream().map(TaskName::getTaskName).collect(Collectors.toList()));
        taskPartitionAssignmentManager.delete(systemStreamPartitions);
    }
    // if the set of standby tasks has changed, e.g., when the replication-factor changed, or the active-tasks-set has
    // changed, we log a warning and delete the existing mapping for these tasks
    Set<String> previousStandbyTasks = taskAssignmentManager.readTaskModes().entrySet().stream().filter(taskNameToTaskModeEntry -> TaskMode.Standby.equals(taskNameToTaskModeEntry.getValue())).map(taskNameToTaskModeEntry -> taskNameToTaskModeEntry.getKey().getTaskName()).collect(Collectors.toSet());
    if (!standbyTaskNames.equals(previousStandbyTasks)) {
        LOG.info(String.format("The set of standby tasks has changed, current standby tasks %s, previous standby tasks %s", standbyTaskNames, previousStandbyTasks));
        taskAssignmentManager.deleteTaskContainerMappings(previousStandbyTasks);
    }
    // Task to partition assignments is stored as {@see SystemStreamPartition} to list of {@see TaskName} in
    // coordinator stream. This is done due to the 1 MB value size limit in a kafka topic.
    Map<SystemStreamPartition, List<String>> sspToTaskNameMap = new HashMap<>();
    Map<String, Map<String, TaskMode>> taskContainerMappings = new HashMap<>();
    for (ContainerModel containerModel : jobModel.getContainers().values()) {
        containerModel.getTasks().forEach((taskName, taskModel) -> {
            taskContainerMappings.putIfAbsent(containerModel.getId(), new HashMap<>());
            taskContainerMappings.get(containerModel.getId()).put(taskName.getTaskName(), taskModel.getTaskMode());
            taskModel.getSystemStreamPartitions().forEach(systemStreamPartition -> {
                sspToTaskNameMap.putIfAbsent(systemStreamPartition, new ArrayList<>());
                sspToTaskNameMap.get(systemStreamPartition).add(taskName.getTaskName());
            });
        });
    }
    taskAssignmentManager.writeTaskContainerMappings(taskContainerMappings);
    taskPartitionAssignmentManager.writeTaskPartitionAssignments(sspToTaskNameMap);
}

Example 10 with GrouperMetadata

use of org.apache.samza.container.grouper.task.GrouperMetadata in project samza by apache.

the class AzureJobCoordinator method doOnProcessorChange.

/**
 * Called only by the leader, either when the processor becomes the leader, or when the list of live processors changes.
 * @param currentProcessorIds New updated list of processor IDs which caused the rebalancing.
 */
private void doOnProcessorChange(List<String> currentProcessorIds) {
    // if list of processors is empty - it means we are called from 'onBecomeLeader'
    // Check if number of processors is greater than number of tasks
    List<String> initialProcessorIds = new ArrayList<>(currentProcessorIds);
    int numTasks = getMaxNumTasks();
    if (currentProcessorIds.size() > numTasks) {
        int iterator = 0;
        while (currentProcessorIds.size() != numTasks) {
            if (!currentProcessorIds.get(iterator).equals(processorId)) {
                currentProcessorIds.remove(iterator);
                iterator++;
            }
        }
    }
    LOG.info("currentProcessorIds = {}", currentProcessorIds);
    LOG.info("initialProcessorIds = {}", initialProcessorIds);
    String nextJMVersion;
    String prevJMVersion = currentJMVersion.get();
    JobModel prevJobModel = jobModel;
    AtomicBoolean barrierTimeout = new AtomicBoolean(false);
    if (currentProcessorIds.isEmpty()) {
        if (currentJMVersion.get().equals(INITIAL_STATE)) {
            nextJMVersion = "1";
        } else {
            nextJMVersion = Integer.toString(Integer.valueOf(prevJMVersion) + 1);
        }
        currentProcessorIds = new ArrayList<>(table.getActiveProcessorsList(currentJMVersion));
        initialProcessorIds = currentProcessorIds;
    } else {
        // Check if previous barrier not reached, then previous barrier times out.
        String blobJMV = leaderBlob.getJobModelVersion();
        nextJMVersion = Integer.toString(Integer.valueOf(prevJMVersion) + 1);
        if (blobJMV != null && Integer.valueOf(blobJMV) > Integer.valueOf(prevJMVersion)) {
            prevJMVersion = blobJMV;
            prevJobModel = leaderBlob.getJobModel();
            nextJMVersion = Integer.toString(Integer.valueOf(blobJMV) + 1);
            versionUpgradeDetected.getAndSet(false);
            leaderBarrierScheduler.shutdown();
            leaderBlob.publishBarrierState(BarrierState.TIMEOUT.name() + " " + blobJMV, azureLeaderElector.getLeaseId().get());
        }
    }
    // Generate the new JobModel
    GrouperMetadata grouperMetadata = new GrouperMetadataImpl(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    JobModel newJobModel = JobModelCalculator.INSTANCE.calculateJobModel(this.config, Collections.emptyMap(), streamMetadataCache, grouperMetadata);
    LOG.info("pid=" + processorId + "Generated new Job Model. Version = " + nextJMVersion);
    // Publish the new job model
    boolean jmWrite = leaderBlob.publishJobModel(prevJobModel, newJobModel, prevJMVersion, nextJMVersion, azureLeaderElector.getLeaseId().get());
    // Publish barrier state
    boolean barrierWrite = leaderBlob.publishBarrierState(BarrierState.START.name() + " " + nextJMVersion, azureLeaderElector.getLeaseId().get());
    barrierTimeout.set(false);
    // Publish list of processors this function was called with
    boolean processorWrite = leaderBlob.publishLiveProcessorList(initialProcessorIds, azureLeaderElector.getLeaseId().get());
    // Shut down processor if write fails even after retries. These writes have an inherent retry policy.
    if (!jmWrite || !barrierWrite || !processorWrite) {
        LOG.info("Leader failed to publish the job model {}. Stopping the processor with PID: .", jobModel, processorId);
        stop();
    }
    LOG.info("pid=" + processorId + "Published new Job Model. Version = " + nextJMVersion);
    // Start scheduler to check if barrier reached
    long startTime = System.currentTimeMillis();
    leaderBarrierScheduler = new LeaderBarrierCompleteScheduler(errorHandler, table, nextJMVersion, initialProcessorIds, startTime, barrierTimeout, currentJMVersion, processorId);
    leaderBarrierScheduler.setStateChangeListener(createLeaderBarrierCompleteListener(nextJMVersion, barrierTimeout));
    leaderBarrierScheduler.scheduleTask();
}