Examples with StealerBasedRebalanceTask voldemort.client.rebalance.task.StealerBasedRebalanceTask used on opensource projects

Example 1 with StealerBasedRebalanceTask

use of voldemort.client.rebalance.task.StealerBasedRebalanceTask in project voldemort by voldemort.

the class RebalanceScheduler method populateTasksByStealer.

/**
     * Go over the task list and create a map of stealerId -> Tasks
     * 
     * @param sbTaskList List of all stealer-based rebalancing tasks to be
     *        scheduled.
     */
protected void populateTasksByStealer(List<StealerBasedRebalanceTask> sbTaskList) {
    // Setup mapping of stealers to work for this run.
    for (StealerBasedRebalanceTask task : sbTaskList) {
        if (task.getStealInfos().size() != 1) {
            throw new VoldemortException("StealerBasedRebalanceTasks should have a list of RebalancePartitionsInfo of length 1.");
        }
        RebalanceTaskInfo stealInfo = task.getStealInfos().get(0);
        int stealerId = stealInfo.getStealerId();
        if (!this.tasksByStealer.containsKey(stealerId)) {
            this.tasksByStealer.put(stealerId, new ArrayList<StealerBasedRebalanceTask>());
        }
        this.tasksByStealer.get(stealerId).add(task);
    }
    if (tasksByStealer.isEmpty()) {
        return;
    }
    // added to the task list passed in.
    for (List<StealerBasedRebalanceTask> taskList : tasksByStealer.values()) {
        Collections.shuffle(taskList);
    }
}

Example 2 with StealerBasedRebalanceTask

use of voldemort.client.rebalance.task.StealerBasedRebalanceTask in project voldemort by voldemort.

the class RebalanceScheduler method scheduleNextTask.

/**
     * Schedule at most one task.
     * 
     * The scheduled task *must* invoke 'doneTask()' upon
     * completion/termination.
     * 
     * @param executeService flag to control execution of the service, some tests pass
     *        in value 'false'
     * @return The task scheduled or null if not possible to schedule a task at
     *         this time.
     */
protected synchronized StealerBasedRebalanceTask scheduleNextTask(boolean executeService) {
    // Make sure there is work left to do.
    if (doneSignal.getCount() == 0) {
        logger.info("All tasks completion signaled... returning");
        return null;
    }
    // Limit number of tasks outstanding.
    if (this.numTasksExecuting >= maxParallelRebalancing) {
        logger.info("Executing more tasks than [" + this.numTasksExecuting + "] the parallel allowed " + maxParallelRebalancing);
        return null;
    }
    // Shuffle list of stealer IDs each time a new task to schedule needs to
    // be found. Randomizing the order should avoid prioritizing one
    // specific stealer's work ahead of all others.
    List<Integer> stealerIds = new ArrayList<Integer>(tasksByStealer.keySet());
    Collections.shuffle(stealerIds);
    for (int stealerId : stealerIds) {
        if (nodeIdsWithWork.contains(stealerId)) {
            logger.info("Stealer " + stealerId + " is already working... continuing");
            continue;
        }
        for (StealerBasedRebalanceTask sbTask : tasksByStealer.get(stealerId)) {
            int donorId = sbTask.getStealInfos().get(0).getDonorId();
            if (nodeIdsWithWork.contains(donorId)) {
                logger.info("Stealer " + stealerId + " Donor " + donorId + " is already working... continuing");
                continue;
            }
            // Book keeping
            addNodesToWorkerList(Arrays.asList(stealerId, donorId));
            numTasksExecuting++;
            // Remove this task from list thus destroying list being
            // iterated over. This is safe because returning directly out of
            // this branch.
            tasksByStealer.get(stealerId).remove(sbTask);
            try {
                if (executeService) {
                    logger.info("Stealer " + stealerId + " Donor " + donorId + " going to schedule work");
                    service.execute(sbTask);
                }
            } catch (RejectedExecutionException ree) {
                logger.error("Stealer " + stealerId + "Rebalancing task rejected by executor service.", ree);
                throw new VoldemortRebalancingException("Stealer " + stealerId + "Rebalancing task rejected by executor service.");
            }
            return sbTask;
        }
    }
    printRemainingTasks(stealerIds);
    return null;
}

Example 3 with StealerBasedRebalanceTask

use of voldemort.client.rebalance.task.StealerBasedRebalanceTask in project voldemort by voldemort.

the class RebalanceSchedulerTest method test.

@Test
public void test() {
    RebalanceScheduler mockedScheduler = Mockito.spy(scheduler);
    Cluster zzCurrent = ClusterTestUtils.getZZCluster();
    adminClient = ServerTestUtils.getAdminClient(zzCurrent);
    Map<String, List<Integer>> outerMap = new HashMap<String, List<Integer>>();
    List<Integer> someList = Arrays.asList(0, 1, 2);
    outerMap.put("storeA", someList);
    int stealerId = 0;
    int donorId = 1;
    RebalanceTaskInfo partitionsInfo = new RebalanceTaskInfo(stealerId, donorId, (HashMap<String, List<Integer>>) outerMap, zzCurrent);
    StealerBasedRebalanceTask sbTask = new StealerBasedRebalanceTask(0, 0, partitionsInfo, donorPermit, adminClient, progressBar, mockedScheduler);
    sbTaskList.add(sbTask);
    sbTaskList.add(sbTask);
    sbTaskList.add(sbTask);
    sbTaskList.add(sbTask);
    sbTaskList.add(sbTask);
    mockedScheduler.initializeLatch(sbTaskList.size());
    mockedScheduler.populateTasksByStealer(sbTaskList);
    // In the beginning both stealer and donor are idle so scheduler should
    // return the scheduled
    // task
    StealerBasedRebalanceTask scheduledTask = mockedScheduler.scheduleNextTask(false);
    org.junit.Assert.assertNotNull(sbTask);
    org.junit.Assert.assertEquals(sbTask, scheduledTask);
    mockedScheduler.removeNodesFromWorkerList(Arrays.asList(stealerId, donorId));
    // Now lets remove the donor from the worker list so that the donor is
    // idle and add the
    // stealer to the worker list. The scheduler should return null as it
    // won't be able to
    // schedule the task.
    mockedScheduler.addNodesToWorkerList(Arrays.asList(stealerId));
    mockedScheduler.removeNodesFromWorkerList(Arrays.asList(donorId));
    org.junit.Assert.assertEquals(mockedScheduler.scheduleNextTask(false), null);
    // This time stealer doesn't have any work but donor does
    mockedScheduler.addNodesToWorkerList(Arrays.asList(donorId));
    mockedScheduler.removeNodesFromWorkerList(Arrays.asList(stealerId));
    org.junit.Assert.assertEquals(mockedScheduler.scheduleNextTask(false), null);
    // And now both stealer and donor have work to do
    mockedScheduler.addNodesToWorkerList(Arrays.asList(donorId));
    mockedScheduler.addNodesToWorkerList(Arrays.asList(stealerId));
    org.junit.Assert.assertEquals(mockedScheduler.scheduleNextTask(false), null);
    // And stealer and donor are idle again
    mockedScheduler.removeNodesFromWorkerList(Arrays.asList(stealerId, donorId));
    StealerBasedRebalanceTask nextscheduledTask = mockedScheduler.scheduleNextTask(false);
    org.junit.Assert.assertNotNull(sbTask);
    org.junit.Assert.assertEquals(sbTask, nextscheduledTask);
}

Example 4 with StealerBasedRebalanceTask

use of voldemort.client.rebalance.task.StealerBasedRebalanceTask in project voldemort by voldemort.

the class RebalanceController method executeTasks.

private List<RebalanceTask> executeTasks(final int batchId, RebalanceBatchPlanProgressBar progressBar, final ExecutorService service, List<RebalanceTaskInfo> rebalanceTaskPlanList, Map<Integer, Semaphore> donorPermits) {
    List<RebalanceTask> taskList = Lists.newArrayList();
    int taskId = 0;
    RebalanceScheduler scheduler = new RebalanceScheduler(service, maxParallelRebalancing);
    List<StealerBasedRebalanceTask> sbTaskList = Lists.newArrayList();
    for (RebalanceTaskInfo taskInfo : rebalanceTaskPlanList) {
        StealerBasedRebalanceTask rebalanceTask = new StealerBasedRebalanceTask(batchId, taskId, taskInfo, donorPermits.get(taskInfo.getDonorId()), adminClient, progressBar, scheduler);
        taskList.add(rebalanceTask);
        sbTaskList.add(rebalanceTask);
        // service.execute(rebalanceTask);
        taskId++;
    }
    scheduler.run(sbTaskList);
    return taskList;
}

Example 5 with StealerBasedRebalanceTask

use of voldemort.client.rebalance.task.StealerBasedRebalanceTask in project voldemort by voldemort.

the class RebalanceScheduler method printRemainingTasks.

private void printRemainingTasks(List<Integer> stealerIds) {
    for (int stealerId : stealerIds) {
        List<Integer> donorIds = new ArrayList<Integer>();
        for (StealerBasedRebalanceTask sbTask : tasksByStealer.get(stealerId)) {
            int donorId = sbTask.getStealInfos().get(0).getDonorId();
            donorIds.add(donorId);
        }
        logger.info(" Remaining work for Stealer " + stealerId + " Donors : " + donorIds);
    }
}