Examples with Group edu.snu.mist.core.task.groupaware.Group used on opensource projects

Example 1 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupMergerImpl method groupMerging.

@Override
public void groupMerging() {
    LOG.info("MERGING START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final List<EventProcessor> underloadedThreads = new LinkedList<>();
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    final double groupLoad = highLoadGroup.getLoad();
                    if (highLoadGroup.isSplited()) {
                        // Merge splitted group!
                        // 1. find the thread that has the lowest load among threads that hold the splitted groups
                        final Group lowLoadGroup = findLowestLoadThreadSplittedGroup(highLoadGroup);
                        // 2. Check we can move the high load group to the low load thread group
                        if (lowLoadGroup != highLoadGroup && highLoadGroup.getEventProcessor().getLoad() - groupLoad >= targetLoad && lowLoadGroup.getEventProcessor().getLoad() + groupLoad <= targetLoad) {
                            // 3. merge!
                            if (merge(highLoadGroup, highLoadGroups, highLoadThread, lowLoadGroup)) {
                                rebNum += 1;
                            }
                            // Prevent lots of groups from being reassigned
                            if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                                break;
                            }
                        }
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "Merging number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 2 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupMergerImpl method findLowestLoadThreadSplittedGroup.

private Group findLowestLoadThreadSplittedGroup(final Group highLoadGroup) {
    Group g = null;
    double threadLoad = Double.MAX_VALUE;
    for (final Group hg : highLoadGroup.getApplicationInfo().getGroups()) {
        if (hg.getEventProcessor().getLoad() < threadLoad) {
            g = hg;
            threadLoad = hg.getEventProcessor().getLoad();
        }
    }
    return g;
}

Example 3 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupRebalancerImpl method logging.

private void logging(final List<EventProcessor> eventProcessors, final Map<EventProcessor, Double> loadTable) {
    final StringBuilder sb = new StringBuilder();
    sb.append("-------------- TABLE ----------------\n");
    for (final EventProcessor ep : eventProcessors) {
        final Collection<Group> groups = groupAllocationTable.getValue(ep);
        sb.append(ep);
        sb.append(" -> [");
        sb.append(loadTable.get(ep));
        sb.append("], [");
        sb.append(groups.size());
        sb.append("], ");
        sb.append(groups);
        sb.append("\n");
    }
    LOG.info(sb.toString());
}

Example 4 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupSplitterImpl method splitGroup.

@Override
public void splitGroup() {
    LOG.info("GROUP SPLIT START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final PriorityQueue<EventProcessor> underloadedThreads = new PriorityQueue<>(new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                final Double load1 = o1.getLoad();
                final Double load2 = o2.getLoad();
                return load1.compareTo(load2);
            }
        });
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        // LOGGING
        // logging(eventProcessors, loadTable);
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    // Split if the load of the high load thread could be less than targetLoad
                    // when we split the high load group
                    int n = 0;
                    if (highLoadThread.getLoad() - highLoadGroup.getLoad() < targetLoad + epsilon && highLoadGroup.size() > 1) {
                        // Sorting queries
                        final List<Query> queries = highLoadGroup.getQueries();
                        final List<Query> sortedQueries = new ArrayList<>(queries);
                        sortedQueries.sort(new Comparator<Query>() {

                            @Override
                            public int compare(final Query o1, final Query o2) {
                                if (o1.getLoad() < o2.getLoad()) {
                                    return 1;
                                } else if (o1.getLoad() > o2.getLoad()) {
                                    return -1;
                                } else {
                                    return 0;
                                }
                            }
                        });
                        final EventProcessor lowLoadThread = underloadedThreads.poll();
                        Group sameGroup = hasGroupOfSameApp(highLoadGroup, lowLoadThread);
                        if (sameGroup == null) {
                            // Split! Create a new group!
                            final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
                            jcb.bindNamedParameter(GroupId.class, groupIdRequestor.requestGroupId(highLoadGroup.getApplicationInfo().getApplicationId()));
                            final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
                            // TODO[MIST-1096]: We should inject executionDags, configVertexMap ... for creating a new group.
                            sameGroup = injector.getInstance(Group.class);
                            sameGroup.setEventProcessor(lowLoadThread);
                            highLoadGroup.getApplicationInfo().addGroup(sameGroup);
                            groupAllocationTable.getValue(lowLoadThread).add(sameGroup);
                            groupMap.putIfAbsent(sameGroup.getGroupId(), sameGroup);
                        }
                        for (final Query movingQuery : sortedQueries) {
                            if (highLoadThread.getLoad() - movingQuery.getLoad() >= targetLoad - epsilon && lowLoadThread.getLoad() + movingQuery.getLoad() <= targetLoad + epsilon) {
                                // Move to the existing group!
                                sameGroup.addQuery(movingQuery);
                                sameGroup.setLoad(sameGroup.getLoad() + movingQuery.getLoad());
                                highLoadGroup.delete(movingQuery);
                                highLoadGroup.setLoad(highLoadGroup.getLoad() - movingQuery.getLoad());
                                lowLoadThread.setLoad(lowLoadThread.getLoad() + movingQuery.getLoad());
                                highLoadThread.setLoad(highLoadThread.getLoad() - movingQuery.getLoad());
                                rebNum += 1;
                                n += 1;
                            }
                        }
                        // Prevent lots of groups from being reassigned
                        if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                            break;
                        }
                        underloadedThreads.add(lowLoadThread);
                        LOG.log(Level.INFO, "GroupSplit from: {0} to {1}, Splitted Group: {3}, number: {2}", new Object[] { highLoadThread, lowLoadThread, n, highLoadGroup.toString() });
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "GroupSplit number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 5 with Group

use of edu.snu.mist.core.task.groupaware.Group in project mist by snuspl.

the class DefaultGroupSplitterImpl method logging.

private void logging(final List<EventProcessor> eventProcessors, final Map<EventProcessor, Double> loadTable) {
    final StringBuilder sb = new StringBuilder();
    sb.append("-------------- TABLE ----------------\n");
    for (final EventProcessor ep : eventProcessors) {
        final Collection<Group> groups = groupAllocationTable.getValue(ep);
        sb.append(ep);
        sb.append(" -> [");
        sb.append(loadTable.get(ep));
        sb.append("], [");
        sb.append(groups.size());
        sb.append("], ");
        sb.append(groups);
        sb.append("\n");
    }
    LOG.info(sb.toString());
}