Examples with EventProcessor edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor used on opensource projects

Example 1 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class DefaultGroupImpl method processAllEvent.

@Override
public int processAllEvent(final long timeout) {
    int numProcessedEvent = 0;
    Query query = activeQueryQueue.poll();
    final long startTime = System.currentTimeMillis();
    while (query != null) {
        numActiveSubGroup.decrementAndGet();
        if (query.setProcessingFromReady()) {
            final int processedEvent = query.processAllEvent();
            if (processedEvent != 0) {
                query.getProcessingEvent().getAndAdd(processedEvent);
            }
            numProcessedEvent += processedEvent;
            query.setReady();
        }
        // Reschedule this group if it still has events to process
        if (elapsedTime(startTime) > timeout) {
            final EventProcessor ep = eventProcessor.get();
            // This could be null when the group merger merges the group
            if (ep != null) {
                ep.addActiveGroup(this);
            }
            break;
        }
        query = activeQueryQueue.poll();
    }
    return numProcessedEvent;
}

Example 2 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class DefaultGroupMergerImpl method groupMerging.

@Override
public void groupMerging() {
    LOG.info("MERGING START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final List<EventProcessor> underloadedThreads = new LinkedList<>();
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    final double groupLoad = highLoadGroup.getLoad();
                    if (highLoadGroup.isSplited()) {
                        // Merge splitted group!
                        // 1. find the thread that has the lowest load among threads that hold the splitted groups
                        final Group lowLoadGroup = findLowestLoadThreadSplittedGroup(highLoadGroup);
                        // 2. Check we can move the high load group to the low load thread group
                        if (lowLoadGroup != highLoadGroup && highLoadGroup.getEventProcessor().getLoad() - groupLoad >= targetLoad && lowLoadGroup.getEventProcessor().getLoad() + groupLoad <= targetLoad) {
                            // 3. merge!
                            if (merge(highLoadGroup, highLoadGroups, highLoadThread, lowLoadGroup)) {
                                rebNum += 1;
                            }
                            // Prevent lots of groups from being reassigned
                            if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                                break;
                            }
                        }
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "Merging number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 3 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class DefaultGroupSplitterImpl method splitGroup.

@Override
public void splitGroup() {
    LOG.info("GROUP SPLIT START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final PriorityQueue<EventProcessor> underloadedThreads = new PriorityQueue<>(new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                final Double load1 = o1.getLoad();
                final Double load2 = o2.getLoad();
                return load1.compareTo(load2);
            }
        });
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        // LOGGING
        // logging(eventProcessors, loadTable);
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    // Split if the load of the high load thread could be less than targetLoad
                    // when we split the high load group
                    int n = 0;
                    if (highLoadThread.getLoad() - highLoadGroup.getLoad() < targetLoad + epsilon) {
                        final List<Query> queries = highLoadGroup.getQueries();
                        final List<Query> possibleMovingQueries = new ArrayList<>(queries);
                        final EventProcessor lowLoadThread = underloadedThreads.poll();
                        final List<Query> movingQueries = new LinkedList<>();
                        // Move at most half of the queries in the group
                        double lowLoadThreadLoad = lowLoadThread.getLoad();
                        double highLoadThreadLoad = highLoadThread.getLoad();
                        for (final Query possibleMovingQuery : possibleMovingQueries) {
                            if (highLoadThreadLoad - possibleMovingQuery.getLoad() >= targetLoad - epsilon && lowLoadThreadLoad + possibleMovingQuery.getLoad() <= targetLoad + epsilon) {
                                if (movingQueries.size() > possibleMovingQueries.size() / 2) {
                                    break;
                                }
                                highLoadThreadLoad -= possibleMovingQuery.getLoad();
                                lowLoadThreadLoad += possibleMovingQuery.getLoad();
                                movingQueries.add(possibleMovingQuery);
                            }
                        }
                        if (movingQueries.size() > 1) {
                            Group sameGroup = hasGroupOfSameApp(highLoadGroup, lowLoadThread);
                            if (sameGroup == null) {
                                // Split! Create a new group!
                                final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
                                jcb.bindNamedParameter(GroupId.class, groupIdRequestor.requestGroupId(highLoadGroup.getApplicationInfo().getApplicationId()));
                                final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
                                injector.bindVolatileInstance(ExecutionDags.class, highLoadGroup.getExecutionDags());
                                injector.bindVolatileInstance(QueryIdConfigDagMap.class, highLoadGroup.getQueryIdConfigDagMap());
                                injector.bindVolatileInstance(ConfigExecutionVertexMap.class, highLoadGroup.getConfigExecutionVertexMap());
                                sameGroup = injector.getInstance(Group.class);
                                sameGroup.setEventProcessor(lowLoadThread);
                                highLoadGroup.getApplicationInfo().addGroup(sameGroup);
                                groupAllocationTable.getValue(lowLoadThread).add(sameGroup);
                                groupMap.putIfAbsent(sameGroup.getGroupId(), sameGroup);
                            }
                            for (final Query movingQuery : movingQueries) {
                                // Move to the existing group!
                                sameGroup.addQuery(movingQuery);
                                sameGroup.setLoad(sameGroup.getLoad() + movingQuery.getLoad());
                                highLoadGroup.delete(movingQuery);
                                highLoadGroup.setLoad(highLoadGroup.getLoad() - movingQuery.getLoad());
                                lowLoadThread.setLoad(lowLoadThread.getLoad() + movingQuery.getLoad());
                                highLoadThread.setLoad(highLoadThread.getLoad() - movingQuery.getLoad());
                                n += 1;
                            }
                            sameGroup.getEventProcessor().addActiveGroup(sameGroup);
                            highLoadGroup.getEventProcessor().addActiveGroup(highLoadGroup);
                            rebNum += 1;
                            LOG.log(Level.INFO, "GroupSplit from: {0} to {1}, Splitted Group: {3}, number: {2}", new Object[] { highLoadThread, lowLoadThread, n, highLoadGroup.toString() });
                        }
                        // Prevent lots of groups from being reassigned
                        if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                            break;
                        }
                        underloadedThreads.add(lowLoadThread);
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "GroupSplit number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 4 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class DefaultGroupSplitterImpl method logging.

private void logging(final List<EventProcessor> eventProcessors, final Map<EventProcessor, Double> loadTable) {
    final StringBuilder sb = new StringBuilder();
    sb.append("-------------- TABLE ----------------\n");
    for (final EventProcessor ep : eventProcessors) {
        final Collection<Group> groups = groupAllocationTable.getValue(ep);
        sb.append(ep);
        sb.append(" -> [");
        sb.append(loadTable.get(ep));
        sb.append("], [");
        sb.append(groups.size());
        sb.append("], ");
        sb.append(groups);
        sb.append("\n");
    }
    LOG.info(sb.toString());
}

Example 5 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class MinLoadGroupAssignerImpl method assignGroup.

/**
 * Assign the new group to the event processor.
 * @param groupInfo new group
 */
@Override
public void assignGroup(final Group groupInfo) {
    final List<EventProcessor> uThreads = underloadedThreads();
    if (uThreads.size() > 0) {
        final int index = random.nextInt(uThreads.size());
        final EventProcessor eventProcessor = uThreads.get(index);
        groupAllocationTable.getValue(eventProcessor).add(groupInfo);
        groupInfo.setEventProcessor(eventProcessor);
        eventProcessor.setLoad(eventProcessor.getLoad() + groupInfo.getLoad());
    } else {
        // Reselect the event processor that has the minimum
        final EventProcessor minLoadEventProcessor = findMinLoadEventProcessor();
        groupAllocationTable.getValue(minLoadEventProcessor).add(groupInfo);
        groupInfo.setEventProcessor(minLoadEventProcessor);
        minLoadEventProcessor.setLoad(minLoadEventProcessor.getLoad() + groupInfo.getLoad());
    }
}