Examples with Query edu.snu.mist.core.task.Query used on opensource projects

Example 1 with Query

use of edu.snu.mist.core.task.Query in project mist by snuspl.

the class DefaultGroupSplitterImpl method splitGroup.

@Override
public void splitGroup() {
    LOG.info("GROUP SPLIT START");
    long rebalanceStart = System.currentTimeMillis();
    try {
        // Skip if it is an isolated processor that runs an isolated group
        final List<EventProcessor> eventProcessors = groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup();
        // Overloaded threads
        final List<EventProcessor> overloadedThreads = new LinkedList<>();
        // Underloaded threads
        final PriorityQueue<EventProcessor> underloadedThreads = new PriorityQueue<>(new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                final Double load1 = o1.getLoad();
                final Double load2 = o2.getLoad();
                return load1.compareTo(load2);
            }
        });
        // Calculate each load and total load
        for (final EventProcessor eventProcessor : eventProcessors) {
            final double load = eventProcessor.getLoad();
            if (load > beta) {
                overloadedThreads.add(eventProcessor);
            } else if (load < alpha) {
                underloadedThreads.add(eventProcessor);
            }
        }
        // LOGGING
        // logging(eventProcessors, loadTable);
        double targetLoad = (alpha + beta) / 2;
        int rebNum = 0;
        Collections.sort(overloadedThreads, new Comparator<EventProcessor>() {

            @Override
            public int compare(final EventProcessor o1, final EventProcessor o2) {
                if (o1.getLoad() < o2.getLoad()) {
                    return 1;
                } else if (o1.getLoad() > o2.getLoad()) {
                    return -1;
                } else {
                    return 0;
                }
            }
        });
        if (!overloadedThreads.isEmpty() && !underloadedThreads.isEmpty()) {
            for (final EventProcessor highLoadThread : overloadedThreads) {
                final Collection<Group> highLoadGroups = groupAllocationTable.getValue(highLoadThread);
                final List<Group> sortedHighLoadGroups = new LinkedList<>(highLoadGroups);
                Collections.sort(sortedHighLoadGroups, new Comparator<Group>() {

                    @Override
                    public int compare(final Group o1, final Group o2) {
                        if (o1.getLoad() < o2.getLoad()) {
                            return 1;
                        } else if (o1.getLoad() > o2.getLoad()) {
                            return -1;
                        } else {
                            return 0;
                        }
                    }
                });
                for (final Group highLoadGroup : sortedHighLoadGroups) {
                    // Split if the load of the high load thread could be less than targetLoad
                    // when we split the high load group
                    int n = 0;
                    if (highLoadThread.getLoad() - highLoadGroup.getLoad() < targetLoad + epsilon && highLoadGroup.size() > 1) {
                        // Sorting queries
                        final List<Query> queries = highLoadGroup.getQueries();
                        final List<Query> sortedQueries = new ArrayList<>(queries);
                        sortedQueries.sort(new Comparator<Query>() {

                            @Override
                            public int compare(final Query o1, final Query o2) {
                                if (o1.getLoad() < o2.getLoad()) {
                                    return 1;
                                } else if (o1.getLoad() > o2.getLoad()) {
                                    return -1;
                                } else {
                                    return 0;
                                }
                            }
                        });
                        final EventProcessor lowLoadThread = underloadedThreads.poll();
                        Group sameGroup = hasGroupOfSameApp(highLoadGroup, lowLoadThread);
                        if (sameGroup == null) {
                            // Split! Create a new group!
                            final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
                            jcb.bindNamedParameter(GroupId.class, groupIdRequestor.requestGroupId(highLoadGroup.getApplicationInfo().getApplicationId()));
                            final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
                            // TODO[MIST-1096]: We should inject executionDags, configVertexMap ... for creating a new group.
                            sameGroup = injector.getInstance(Group.class);
                            sameGroup.setEventProcessor(lowLoadThread);
                            highLoadGroup.getApplicationInfo().addGroup(sameGroup);
                            groupAllocationTable.getValue(lowLoadThread).add(sameGroup);
                            groupMap.putIfAbsent(sameGroup.getGroupId(), sameGroup);
                        }
                        for (final Query movingQuery : sortedQueries) {
                            if (highLoadThread.getLoad() - movingQuery.getLoad() >= targetLoad - epsilon && lowLoadThread.getLoad() + movingQuery.getLoad() <= targetLoad + epsilon) {
                                // Move to the existing group!
                                sameGroup.addQuery(movingQuery);
                                sameGroup.setLoad(sameGroup.getLoad() + movingQuery.getLoad());
                                highLoadGroup.delete(movingQuery);
                                highLoadGroup.setLoad(highLoadGroup.getLoad() - movingQuery.getLoad());
                                lowLoadThread.setLoad(lowLoadThread.getLoad() + movingQuery.getLoad());
                                highLoadThread.setLoad(highLoadThread.getLoad() - movingQuery.getLoad());
                                rebNum += 1;
                                n += 1;
                            }
                        }
                        // Prevent lots of groups from being reassigned
                        if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                            break;
                        }
                        underloadedThreads.add(lowLoadThread);
                        LOG.log(Level.INFO, "GroupSplit from: {0} to {1}, Splitted Group: {3}, number: {2}", new Object[] { highLoadThread, lowLoadThread, n, highLoadGroup.toString() });
                    }
                }
                // Prevent lots of groups from being reassigned
                if (rebNum >= TimeUnit.MILLISECONDS.toSeconds(rebalancingPeriod)) {
                    break;
                }
            }
        }
        long rebalanceEnd = System.currentTimeMillis();
        LOG.log(Level.INFO, "GroupSplit number: {0}, elapsed time: {1}", new Object[] { rebNum, rebalanceEnd - rebalanceStart });
    // LOG.log(Level.INFO, "-------------TABLE-------------\n{0}",
    // new Object[]{groupAllocationTable.toString()});
    } catch (final Exception e) {
        LOG.log(Level.WARNING, "Exception " + e);
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 2 with Query

use of edu.snu.mist.core.task.Query in project mist by snuspl.

the class DefaultCheckpointManagerImpl method deleteGroup.

@Override
public void deleteGroup(final String groupId) {
    final Group group = groupMap.get(groupId);
    if (group == null) {
        LOG.log(Level.WARNING, "There is no such group {0}.", new Object[] { groupId });
        return;
    }
    final QueryRemover remover = group.getApplicationInfo().getQueryRemover();
    for (final Query query : group.getQueries()) {
        remover.deleteQuery(query.getId());
    }
    applicationMap.remove(groupId);
    groupAllocationTableModifier.addEvent(new WritingEvent(WritingEvent.EventType.GROUP_REMOVE_ALL, null));
}

Example 3 with Query

use of edu.snu.mist.core.task.Query in project mist by snuspl.

the class UtilizationLoadUpdater method updateGroupAndThreadLoad.

/**
 * Update the load of the event processor and the assigned groups.
 * @param eventProcessor event processor
 * @param groups assigned groups
 */
private void updateGroupAndThreadLoad(final EventProcessor eventProcessor, final Collection<Group> groups) {
    // boolean isOverloaded = false;
    double eventProcessorLoad = 0.0;
    final long elapsedTime = startTime - previousUpdateTime;
    final List<Group> skipGroups = new LinkedList<>();
    for (final Group group : groups) {
        double load = 0.0;
        final List<Query> queries = group.getQueries();
        final long processingEvent = group.getProcessingEvent().get();
        group.getProcessingEvent().addAndGet(-processingEvent);
        final long incomingEvent = processingEvent + group.numberOfRemainingEvents();
        final long processingEventTime = group.getProcessingTime().get();
        group.getProcessingTime().addAndGet(-processingEventTime);
        if (LOG.isLoggable(Level.FINE)) {
            LOG.log(Level.FINE, "Group {0}, ProcessingEvent: {1}, IncomingEvent: {2}, ProcessingTime: {3}", new Object[] { group.getGroupId(), processingEvent, incomingEvent, processingEventTime });
        }
        // Just use the previous load
        if (processingEventTime == 0 && incomingEvent != 0) {
            // isOverloaded = true;
            load = group.getLoad();
        } else if (incomingEvent == 0) {
            // No incoming event
            load = defaultGroupLoad;
        } else {
            // processed event, incoming event
            final double inputRate = (incomingEvent * 1000) / (double) elapsedTime;
            final double processingRate = (processingEvent * NS_UNIT) / (double) processingEventTime;
            if (processingEvent == 0 || processingRate == 0) {
                load = (1 * NS_UNIT) / (double) processingEventTime;
            } else {
                final double groupLoad = Math.min(1.5, inputRate / processingRate);
                load = groupLoad;
            }
            if (LOG.isLoggable(Level.FINE)) {
                LOG.log(Level.FINE, "Group {0}, InputRate: {1}, ProcessingRate: {2}, GroupLoad: {3}", new Object[] { group.getGroupId(), inputRate, processingRate, load });
            }
        }
        eventProcessorLoad += load;
        group.setLoad(load);
        // Calculate query load based on the group load!
        for (final Query query : queries) {
            // Number of processed events
            final long queryProcessingEvent = query.getProcessingEvent().get();
            query.getProcessingEvent().getAndAdd(-queryProcessingEvent);
            final long incomingE = query.numberOfRemainingEvents();
            final long queryIncomingEvent = incomingE + queryProcessingEvent;
            if (incomingEvent == 0) {
                query.setLoad(0);
            } else {
                query.setLoad(load * (queryIncomingEvent / (double) incomingEvent));
            }
        }
    }
    eventProcessor.setLoad(eventProcessorLoad);
// Overloaded!
/*
    if (isOverloaded) {
      eventProcessor.setLoad(1.0);
      // distribute the load
      final int size = groups.size();
      final double balancedLoad = 1.0 / size;

      for (final Group group : groups) {
        group.setLoad(balancedLoad);
        final int querySize = group.getQueries().size();
        final double queryLoad = balancedLoad / querySize;
        for (final Query query : group.getQueries()) {
          query.setLoad(queryLoad);
        }
      }
    } else {
      eventProcessor.setLoad(eventProcessorLoad);
    }
    */
}

Example 4 with Query

use of edu.snu.mist.core.task.Query in project mist by snuspl.

the class EventProcessorTest method eventProcessorProcessingTest.

@Test
public void eventProcessorProcessingTest() throws InjectionException, InterruptedException {
    final BlockingQueue<Group> queue = new LinkedBlockingQueue<>();
    final Group group1 = createGroup("group1");
    final Group group2 = createGroup("group2");
    final Group group3 = createGroup("group3");
    final CountDownLatch countDownLatch = new CountDownLatch(31);
    final AtomicInteger numEvent1 = new AtomicInteger(10);
    final SourceOutputEmitter oc1 = mock(SourceOutputEmitter.class);
    final AtomicInteger numEvent2 = new AtomicInteger(20);
    final SourceOutputEmitter oc2 = mock(SourceOutputEmitter.class);
    final AtomicInteger numEvent3 = new AtomicInteger(1);
    final SourceOutputEmitter oc3 = mock(SourceOutputEmitter.class);
    when(oc1.numberOfEvents()).thenReturn(numEvent1.get());
    when(oc1.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent1.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    when(oc2.numberOfEvents()).thenReturn(numEvent2.get());
    when(oc2.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent2.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    when(oc3.numberOfEvents()).thenReturn(numEvent3.get());
    when(oc3.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent3.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    final NextGroupSelector nextGroupSelector = new TestNextGroupSelector(queue);
    final EventProcessor eventProcessor = new DefaultEventProcessor(nextGroupSelector, 1, Long.MAX_VALUE);
    group1.setEventProcessor(eventProcessor);
    group2.setEventProcessor(eventProcessor);
    group3.setEventProcessor(eventProcessor);
    final Query query1 = new DefaultQueryImpl("q1");
    group1.addQuery(query1);
    final Query query2 = new DefaultQueryImpl("q2");
    group2.addQuery(query2);
    final Query query3 = new DefaultQueryImpl("q3");
    group3.addQuery(query3);
    query1.insert(oc1);
    query2.insert(oc2);
    query3.insert(oc3);
    eventProcessor.start();
    queue.add(group1);
    queue.add(group2);
    queue.add(group3);
    countDownLatch.await();
}

Example 5 with Query

use of edu.snu.mist.core.task.Query in project mist by snuspl.

the class DefaultGroupMergerImpl method merge.

private boolean merge(final Group highLoadGroup, final Collection<Group> highLoadGroups, final EventProcessor highLoadThread, final Group lowLoadGroup) {
    double incLoad = 0.0;
    synchronized (highLoadGroup.getQueries()) {
        for (final Query query : highLoadGroup.getQueries()) {
            lowLoadGroup.addQuery(query);
            incLoad += query.getLoad();
        }
    }
    // memory barrier
    synchronized (lowLoadGroup.getQueries()) {
        while (highLoadThread.removeActiveGroup(highLoadGroup)) {
        // remove all elements
        }
        highLoadGroups.remove(highLoadGroup);
        highLoadGroup.setEventProcessor(null);
    }
    synchronized (highLoadGroup.getApplicationInfo().getGroups()) {
        highLoadGroup.getApplicationInfo().getGroups().remove(highLoadGroup);
        highLoadGroup.getApplicationInfo().numGroups().decrementAndGet();
    }
    groupMap.remove(highLoadGroup.getGroupId());
    // Update overloaded thread load
    highLoadThread.setLoad(highLoadThread.getLoad() - incLoad);
    // Update underloaded thread load
    lowLoadGroup.setLoad(lowLoadGroup.getLoad() + incLoad);
    lowLoadGroup.getEventProcessor().setLoad(lowLoadGroup.getEventProcessor().getLoad() + incLoad);
    // Add one more
    lowLoadGroup.getEventProcessor().addActiveGroup(lowLoadGroup);
    LOG.log(Level.INFO, "Merge {0} from {1} to {2}", new Object[] { highLoadGroup, highLoadThread, lowLoadGroup.getEventProcessor() });
    return true;
}