Examples with EventProcessor edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor used on opensource projects

Example 6 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class MinLoadGroupAssignerImpl method findMinLoadEventProcessor.

/**
 * Find the event processor that has the minimum load.
 */
private EventProcessor findMinLoadEventProcessor() {
    EventProcessor minEventProcessor = null;
    double minLoad = Double.MAX_VALUE;
    for (final EventProcessor eventProcessor : groupAllocationTable.getEventProcessorsNotRunningIsolatedGroup()) {
        final double load = eventProcessor.getLoad();
        if (load < minLoad) {
            minEventProcessor = eventProcessor;
            minLoad = load;
        } else if (load == minLoad) {
            if (groupAllocationTable.getValue(eventProcessor).size() < groupAllocationTable.getValue(minEventProcessor).size()) {
                minEventProcessor = eventProcessor;
                minLoad = load;
            }
        }
    }
    return minEventProcessor;
}

Example 7 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class EventProcessorTest method eventProcessorProcessingTest.

@Test
public void eventProcessorProcessingTest() throws InjectionException, InterruptedException {
    final BlockingQueue<Group> queue = new LinkedBlockingQueue<>();
    final Group group1 = createGroup("group1");
    final Group group2 = createGroup("group2");
    final Group group3 = createGroup("group3");
    final CountDownLatch countDownLatch = new CountDownLatch(31);
    final AtomicInteger numEvent1 = new AtomicInteger(10);
    final SourceOutputEmitter oc1 = mock(SourceOutputEmitter.class);
    final AtomicInteger numEvent2 = new AtomicInteger(20);
    final SourceOutputEmitter oc2 = mock(SourceOutputEmitter.class);
    final AtomicInteger numEvent3 = new AtomicInteger(1);
    final SourceOutputEmitter oc3 = mock(SourceOutputEmitter.class);
    when(oc1.numberOfEvents()).thenReturn(numEvent1.get());
    when(oc1.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent1.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    when(oc2.numberOfEvents()).thenReturn(numEvent2.get());
    when(oc2.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent2.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    when(oc3.numberOfEvents()).thenReturn(numEvent3.get());
    when(oc3.processAllEvent()).thenAnswer((icm) -> {
        int cnt = 0;
        while (numEvent3.getAndDecrement() != 0) {
            Thread.sleep(10);
            countDownLatch.countDown();
            cnt += 1;
        }
        return cnt;
    });
    final NextGroupSelector nextGroupSelector = new TestNextGroupSelector(queue);
    final EventProcessor eventProcessor = new DefaultEventProcessor(nextGroupSelector, 1, Long.MAX_VALUE);
    group1.setEventProcessor(eventProcessor);
    group2.setEventProcessor(eventProcessor);
    group3.setEventProcessor(eventProcessor);
    final Query query1 = new DefaultQueryImpl("q1");
    group1.addQuery(query1);
    final Query query2 = new DefaultQueryImpl("q2");
    group2.addQuery(query2);
    final Query query3 = new DefaultQueryImpl("q3");
    group3.addQuery(query3);
    query1.insert(oc1);
    query2.insert(oc2);
    query3.insert(oc3);
    eventProcessor.start();
    queue.add(group1);
    queue.add(group2);
    queue.add(group3);
    countDownLatch.await();
}

Example 8 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class GroupRebalancerTest method defaultRebalancerTest1.

/**
 * Reassignment.
 * (alpha: 0.85, beta: 0.9, targetload = 0.875)
 * t1: [0.1, 0.05, 0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05] (0.95) overloaded
 * t2: [0.05, 0.05, 0.05, 0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05] (0.95) overloaded
 * t3: [0.1, 0.2, 0.3, 0.2, 0.05] (0.85) normal
 * t4: [0.1, 0.1] (0.2) underloaded
 * t5: [0.1, 0.1, 0.1, 0.1] (0.4) underloaded
 *
 * After rebalancing
 * t1: [0.1,  0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05] (0.9) overloaded
 * t2: [0.05, 0.05, 0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05] (0.9) overloaded
 * t3: [0.1, 0.2, 0.3, 0.2, 0.05] (0.85) normal
 * t4: [0.1, 0.1, 0.05, 0.05] (0.3) underloaded
 * t5: [0.1, 0.1, 0.1, 0.1] (0.4) underloaded
 */
@Test
public void defaultRebalancerTest1() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "0");
    jcb.bindImplementation(LoadUpdater.class, TestLoadUpdater.class);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final GroupAllocationTable groupAllocationTable = injector.getInstance(GroupAllocationTable.class);
    final GroupRebalancer rebalancer = injector.getInstance(DefaultGroupRebalancerImpl.class);
    final LoadUpdater loadUpdater = injector.getInstance(LoadUpdater.class);
    final EventProcessorFactory epFactory = injector.getInstance(DefaultEventProcessorFactory.class);
    final List<EventProcessor> eventProcessors = new LinkedList<>();
    for (int i = 0; i < 5; i++) {
        eventProcessors.add(epFactory.newEventProcessor());
        groupAllocationTable.put(eventProcessors.get(i));
    }
    final List<Double> loadsForEp1 = Arrays.asList(0.1, 0.05, 0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05);
    final List<Double> loadsForEp2 = Arrays.asList(0.05, 0.05, 0.05, 0.1, 0.2, 0.1, 0.05, 0.1, 0.2, 0.05);
    final List<Double> loadsForEp3 = Arrays.asList(0.1, 0.2, 0.3, 0.2, 0.05);
    final List<Double> loadsForEp4 = Arrays.asList(0.1, 0.1);
    final List<Double> loadsForEp5 = Arrays.asList(0.1, 0.1, 0.1, 0.1);
    final List<List<Double>> loads = Arrays.asList(loadsForEp1, loadsForEp2, loadsForEp3, loadsForEp4, loadsForEp5);
    for (int i = 0; i < 5; i++) {
        final EventProcessor eventProcessor = eventProcessors.get(i);
        final List<Double> loadList = loads.get(i);
        for (final Double load : loadList) {
            final Group group = mock(Group.class);
            when(group.getLoad()).thenReturn(load);
            when(group.toString()).thenReturn(Double.toString(load));
            when(group.isSplited()).thenReturn(false);
            groupAllocationTable.getValue(eventProcessor).add(group);
        }
    }
    loadUpdater.update();
    rebalancer.triggerRebalancing();
    Assert.assertEquals(0.9, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(0))), 0.0001);
    Assert.assertEquals(0.9, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(1))), 0.0001);
    Assert.assertEquals(0.85, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(2))), 0.0001);
    Assert.assertEquals(0.3, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(3))), 0.0001);
    Assert.assertEquals(0.4, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(4))), 0.0001);
}

Example 9 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class GroupSplitterTest method defaultGroupSplitterTest1.

/**
 * alpha: 0.6
 * beta: 0.8
 * target: 0.7.
 *
 * [0.3 group]: [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]
 * [0.5 group]: [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]
 *
 *                     **
 * t1: [0.35, 0.2, 0.2, 0.3] (1.05) overloaded
 *            **
 * t2: [0.2, 0.5, 0.2] (0.9) overloaded
 * t3: [0.58] (0.58) underloaded
 * t4: [0.5, 0.05] (0.55) underloaded
 *
 * After splitting
 * t1: [0.3, 0.2, 0.2, 0.15] (0.85)
 * t2: [0.25, 0.4, 0.2] (0.8)
 * t3: [0.6, 0.1] (0.7)
 * t4: [0.5, 0.2] (0.7)
 */
@Test
public void defaultGroupSplitterTest1() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "0");
    jcb.bindImplementation(LoadUpdater.class, TestLoadUpdater.class);
    jcb.bindNamedParameter(OverloadedThreshold.class, "0.8");
    jcb.bindNamedParameter(UnderloadedThreshold.class, "0.6");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final GroupIdRequestor requestor = new TestGroupIdRequestor();
    injector.bindVolatileInstance(GroupIdRequestor.class, requestor);
    final GroupAllocationTable groupAllocationTable = injector.getInstance(GroupAllocationTable.class);
    final GroupSplitter groupSplitter = injector.getInstance(GroupSplitter.class);
    final LoadUpdater loadUpdater = injector.getInstance(LoadUpdater.class);
    final EventProcessorFactory epFactory = injector.getInstance(DefaultEventProcessorFactory.class);
    final List<EventProcessor> eventProcessors = new LinkedList<>();
    for (int i = 0; i < 4; i++) {
        eventProcessors.add(epFactory.newEventProcessor());
        groupAllocationTable.put(eventProcessors.get(i));
    }
    final EventProcessor ep1 = eventProcessors.get(0);
    final EventProcessor ep2 = eventProcessors.get(1);
    final EventProcessor ep3 = eventProcessors.get(2);
    final EventProcessor ep4 = eventProcessors.get(3);
    final ApplicationInfo mg1 = createApplication();
    final Group g1 = createGroup("g1");
    mg1.addGroup(g1);
    g1.setLoad(0.3);
    g1.setEventProcessor(ep1);
    final ApplicationInfo mg2 = createApplication();
    final Group g2 = createGroup("g2");
    mg2.addGroup(g2);
    g2.setLoad(0.2);
    g2.setEventProcessor(ep1);
    final ApplicationInfo mg3 = createApplication();
    final Group g3 = createGroup("g3");
    mg3.addGroup(g3);
    g3.setLoad(0.2);
    g3.setEventProcessor(ep1);
    final ApplicationInfo mg4 = createApplication();
    final Group g4 = createGroup("g4");
    mg4.addGroup(g4);
    g4.setLoad(0.3);
    g4.setEventProcessor(ep1);
    final List<Query> g4Query = new LinkedList<>();
    for (int i = 0; i < 6; i++) {
        final Query sg1 = createQuery("sg" + i + "_of_g4");
        sg1.setLoad(0.05);
        g4.addQuery(sg1);
    }
    final ApplicationInfo mg5 = createApplication();
    final Group g5 = createGroup("g5");
    mg5.addGroup(g5);
    g5.setLoad(0.2);
    g5.setEventProcessor(ep2);
    final ApplicationInfo mg6 = createApplication();
    final Group g6 = createGroup("g6");
    mg6.addGroup(g6);
    g6.setLoad(0.5);
    g6.setEventProcessor(ep2);
    final List<Query> g6SubGroups = new LinkedList<>();
    for (int i = 0; i < 10; i++) {
        final Query sg1 = createQuery("sg" + i + "_of_g6");
        sg1.setLoad(0.05);
        g6.addQuery(sg1);
    }
    final ApplicationInfo mg7 = createApplication();
    final Group g7 = createGroup("g7");
    g7.setLoad(0.2);
    g7.setEventProcessor(ep2);
    mg7.addGroup(g7);
    final ApplicationInfo mg8 = createApplication();
    final Group g8 = createGroup("g8");
    g8.setLoad(0.58);
    g8.setEventProcessor(ep3);
    mg8.addGroup(g8);
    final ApplicationInfo mg9 = createApplication();
    final Group g9 = createGroup("g9");
    g9.setLoad(0.5);
    g9.setEventProcessor(ep4);
    mg9.addGroup(g9);
    final Group g10 = createGroup("g4");
    g10.setLoad(0.05);
    g10.setEventProcessor(ep4);
    mg4.addGroup(g10);
    groupAllocationTable.getValue(ep1).add(g1);
    groupAllocationTable.getValue(ep1).add(g2);
    groupAllocationTable.getValue(ep1).add(g3);
    groupAllocationTable.getValue(ep1).add(g4);
    groupAllocationTable.getValue(ep2).add(g5);
    groupAllocationTable.getValue(ep2).add(g6);
    groupAllocationTable.getValue(ep2).add(g7);
    groupAllocationTable.getValue(ep3).add(g8);
    groupAllocationTable.getValue(ep4).add(g9);
    groupAllocationTable.getValue(ep4).add(g10);
    loadUpdater.update();
    groupSplitter.splitGroup();
    // Result
    // After splitting
    // t1: [0.3, 0.2, 0.2, 0.15] (0.85)
    // t2: [0.25, 0.4, 0.2] (0.9)
    // t3: [0.58, 0.1] (0.68)
    // t4: [0.5, 0.2] (0.7)
    Assert.assertEquals(2, mg4.getGroups().size());
    Assert.assertEquals(2, mg6.getGroups().size());
    Assert.assertEquals(3, g4.getQueries().size());
    Assert.assertEquals(8, g6.getQueries().size());
    Assert.assertEquals(3, g10.getQueries().size());
    Assert.assertEquals(0.2, g10.getLoad(), 0.0000001);
    Assert.assertEquals(0.15, g4.getLoad(), 0.000001);
    Assert.assertEquals(0.4, g6.getLoad(), 0.0000001);
    Assert.assertEquals(2, groupAllocationTable.getValue(ep3).size());
    Assert.assertEquals(0.85, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(0))), 0.0001);
    Assert.assertEquals(0.8, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(1))), 0.0001);
    Assert.assertEquals(0.68, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(2))), 0.0001);
    Assert.assertEquals(0.7, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(3))), 0.0001);
}

Example 10 with EventProcessor

use of edu.snu.mist.core.task.groupaware.eventprocessor.EventProcessor in project mist by snuspl.

the class DefaultGroupRebalancerImpl method logging.

private void logging(final List<EventProcessor> eventProcessors, final Map<EventProcessor, Double> loadTable) {
    final StringBuilder sb = new StringBuilder();
    sb.append("-------------- TABLE ----------------\n");
    for (final EventProcessor ep : eventProcessors) {
        final Collection<Group> groups = groupAllocationTable.getValue(ep);
        sb.append(ep);
        sb.append(" -> [");
        sb.append(loadTable.get(ep));
        sb.append("], [");
        sb.append(groups.size());
        sb.append("], ");
        sb.append(groups);
        sb.append("\n");
    }
    LOG.info(sb.toString());
}