Examples with JavaConfigurationBuilder org.apache.reef.tang.JavaConfigurationBuilder used on opensource projects

Example 41 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class QueryManagerTest method testSubmitComplextQueryInMIST.

// TODO: Re-enable this test
// @Test(timeout = 10000)
public void testSubmitComplextQueryInMIST() throws Exception {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(ClientToTaskPort.class, "20338");
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "4");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final MistTaskConfigs taskConfigs = injector.getInstance(MistTaskConfigs.class);
    testSubmitComplexQueryHelper(taskConfigs.getConfiguration());
}

Example 42 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class GroupRecoveryTest method testSingleQueryRecovery.

@Test(timeout = 500000)
public void testSingleQueryRecovery() throws Exception {
    // Start source servers.
    final CountDownLatch sourceCountDownLatch1 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator1 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch1));
    final TestSinkHandler handler1 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver1 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler1);
    // Submit query.
    final MISTQuery query = buildQuery();
    // Generate avro chained dag : needed parts from MISTExamplesUtils.submitQuery()
    final Tuple<List<AvroVertex>, List<Edge>> initialAvroOpChainDag = query.getAvroOperatorDag();
    final String appId = "testApp";
    final String queryId = "testQuery";
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag = avroDagBuilder.setAppId(appId).setQueryId(queryId).setJarPaths(new ArrayList<>()).setAvroVertices(initialAvroOpChainDag.getKey()).setEdges(initialAvroOpChainDag.getValue()).build();
    // Build QueryManager.
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(PeriodicCheckpointPeriod.class, "1000");
    jcb.bindImplementation(QueryManager.class, GroupAwareQueryManagerImpl.class);
    jcb.bindImplementation(QueryStarter.class, ImmediateQueryMergingStarter.class);
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(GroupIdRequestor.class, new TestGroupIdRequestor());
    injector.bindVolatileInstance(TaskStatsUpdater.class, mock(TaskStatsUpdater.class));
    final CheckpointManager checkpointManager = injector.getInstance(CheckpointManager.class);
    final QueryManager queryManager = injector.getInstance(QueryManager.class);
    final ApplicationInfo appInfo = queryManager.createApplication(appId, Arrays.asList(""));
    queryManager.createGroup(appInfo);
    queryManager.create(avroDag);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch1.await();
    final ExecutionDags executionDags = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags.values().size(), 1);
    // 1st stage. Push inputs to all sources and see if the results are proper.
    SRC0INPUT1.forEach(textMessageStreamGenerator1::write);
    LATCH1.await();
    Assert.assertEquals("{aa=2, bb=1, cc=1}", handler1.getResults().get(handler1.getResults().size() - 1));
    // Sleep 2 seconds for the checkpoint events to be sent out.
    sleep(2000);
    // Checkpoint the entire MISTTask, delete it, and restore it to see if it works.
    final String groupId = checkpointManager.getApplication(appId).getGroups().get(0).getGroupId();
    checkpointManager.checkpointGroup(groupId);
    checkpointManager.deleteGroup(groupId);
    // Close the generator.
    textMessageStreamGenerator1.close();
    receiver1.close();
    // Restart source servers.
    final CountDownLatch sourceCountDownLatch2 = new CountDownLatch(1);
    final NettyTextMessageStreamGenerator textMessageStreamGenerator2 = new NettyTextMessageStreamGenerator(SERVER_ADDR, SOURCE_PORT1, new TestChannelHandler(sourceCountDownLatch2));
    final TestSinkHandler handler2 = new TestSinkHandler();
    final NettyTextMessageOutputReceiver receiver2 = new NettyTextMessageOutputReceiver("localhost", SINK_PORT, handler2);
    // Recover the group.
    checkpointManager.recoverGroup(groupId);
    // Wait until all sources connect to stream generator
    sourceCountDownLatch2.await();
    final ExecutionDags executionDags2 = checkpointManager.getApplication(appId).getGroups().get(0).getExecutionDags();
    Assert.assertEquals(executionDags2.values().size(), 1);
    final ExecutionDag executionDag2 = executionDags2.values().iterator().next();
    // 2nd stage. Push inputs to the recovered the query to see if it works.
    SRC0INPUT2.forEach(textMessageStreamGenerator2::write);
    LATCH2.await();
    Assert.assertEquals("{aa=3, bb=3, cc=3}", handler2.getResults().get(handler2.getResults().size() - 1));
    // Close the generators.
    textMessageStreamGenerator2.close();
    receiver2.close();
}

Example 43 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class EventProcessorTest method createGroup.

/**
 * Create a new group.
 * @throws InjectionException
 */
private Group createGroup(final String groupId) throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(GroupId.class, groupId);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    return injector.getInstance(Group.class);
}

Example 44 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class GroupMergerTest method defaultGroupMergerTest1.

/**
 * t1: [0.4, 0.2, 0.3, 0.1, 0.05] (1.05) overloaded.
 * t2: [0.1, 0.1, 0.05, 0.05] (0.3) underloaded.
 *
 * After merging.
 * t1: [0.4, 0.2, 0.3] (0.9)
 * t2: [0.1, 0.1, 0.15, 0.1] (0.45)
 */
@Test
public void defaultGroupMergerTest1() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "0");
    jcb.bindImplementation(LoadUpdater.class, TestLoadUpdater.class);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final GroupAllocationTable groupAllocationTable = injector.getInstance(GroupAllocationTable.class);
    final GroupMerger groupMerger = injector.getInstance(GroupMerger.class);
    final LoadUpdater loadUpdater = injector.getInstance(LoadUpdater.class);
    final EventProcessorFactory epFactory = injector.getInstance(DefaultEventProcessorFactory.class);
    final List<EventProcessor> eventProcessors = new LinkedList<>();
    for (int i = 0; i < 2; i++) {
        eventProcessors.add(epFactory.newEventProcessor());
        groupAllocationTable.put(eventProcessors.get(i));
    }
    final EventProcessor ep1 = eventProcessors.get(0);
    final EventProcessor ep2 = eventProcessors.get(1);
    final ApplicationInfo mg1 = createMetaGroup();
    final Group g1 = createGroup("g1");
    mg1.addGroup(g1);
    g1.setLoad(0.4);
    g1.setEventProcessor(ep1);
    final ApplicationInfo mg2 = createMetaGroup();
    final Group g2 = createGroup("g2");
    mg2.addGroup(g2);
    g2.setLoad(0.2);
    g2.setEventProcessor(ep1);
    final ApplicationInfo mg3 = createMetaGroup();
    final Group g3 = createGroup("g3");
    mg3.addGroup(g3);
    g3.setLoad(0.3);
    g3.setEventProcessor(ep1);
    final ApplicationInfo mg4 = createMetaGroup();
    final Group g4 = createGroup("g4");
    mg4.addGroup(g4);
    g4.setLoad(0.1);
    g4.setEventProcessor(ep1);
    final Query sg1 = createQuery("sg1");
    sg1.setLoad(0.02);
    g4.addQuery(sg1);
    final Query sg2 = createQuery("sg2");
    sg2.setLoad(0.04);
    g4.addQuery(sg2);
    final Query sg3 = createQuery("sg3");
    sg3.setLoad(0.04);
    g4.addQuery(sg3);
    final ApplicationInfo mg5 = createMetaGroup();
    final Group g5 = createGroup("g5");
    mg5.addGroup(g5);
    g5.setLoad(0.05);
    g5.setEventProcessor(ep1);
    final Query sg4 = createQuery("sg4");
    sg4.setLoad(0.02);
    g5.addQuery(sg4);
    final Query sg5 = createQuery("sg5");
    sg5.setLoad(0.03);
    g5.addQuery(sg5);
    final ApplicationInfo mg6 = createMetaGroup();
    final Group g6 = createGroup("g6");
    mg6.addGroup(g6);
    g6.setLoad(0.1);
    g6.setEventProcessor(ep2);
    final ApplicationInfo mg7 = createMetaGroup();
    final Group g7 = createGroup("g7");
    mg7.addGroup(g7);
    g7.setLoad(0.1);
    g7.setEventProcessor(ep2);
    final Group g44 = createGroup("g4");
    g44.setLoad(0.05);
    mg4.addGroup(g44);
    g44.setEventProcessor(ep2);
    final Query sg6 = createQuery("sg6");
    sg6.setLoad(0.05);
    g44.addQuery(sg6);
    final Group g55 = createGroup("g5");
    mg5.addGroup(g55);
    g55.setLoad(0.05);
    g55.setEventProcessor(ep2);
    final Query sg7 = createQuery("sg7");
    sg7.setLoad(0.05);
    g55.addQuery(sg7);
    groupAllocationTable.getValue(ep1).add(g1);
    groupAllocationTable.getValue(ep1).add(g2);
    groupAllocationTable.getValue(ep1).add(g3);
    groupAllocationTable.getValue(ep1).add(g4);
    groupAllocationTable.getValue(ep1).add(g5);
    groupAllocationTable.getValue(ep2).add(g6);
    groupAllocationTable.getValue(ep2).add(g7);
    groupAllocationTable.getValue(ep2).add(g44);
    groupAllocationTable.getValue(ep2).add(g55);
    loadUpdater.update();
    groupMerger.groupMerging();
    Assert.assertEquals(1, mg4.getGroups().size());
    Assert.assertEquals(1, mg5.getGroups().size());
    Assert.assertEquals(4, g44.size());
    Assert.assertEquals(3, g55.size());
    Assert.assertEquals(g44, sg1.getGroup());
    Assert.assertEquals(g44, sg2.getGroup());
    Assert.assertEquals(g44, sg3.getGroup());
    Assert.assertEquals(g55, sg4.getGroup());
    Assert.assertEquals(g55, sg5.getGroup());
    Assert.assertEquals(0.9, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(0))), 0.0001);
    Assert.assertEquals(0.45, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(1))), 0.0001);
}

Example 45 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class GroupMergerTest method createMetaGroup.

private ApplicationInfo createMetaGroup() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(ApplicationIdentifier.class, "app");
    jcb.bindNamedParameter(JarFilePath.class, "");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final QueryStarter queryStarter = mock(QueryStarter.class);
    final QueryRemover queryRemover = mock(QueryRemover.class);
    final ExecutionDags executionDags = mock(ExecutionDags.class);
    injector.bindVolatileInstance(QueryStarter.class, queryStarter);
    injector.bindVolatileInstance(QueryRemover.class, queryRemover);
    injector.bindVolatileInstance(ExecutionDags.class, executionDags);
    return injector.getInstance(ApplicationInfo.class);
}