Examples with JavaConfigurationBuilder org.apache.reef.tang.JavaConfigurationBuilder used on opensource projects

Example 46 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class GroupRebalancerTest method firstFitRebalancerTest1.

// ep1: [1.0, 2.0, 3.0, 4.0, 0.5, 0.5, 1.0] (total 12)
// ep2: [1.0, 1.0, 1.0, 1.0, 0.5, 0.5] (total 5)
// ep3: [0.5, 5.0, 0.5, 0.5, 0.5, 0.5] (total 7.5)
// ep4: [2.0, 0.3, 0.2, 0.5, 0.5] (total 3.5)
// ==> total_load: 28
// ==> desirable load: 7
// Items: [1.0, 2.0, 3.0, 0.5]
// ep1: [4.0, 0.5, 0.5, 1.0] (total 6)
// ep2: [1.0, 1.0, 1.0, 1.0, 0.5, 0.5] (total 5)
// ep3: [5.0, 0.5, 0.5, 0.5, 0.5] (total 7)
// ep4: [2.0, 0.3, 0.2, 0.5, 0.5] (total 3.5)
// First-fit algorithm:
// ep1: [4.0, 0.5, 0.5, 1.0, 1.0] (total 7)
// ep2: [1.0, 1.0, 1.0, 1.0, 0.5, 0.5, 2] (total 7)
// ep3: [5.0, 0.5, 0.5, 0.5, 0.5] (total 7)
// ep4: [2.0, 0.3, 0.2, 0.5, 0.5, 3.0, 0.5] (total 7)
@Test
public void firstFitRebalancerTest1() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "0");
    jcb.bindImplementation(LoadUpdater.class, TestLoadUpdater.class);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final GroupAllocationTable groupAllocationTable = injector.getInstance(GroupAllocationTable.class);
    final GroupRebalancer rebalancer = injector.getInstance(FirstFitRebalancerImpl.class);
    final LoadUpdater loadUpdater = injector.getInstance(LoadUpdater.class);
    final EventProcessorFactory epFactory = injector.getInstance(DefaultEventProcessorFactory.class);
    final List<EventProcessor> eventProcessors = new LinkedList<>();
    for (int i = 0; i < 4; i++) {
        eventProcessors.add(epFactory.newEventProcessor());
        groupAllocationTable.put(eventProcessors.get(i));
    }
    final List<Double> loadsForEp1 = Arrays.asList(1.0, 2.0, 3.0, 4.0, 0.5, 0.5, 1.0);
    final List<Double> loadsForEp2 = Arrays.asList(1.0, 1.0, 1.0, 1.0, 0.5, 0.5);
    final List<Double> loadsForEp3 = Arrays.asList(0.5, 5.0, 0.5, 0.5, 0.5, 0.5);
    final List<Double> loadsForEp4 = Arrays.asList(2.0, 0.3, 0.2, 0.5, 0.5);
    final List<List<Double>> loads = Arrays.asList(loadsForEp1, loadsForEp2, loadsForEp3, loadsForEp4);
    for (int i = 0; i < 4; i++) {
        final EventProcessor eventProcessor = eventProcessors.get(i);
        final List<Double> loadList = loads.get(i);
        for (final Double load : loadList) {
            final Group group = mock(Group.class);
            when(group.getLoad()).thenReturn(load);
            when(group.isSplited()).thenReturn(false);
            when(group.toString()).thenReturn(Double.toString(load));
            groupAllocationTable.getValue(eventProcessor).add(group);
        }
    }
    loadUpdater.update();
    rebalancer.triggerRebalancing();
    Assert.assertEquals(7.0, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(0))));
    Assert.assertEquals(7.0, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(1))));
    Assert.assertEquals(7.0, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(2))));
    Assert.assertEquals(7.0, calculateLoadOfGroups(groupAllocationTable.getValue(eventProcessors.get(3))));
}

Example 47 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MqttSinkTest method setUp.

@Before
public void setUp() throws IOException, InjectionException, MqttException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    mqttSharedResource = injector.getInstance(MQTTSharedResource.class);
    broker = MqttUtils.createMqttBroker();
    subscriber = new MqttClient(MqttUtils.BROKER_URI, "mqttClient");
}

Example 48 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistDriverTest method launchDriverTestHelper.

/**
 * Test whether MistDriver runs MistTaks successfully.
 * @throws InjectionException
 */
public void launchDriverTestHelper() throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DriverRuntimeType.class, "LOCAL");
    jcb.bindNamedParameter(NumTaskCores.class, "1");
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, "1");
    jcb.bindNamedParameter(TaskMemorySize.class, "256");
    jcb.bindNamedParameter(NumMasterCores.class, "1");
    jcb.bindNamedParameter(MasterMemorySize.class, "256");
    final Configuration runtimeConf = LocalRuntimeConfiguration.CONF.build();
    final Configuration driverConf = MistLauncher.getDriverConfiguration(jcb.build());
    final LauncherStatus state = TestLauncher.run(runtimeConf, driverConf, 4000);
    final Optional<Throwable> err = state.getError();
    System.out.println("Job state after execution: " + state);
    System.out.println("Error: " + err.get());
    Assert.assertFalse(err.isPresent());
}

Example 49 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class QueryInfoStoreTest method diskStoreTest.

/**
 * Tests whether the PlanStore correctly saves, deletes and loads the operator chain dag.
 * @throws InjectionException
 * @throws IOException
 */
@Test(timeout = 1000)
public void diskStoreTest() throws InjectionException, IOException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Jar files
    final List<ByteBuffer> jarFiles = new LinkedList<>();
    final ByteBuffer byteBuffer1 = ByteBuffer.wrap(new byte[] { 0, 1, 0, 1, 1, 1 });
    jarFiles.add(byteBuffer1);
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DriverHostname.class, DRIVER_HOSTNAME);
    jcb.bindNamedParameter(MasterToDriverPort.class, String.valueOf(DRIVER_PORT));
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final QueryInfoStore store = injector.getInstance(QueryInfoStore.class);
    final ApplicationCodeManager applicationCodeManager = injector.getInstance(ApplicationCodeManager.class);
    final ApplicationMap applicationMap = injector.getInstance(ApplicationMap.class);
    final String queryId1 = "testQuery1";
    final String queryId2 = "testQuery2";
    final String tmpFolderPath = injector.getNamedInstance(SharedStorePath.class);
    final File folder = new File(tmpFolderPath);
    // Store jar files
    final List<String> paths = applicationCodeManager.registerNewAppCode(jarFiles).getJarPaths();
    for (int i = 0; i < jarFiles.size(); i++) {
        final ByteBuffer buf = ByteBuffer.allocateDirect(jarFiles.get(i).capacity());
        final String path = paths.get(i);
        final FileInputStream fis = new FileInputStream(path);
        final FileChannel channel = fis.getChannel();
        channel.read(buf);
        Assert.assertEquals(jarFiles.get(i), buf);
    }
    final ApplicationInfo applicationInfo = mock(ApplicationInfo.class);
    when(applicationInfo.getApplicationId()).thenReturn(TestParameters.SUPER_GROUP_ID);
    when(applicationInfo.getJarFilePath()).thenReturn(paths);
    applicationMap.putIfAbsent(TestParameters.SUPER_GROUP_ID, applicationInfo);
    // Generate logical plan
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroDag1 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final AvroDag avroDag2 = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(paths).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    // Store the chained dag
    store.saveAvroDag(new Tuple<>(queryId1, avroDag1));
    store.saveAvroDag(new Tuple<>(queryId2, avroDag2));
    while (!(store.isStored(queryId1) && store.isStored(queryId2))) {
    // Wait until the plan is stored
    }
    Assert.assertTrue(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertTrue(new File(tmpFolderPath, queryId2 + ".plan").exists());
    // Test stored file
    final AvroDag loadedDag1 = store.load(queryId1);
    Assert.assertEquals(avroDag1.getEdges(), loadedDag1.getEdges());
    Assert.assertEquals(avroDag1.getSchema(), loadedDag1.getSchema());
    testVerticesEqual(avroDag1.getAvroVertices(), loadedDag1.getAvroVertices());
    final AvroDag loadedDag2 = store.load(queryId2);
    Assert.assertEquals(avroDag2.getEdges(), loadedDag2.getEdges());
    Assert.assertEquals(avroDag2.getSchema(), loadedDag2.getSchema());
    testVerticesEqual(avroDag2.getAvroVertices(), loadedDag2.getAvroVertices());
    // Test deletion
    store.delete(queryId1);
    store.delete(queryId2);
    Assert.assertFalse(store.isStored(queryId1));
    Assert.assertFalse(new File(tmpFolderPath, queryId1 + ".plan").exists());
    Assert.assertFalse(store.isStored(queryId2));
    Assert.assertFalse(new File(tmpFolderPath, queryId2 + ".plan").exists());
    for (final String path : paths) {
        Assert.assertFalse(new File(path).exists());
    }
    folder.delete();
}

Example 50 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class GroupSplitterTest method createGroup.

private Group createGroup(final String id) throws InjectionException {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(GroupId.class, id);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    return injector.getInstance(Group.class);
}