Examples with JavaConfigurationBuilder org.apache.reef.tang.JavaConfigurationBuilder used on opensource projects

Example 6 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistCommonConfigs method getConfiguration.

@Override
public Configuration getConfiguration() {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(ClientToMasterPort.class, String.valueOf(clientToMasterPort));
    jcb.bindNamedParameter(ClientToTaskPort.class, String.valueOf(clientToTaskPort));
    jcb.bindNamedParameter(MasterToTaskPort.class, String.valueOf(masterToTaskPort));
    jcb.bindNamedParameter(MasterToDriverPort.class, String.valueOf(masterToDriverPort));
    jcb.bindNamedParameter(TaskToMasterPort.class, String.valueOf(taskToMasterPort));
    jcb.bindNamedParameter(DriverToMasterPort.class, String.valueOf(driverToMasterPort));
    jcb.bindNamedParameter(SharedStorePath.class, sharedStorePath);
    return jcb.build();
}

Example 7 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class MistMasterConfigs method getConfiguration.

@Override
public Configuration getConfiguration() {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    // Named parameters.
    jcb.bindNamedParameter(NumTasks.class, String.valueOf(numTasks));
    jcb.bindNamedParameter(TaskMemorySize.class, String.valueOf(taskMemSize));
    jcb.bindNamedParameter(NumTaskCores.class, String.valueOf(numTaskCores));
    jcb.bindNamedParameter(NewRatio.class, String.valueOf(newRatio));
    jcb.bindNamedParameter(ReservedCodeCacheSize.class, String.valueOf(reservedCodeCacheSize));
    jcb.bindNamedParameter(IdleTaskLoadThreshold.class, String.valueOf(idleTaskThreshold));
    jcb.bindNamedParameter(UnderloadedTaskLoadThreshold.class, String.valueOf(underloadedTaskThreshold));
    jcb.bindNamedParameter(OverloadedTaskLoadThreshold.class, String.valueOf(overloadedTaskThreshold));
    jcb.bindNamedParameter(RecoveryUnitSize.class, String.valueOf(recoveryUnitSize));
    jcb.bindNamedParameter(DynamicScalingPeriod.class, String.valueOf(dynamicScalingPeriod));
    jcb.bindNamedParameter(MaxTaskNum.class, String.valueOf(maxTaskNum));
    jcb.bindNamedParameter(MinTaskNum.class, String.valueOf(minTaskNum));
    jcb.bindNamedParameter(ScaleInGracePeriod.class, String.valueOf(scaleInGracePeriod));
    jcb.bindNamedParameter(ScaleInIdleTaskRatio.class, String.valueOf(scaleInIdleTaskRatio));
    jcb.bindNamedParameter(ScaleOutGracePeriod.class, String.valueOf(scaleOutGracePeriod));
    jcb.bindNamedParameter(ScaleOutOverloadedTaskRatio.class, String.valueOf(scaleOutOverloadedTaskRatio));
    // Implementations.
    jcb.bindImplementation(QueryAllocationManager.class, getQueryAllocationImplClass());
    jcb.bindImplementation(RecoveryScheduler.class, getRecoverySchedulerImplClass());
    jcb.bindImplementation(DynamicScalingManager.class, getDynamicScalingManagerImplClass());
    jcb.bindImplementation(ClientToMasterMessage.class, DefaultClientToMasterMessageImpl.class);
    jcb.bindImplementation(DriverToMasterMessage.class, DefaultDriverToMasterMessageImpl.class);
    jcb.bindImplementation(TaskToMasterMessage.class, DefaultTaskToMasterMessageImpl.class);
    return jcb.build();
}

Example 8 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class Mist method getCommandLineConf.

/**
 * Gets configurations from command line args.
 */
private static Configuration getCommandLineConf(final String[] args) throws Exception {
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    CommandLine commandLine = new CommandLine(jcb).registerShortNameOfClass(DriverRuntimeType.class).registerShortNameOfClass(DriverMemorySize.class);
    commandLine = MistCommandLineOptions.addCommandLineConf(commandLine);
    commandLine = commandLine.processCommandLine(args);
    if (commandLine == null) {
        // Option '?' was entered and processCommandLine printed the help.
        return null;
    }
    return jcb.build();
}

Example 9 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 10 with JavaConfigurationBuilder

use of org.apache.reef.tang.JavaConfigurationBuilder in project mist by snuspl.

the class EventProcessorManagerTest method testGracePeriod.

/**
 * Test whether EventPrcoessorManager does not adjust the number of events processors in grace period.
 */
@Test
public void testGracePeriod() throws InjectionException {
    final int gracePeriod = 3;
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(DefaultNumEventProcessors.class, Integer.toString(DEFAULT_NUM_THREADS));
    jcb.bindNamedParameter(EventProcessorUpperBound.class, Integer.toString(MAX_NUM_THREADS));
    jcb.bindNamedParameter(EventProcessorLowerBound.class, Integer.toString(MIN_NUM_THREADS));
    jcb.bindNamedParameter(GracePeriod.class, Integer.toString(gracePeriod));
    jcb.bindImplementation(EventProcessorFactory.class, TestEventProcessorFactory.class);
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    injector.bindVolatileInstance(GroupIdRequestor.class, mock(GroupIdRequestor.class));
    injector.bindVolatileInstance(TaskStatsUpdater.class, mock(TaskStatsUpdater.class));
    long prevAdjustTime = System.nanoTime();
    eventProcessorManager = injector.getInstance(DefaultEventProcessorManager.class);
    eventProcessorManager.increaseEventProcessors(MAX_NUM_THREADS - DEFAULT_NUM_THREADS);
    // EventProcessorManager should not adjust the number of event processors during grace period
    Assert.assertEquals(DEFAULT_NUM_THREADS, eventProcessorManager.size());
    while (TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - prevAdjustTime) < gracePeriod + 1) {
    // wait
    }
    prevAdjustTime = System.nanoTime();
    eventProcessorManager.increaseEventProcessors(MAX_NUM_THREADS - DEFAULT_NUM_THREADS);
    // EventProcessorManager should adjust the number of event processors
    Assert.assertEquals(MAX_NUM_THREADS, eventProcessorManager.size());
    eventProcessorManager.decreaseEventProcessors(1);
    // EventProcessorManager should not adjust the number of event processors during grace period
    Assert.assertEquals(MAX_NUM_THREADS, eventProcessorManager.size());
    while (TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - prevAdjustTime) < gracePeriod + 1) {
    // wait
    }
    eventProcessorManager.decreaseEventProcessors(1);
    // EventProcessorManager should adjust the number of event processors
    Assert.assertEquals(MAX_NUM_THREADS - 1, eventProcessorManager.size());
}