Examples with Configuration org.apache.reef.tang.Configuration used on opensource projects

Example 1 with Configuration

use of org.apache.reef.tang.Configuration in project heron by twitter.

the class YarnLauncherTest method getHMDriverConfConstructsReefConfig.

// This test verifies if launcher correctly provides all heron specific configurations are to reef
// framework. It does so by ensuring required configs exist. The test fails if an unknown config
// is found. Presence of unknown config should be verified and then added to the test below
@Test
public void getHMDriverConfConstructsReefConfig() throws Exception {
    YarnLauncher launcher = new YarnLauncher();
    YarnLauncher spyLauncher = Mockito.spy(launcher);
    Mockito.doNothing().when(spyLauncher).addLibraryToClasspathSet(Mockito.anyString());
    // inputConf contains configs provided to the launcher
    Map<String, String> inputConf = new HashMap<>();
    // expected contains the reef configs launcher is expected to construct using testConfigs
    Map<String, String> expected = new HashMap<>();
    setConfigs(inputConf, expected, Key.TOPOLOGY_NAME, "topology", TopologyName.class);
    setConfigs(inputConf, expected, Key.TOPOLOGY_BINARY_FILE, "binary", TopologyJar.class);
    setConfigs(inputConf, expected, Key.TOPOLOGY_PACKAGE_FILE, "pack", TopologyPackageName.class);
    setConfigs(inputConf, expected, Key.CLUSTER, "cluster", Cluster.class);
    setConfigs(inputConf, expected, Key.ROLE, "role", Role.class);
    setConfigs(inputConf, expected, Key.ENVIRON, "env", Environ.class);
    setConfigs(inputConf, expected, YarnKey.HERON_SCHEDULER_YARN_QUEUE.value(), "q", JobQueue.class);
    setConfigs(inputConf, expected, YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), "123", DriverMemory.class);
    setConfigs(inputConf, expected, Key.CORE_PACKAGE_URI, new File(".").getName(), HeronCorePackageName.class);
    // the following expected values are mostly specific to reef runtime
    expected.put(JobControlHandler.class.getSimpleName(), ClientManager.class.getName());
    expected.put(NodeDescriptorHandler.class.getSimpleName(), NodeDescriptorHandler.class.getName());
    expected.put(ResourceAllocationHandler.class.getSimpleName(), ResourceAllocationHandler.class.getName());
    expected.put(ResourceStatusHandler.class.getSimpleName(), ResourceStatusHandler.class.getName());
    expected.put(RuntimeParameters.RuntimeStatusHandler.class.getSimpleName(), ResourceManagerStatus.class.getName());
    expected.put(VerboseLogMode.class.getSimpleName(), "false");
    expected.put(HttpPort.class.getSimpleName(), "0");
    expected.put(DriverIdentifier.class.getSimpleName(), "topology");
    Config.Builder builder = new Config.Builder();
    for (String configName : inputConf.keySet()) {
        builder.put(configName, inputConf.get(configName));
    }
    builder.put(Key.STATE_MANAGER_CLASS, "statemanager");
    builder.put(Key.PACKING_CLASS, "packing");
    Config config = builder.build();
    spyLauncher.initialize(config, null);
    Configuration resultConf = spyLauncher.getHMDriverConf();
    for (NamedParameterNode<?> reefConfigNode : resultConf.getNamedParameters()) {
        String resultValue = resultConf.getNamedParameter(reefConfigNode);
        String expectedValue = expected.get(reefConfigNode.getName());
        if (expectedValue == null) {
            Assert.fail(String.format("Unknown config, %s:%s, provided to heron driver", reefConfigNode.getName(), resultConf.getNamedParameter(reefConfigNode)));
        } else {
            Assert.assertEquals(expectedValue, resultValue);
        }
        expected.remove(reefConfigNode.getName());
    }
    Assert.assertEquals(String.format("Missing expected configurations: %s", expected), 0, expected.size());
}

Example 2 with Configuration

use of org.apache.reef.tang.Configuration in project heron by twitter.

the class HeronMasterDriver method submitHeronExecutorTask.

void submitHeronExecutorTask(int workerId) {
    Optional<HeronWorker> worker = multiKeyWorkerMap.lookupByWorkerId(workerId);
    if (!worker.isPresent()) {
        return;
    }
    LOG.log(Level.INFO, "Submitting evaluator task for id: {0}", workerId);
    // topologyName and other configurations are required by Heron Executor and Task to load
    // configuration files. Using REEF configuration model is better than depending on external
    // persistence.
    final Configuration taskConf = HeronTaskConfiguration.CONF.set(TaskConfiguration.TASK, HeronExecutorTask.class).set(TaskConfiguration.ON_CLOSE, HeronExecutorTask.HeronExecutorTaskTerminator.class).set(TaskConfiguration.IDENTIFIER, workerId + "").set(HeronTaskConfiguration.TOPOLOGY_NAME, topologyName).set(HeronTaskConfiguration.TOPOLOGY_JAR, topologyJar).set(HeronTaskConfiguration.TOPOLOGY_PACKAGE_NAME, topologyPackageName).set(HeronTaskConfiguration.HERON_CORE_PACKAGE_NAME, heronCorePackageName).set(HeronTaskConfiguration.ROLE, role).set(HeronTaskConfiguration.ENV, env).set(HeronTaskConfiguration.CLUSTER, cluster).set(HeronTaskConfiguration.COMPONENT_RAM_MAP, getComponentRamMap()).set(HeronTaskConfiguration.CONTAINER_ID, workerId).set(HeronTaskConfiguration.VERBOSE, verboseMode).build();
    worker.get().context.submitTask(taskConf);
}

Example 3 with Configuration

use of org.apache.reef.tang.Configuration in project heron by twitter.

the class YarnLauncher method launch.

@Override
public boolean launch(PackingPlan packing) {
    Configuration reefRuntimeConf = getRuntimeConf();
    Configuration reefDriverConf = getHMDriverConf();
    Configuration reefClientConf = getClientConf();
    boolean ret;
    try {
        final Injector injector = Tang.Factory.getTang().newInjector(reefRuntimeConf, reefClientConf);
        final REEF reef = injector.getInstance(REEF.class);
        final ReefClientSideHandlers client = injector.getInstance(ReefClientSideHandlers.class);
        reef.submit(reefDriverConf);
        ret = client.waitForSchedulerJobResponse();
    } catch (InjectionException | InterruptedException e) {
        LOG.log(Level.WARNING, "Failed to launch REEF app", e);
        return false;
    }
    return ret;
}

Example 4 with Configuration

use of org.apache.reef.tang.Configuration in project heron by twitter.

the class HeronMasterDriverTest method createContextConfigCreatesForGivenWorkerId.

@Test
public void createContextConfigCreatesForGivenWorkerId() {
    Configuration config = driver.createContextConfig(4);
    boolean found = false;
    for (NamedParameterNode<?> namedParameterNode : config.getNamedParameters()) {
        if (namedParameterNode.getName().equals(ContextIdentifier.class.getSimpleName())) {
            Assert.assertEquals("4", config.getNamedParameter(namedParameterNode));
            found = true;
        }
    }
    assertTrue("ContextIdentifier didn't exist.", found);
}