use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class LocalFileSystemConfigTest method testOverrideConfig.
@Test
public void testOverrideConfig() throws Exception {
String overrideDirectory = "/users/twitter";
Config config = Config.toLocalMode(Config.newBuilder().putAll(getDefaultConfig()).put(LocalFileSystemKey.FILE_SYSTEM_DIRECTORY.value(), overrideDirectory).build());
Assert.assertEquals(LocalFileSystemContext.fileSystemDirectory(config), overrideDirectory);
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class LocalFileSystemUploaderTest method testUndo.
@Test
public void testUndo() throws Exception {
// identify the location of the test topology tar file
String topologyPackage = Paths.get(testTopologyDirectory, "some-topology.tar").toString();
Config newconfig = Config.newBuilder().putAll(config).put(Key.TOPOLOGY_PACKAGE_FILE, topologyPackage).build();
// create the uploader and load the package
LocalFileSystemUploader uploader = new LocalFileSystemUploader();
uploader.initialize(newconfig);
Assert.assertNotNull(uploader.uploadPackage());
// verify if the file exists
String destFile = uploader.getTopologyFile();
Assert.assertTrue(new File(destFile).isFile());
// now undo the file
uploader.undo();
Assert.assertFalse(new File(destFile).isFile());
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class HdfsUploaderTest method setUp.
@Before
public void setUp() throws Exception {
Config config = Mockito.mock(Config.class);
// Insert mock HdfsController
uploader = Mockito.spy(new HdfsUploader());
controller = Mockito.mock(HdfsController.class);
Mockito.doReturn(controller).when(uploader).getHdfsController();
uploader.initialize(config);
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class LocalFileSystemConfigTest method testDefaultConfig.
@Test
public void testDefaultConfig() throws Exception {
Config config = Config.toLocalMode(getDefaultConfig());
Assert.assertEquals(LocalFileSystemContext.fileSystemDirectory(config), TokenSub.substitute(config, LocalFileSystemKey.FILE_SYSTEM_DIRECTORY.getDefaultString()));
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class YarnLauncherTest method getHMDriverConfConstructsReefConfig.
// This test verifies if launcher correctly provides all heron specific configurations are to reef
// framework. It does so by ensuring required configs exist. The test fails if an unknown config
// is found. Presence of unknown config should be verified and then added to the test below
@Test
public void getHMDriverConfConstructsReefConfig() throws Exception {
YarnLauncher launcher = new YarnLauncher();
YarnLauncher spyLauncher = Mockito.spy(launcher);
Mockito.doNothing().when(spyLauncher).addLibraryToClasspathSet(Mockito.anyString());
// inputConf contains configs provided to the launcher
Map<String, String> inputConf = new HashMap<>();
// expected contains the reef configs launcher is expected to construct using testConfigs
Map<String, String> expected = new HashMap<>();
setConfigs(inputConf, expected, Key.TOPOLOGY_NAME, "topology", TopologyName.class);
setConfigs(inputConf, expected, Key.TOPOLOGY_BINARY_FILE, "binary", TopologyJar.class);
setConfigs(inputConf, expected, Key.TOPOLOGY_PACKAGE_FILE, "pack", TopologyPackageName.class);
setConfigs(inputConf, expected, Key.CLUSTER, "cluster", Cluster.class);
setConfigs(inputConf, expected, Key.ROLE, "role", Role.class);
setConfigs(inputConf, expected, Key.ENVIRON, "env", Environ.class);
setConfigs(inputConf, expected, YarnKey.HERON_SCHEDULER_YARN_QUEUE.value(), "q", JobQueue.class);
setConfigs(inputConf, expected, YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), "123", DriverMemory.class);
setConfigs(inputConf, expected, Key.CORE_PACKAGE_URI, new File(".").getName(), HeronCorePackageName.class);
// the following expected values are mostly specific to reef runtime
expected.put(JobControlHandler.class.getSimpleName(), ClientManager.class.getName());
expected.put(NodeDescriptorHandler.class.getSimpleName(), NodeDescriptorHandler.class.getName());
expected.put(ResourceAllocationHandler.class.getSimpleName(), ResourceAllocationHandler.class.getName());
expected.put(ResourceStatusHandler.class.getSimpleName(), ResourceStatusHandler.class.getName());
expected.put(RuntimeParameters.RuntimeStatusHandler.class.getSimpleName(), ResourceManagerStatus.class.getName());
expected.put(VerboseLogMode.class.getSimpleName(), "false");
expected.put(HttpPort.class.getSimpleName(), "0");
expected.put(DriverIdentifier.class.getSimpleName(), "topology");
Config.Builder builder = new Config.Builder();
for (String configName : inputConf.keySet()) {
builder.put(configName, inputConf.get(configName));
}
builder.put(Key.STATE_MANAGER_CLASS, "statemanager");
builder.put(Key.PACKING_CLASS, "packing");
Config config = builder.build();
spyLauncher.initialize(config, null);
Configuration resultConf = spyLauncher.getHMDriverConf();
for (NamedParameterNode<?> reefConfigNode : resultConf.getNamedParameters()) {
String resultValue = resultConf.getNamedParameter(reefConfigNode);
String expectedValue = expected.get(reefConfigNode.getName());
if (expectedValue == null) {
Assert.fail(String.format("Unknown config, %s:%s, provided to heron driver", reefConfigNode.getName(), resultConf.getNamedParameter(reefConfigNode)));
} else {
Assert.assertEquals(expectedValue, resultValue);
}
expected.remove(reefConfigNode.getName());
}
Assert.assertEquals(String.format("Missing expected configurations: %s", expected), 0, expected.size());
}
Aggregations