Examples with OrbConfiguration org.goldenorb.conf.OrbConfiguration used on opensource projects

Example 16 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class MockPartitionThread method launch.

//  public MockPartitionThread() {
//    thread = new Thread(this);
//  }
@Override
public void launch(OutputStream outStream, OutputStream errStream) {
    thread = new Thread(this);
    partitionsPort = getOrbConf().getOrbBasePort() + processNum;
    managerPort = getOrbConf().getOrbBasePort() + processNum + 100;
    jobPath = "/GoldenOrb/" + getOrbConf().getOrbClusterName() + "/JobQueue/" + jobNumber;
    jobInProgressPath = "/GoldenOrb/" + getOrbConf().getOrbClusterName() + "/JobsInProgress/" + jobNumber;
    logger.info("jobPath: " + jobPath);
    logger.info("jobInProgressPath: " + jobInProgressPath);
    try {
        zk = ZookeeperUtils.connect(getOrbConf().getOrbZooKeeperQuorum());
    } catch (Exception e) {
        logger.error("Unable to establish a connection with ZooKeeper" + getOrbConf().getOrbZooKeeperQuorum(), e);
    }
    OrbConfiguration jobConf = null;
    try {
        jobConf = (OrbConfiguration) ZookeeperUtils.getNodeWritable(zk, jobPath, OrbConfiguration.class, getOrbConf());
    } catch (OrbZKFailure e) {
        logger.error("Unable to retrieve job from ZooKeeper: " + jobPath, e);
    }
    if (jobConf != null) {
        setOrbConf(jobConf);
    }
    thread.start();
}

Example 17 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class OrbRunnerTest method testDistributeFiles.

@Test
public void testDistributeFiles() throws IOException, KeeperException, InterruptedException, OrbZKFailure {
    OrbConfiguration orbConf = new OrbConfiguration(true);
    orbConf.setOrbClusterName("TestOrbCluster");
    orbConf.setOrbZooKeeperQuorum("localhost:21810");
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.addFileToDistribute("src/test/resources/distributeTest1.txt");
    orbConf.addFileToDistribute("src/test/resources/distributeTest2.txt");
    orbConf.addFileToDistribute("src/test/resources/HelloWorld.jar");
    runJob(orbConf);
    FileSystem fs = cluster.getFileSystem();
    //Files were copied from local to HDFS
    assertTrue(fs.exists(new Path("/DistributeFiles/distributeTest1.txt")));
    assertTrue(fs.exists(new Path("/DistributeFiles/distributeTest2.txt")));
    assertTrue(fs.exists(new Path("/DistributeFiles/HelloWorld.jar")));
    // Check Paths are set in orbConfiguration
    Path[] localFiles = orbConf.getHDFSdistributedFiles();
    for (Path path : localFiles) {
        System.out.println(path.toString());
    }
    assertEquals(3, localFiles.length);
    List<String> jobList = ZK.getChildren("/GoldenOrb/" + orbConf.getOrbClusterName() + "/JobQueue", false);
    for (String jobName : jobList) {
        OrbConfiguration compareOrbConf = (OrbConfiguration) ZookeeperUtils.getNodeWritable(ZK, "/GoldenOrb/" + orbConf.getOrbClusterName() + "/JobQueue/" + jobName, OrbConfiguration.class, orbConf);
        assertEquals(compareOrbConf.getHDFSdistributedFiles(), orbConf.getHDFSdistributedFiles());
    }
}

Example 18 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class TestJobManager method jobManagerTest.

/**
 * 
 */
@Test
public void jobManagerTest() throws IOException, InterruptedException, OrbZKFailure {
    zk = ZookeeperUtils.connect("localhost:21810");
    ZookeeperUtils.tryToCreateNode(zk, "/GoldenOrb");
    ZookeeperUtils.tryToCreateNode(zk, "/GoldenOrb/OrbCluster");
    //everyone join?
    CountDownLatch joinLeaderGroup = new CountDownLatch(NUM_OF_MEMBERS);
    //everyone ready to exit?
    CountDownLatch exit = new CountDownLatch(NUM_OF_MEMBERS);
    CountDownLatch jobCreated = new CountDownLatch(1);
    CountDownLatch newLeader = new CountDownLatch(1);
    OrbConfiguration orbConf = new OrbConfiguration();
    orbConf.setOrbClusterName("OrbCluster");
    orbConf.setJobHeartbeatTimeout(1000);
    orbConf.setMaximumJobTries(3);
    System.out.println(orbConf.getJobHeartbeatTimeout());
    // Create all of the test Trackers
    for (int i = 0; i < NUM_OF_MEMBERS; i++) {
        TJTracker tracker = new TJTracker(zk, joinLeaderGroup, exit, orbConf, i, "/GoldenOrb/OrbCluster");
        trackers.add(tracker);
        threads.add(new Thread(tracker));
        threads.get(i).start();
    }
    joinLeaderGroup.await();
    orbConf.setOrbZooKeeperQuorum("localhost:21810");
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.addFileToDistribute("src/test/resources/distributeTest1.txt");
    orbConf.addFileToDistribute("src/test/resources/HelloWorld.jar");
    String path1 = runJob(orbConf);
    String path2 = runJob(orbConf);
    System.out.println(path1 + " " + path2);
    new Thread(new HeartbeatUpdater(getJobInProgressPath(path1))).start();
    new Thread(new HeartbeatUpdater(getJobInProgressPath(path2))).start();
    jobCreated.await(2, TimeUnit.SECONDS);
    int leader = 0;
    for (int i = 0; i < NUM_OF_MEMBERS; i++) {
        if (trackers.get(i).isLeader()) {
            leader = i;
        }
    }
    trackers.get(leader).leave();
    newLeader.await(5, TimeUnit.SECONDS);
    //exit and shutdown
    for (int i = 0; i < NUM_OF_MEMBERS; i++) {
        trackers.get(i).leave();
    }
    exit.await();
    ZookeeperUtils.recursiveDelete(zk, "/GoldenOrb");
    ZookeeperUtils.deleteNodeIfEmpty(zk, "/GoldenOrb");
}

Example 19 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class TestInputSplitAllocator method inputSplitAllocatorTest.

/**
 * 
 */
@Test
public void inputSplitAllocatorTest() {
    LOG = LoggerFactory.getLogger(TestInputSplitAllocator.class);
    String[] rs1l = { "A", "B" };
    RawSplitWithID rs1 = new RawSplitWithID("rs1", rs1l);
    String[] rs2l = { "B", "C" };
    RawSplitWithID rs2 = new RawSplitWithID("rs2", rs2l);
    String[] rs3l = { "E", "F" };
    RawSplitWithID rs3 = new RawSplitWithID("rs3", rs3l);
    String[] rs4l = { "C" };
    RawSplitWithID rs4 = new RawSplitWithID("rs4", rs4l);
    List<RawSplit> rawSplits = new ArrayList<RawSplit>();
    rawSplits.add(rs1);
    rawSplits.add(rs2);
    rawSplits.add(rs3);
    rawSplits.add(rs4);
    OrbPartitionMember opm1 = new OrbPartitionMember();
    opm1.setHostname("A");
    opm1.setPort(0);
    OrbPartitionMember opm2 = new OrbPartitionMember();
    opm2.setHostname("A");
    opm2.setPort(1);
    OrbPartitionMember opm3 = new OrbPartitionMember();
    opm3.setHostname("B");
    opm3.setPort(0);
    OrbPartitionMember opm4 = new OrbPartitionMember();
    opm4.setHostname("B");
    opm4.setPort(1);
    OrbPartitionMember opm5 = new OrbPartitionMember();
    opm5.setHostname("C");
    opm5.setPort(0);
    OrbPartitionMember opm6 = new OrbPartitionMember();
    opm6.setHostname("C");
    opm6.setPort(1);
    List<OrbPartitionMember> orbPartitionMembers = new ArrayList<OrbPartitionMember>();
    orbPartitionMembers.add(opm1);
    orbPartitionMembers.add(opm2);
    orbPartitionMembers.add(opm3);
    orbPartitionMembers.add(opm4);
    orbPartitionMembers.add(opm5);
    orbPartitionMembers.add(opm6);
    OrbConfiguration orbConf = new OrbConfiguration();
    InputSplitAllocator isa = new InputSplitAllocator(orbConf, orbPartitionMembers);
    Map<OrbPartitionMember, List<RawSplit>> assignedSplits = isa.assignInputSplits(rawSplits);
    for (OrbPartitionMember orbPartitionMember : assignedSplits.keySet()) {
        LOG.info(orbPartitionMember.getHostname() + ":" + orbPartitionMember.getPort() + " | " + assignedSplits.get(orbPartitionMember));
        assertTrue(assignedSplits.get(orbPartitionMember).size() < 2);
    }
}

Example 20 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class CheckPointDataTest method testCheckpointOutput.

/**
   * Tests the CheckPointDataOutput class by writing several different types of Writables to the checkpoint.
   * 
   * @throws Exception
   */
@Test
public void testCheckpointOutput() throws Exception {
    int superStep = 0;
    int partition = 0;
    OrbConfiguration orbConf = new OrbConfiguration();
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.setJobNumber("0");
    orbConf.setFileOutputPath("test");
    CheckPointDataOutput checkpointOutput = new CheckPointDataOutput(orbConf, superStep, partition);
    IntWritable intOutput = new IntWritable(4);
    intOutput.write(checkpointOutput);
    LongWritable longOutput = new LongWritable(9223372036854775807L);
    longOutput.write(checkpointOutput);
    Text textOutput = new Text("test");
    textOutput.write(checkpointOutput);
    FloatWritable floatOutput = new FloatWritable(3.14159F);
    floatOutput.write(checkpointOutput);
    checkpointOutput.close();
    assertThat(checkpointOutput, notNullValue());
}