Examples with OrbConfiguration org.goldenorb.conf.OrbConfiguration used on opensource projects

Example 1 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class OrbPartitionProcess method getLocalFilesPath.

/**
   * Calculates what the class paths for the distributed files that are in this partitions temp directory. All
   * files are distributed to /temp directory/GoldenOrb/jobNumber/file name.
   * 
   * @return A list of the class paths that need to be added to this partitions class path
   * @throws IOException
   * @throws InterruptedException
   * @throws OrbZKFailure
   */
private List<String> getLocalFilesPath() throws IOException, InterruptedException, OrbZKFailure {
    List<String> paths = null;
    ZooKeeper zk = ZookeeperUtils.connect(conf.getOrbZooKeeperQuorum());
    String zkPath = "/GoldenOrb/" + conf.getOrbClusterName() + "/JobQueue/" + jobNumber;
    logger.info("Getting node " + zkPath + " from ZooKeeper");
    OrbConfiguration orbConf = (OrbConfiguration) ZookeeperUtils.getNodeWritable(zk, zkPath, OrbConfiguration.class, conf);
    if (orbConf != null) {
        Path[] tempPaths = orbConf.getHDFSdistributedFiles();
        if (tempPaths != null) {
            paths = new ArrayList<String>();
            for (Path path : tempPaths) {
                String[] name = path.toString().split("/");
                // files are always put in /<temp directory>/GoldenOrb/<jobNumber>/<file name>
                paths.add(System.getProperty("java.io.tmpdir") + "/GoldenOrb/" + orbConf.getOrbClusterName() + "/" + jobNumber + "/" + name[name.length - 1]);
            }
        }
    }
    zk.close();
    return paths;
}

Example 2 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class TestJobManager method setUp.

@BeforeClass
public static void setUp() throws IOException {
    OrbConfiguration orbConf = new OrbConfiguration(true);
    cluster = new MiniDFSCluster(orbConf, 3, true, null);
}

Example 3 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class TestRunner method main.

/**
 * 
 * @param  String[] args
 */
public static void main(String[] args) {
    OrbRunner runner = new OrbRunner();
    runner.runJob(new OrbConfiguration(true));
}

Example 4 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class TestInputSplitAllocatorDFS method testInputSplitAllocator.

@Test
public void testInputSplitAllocator() throws Exception {
    LOG = LoggerFactory.getLogger(TestInputSplitAllocatorDFS.class);
    fs.copyFromLocalFile(new Path("src/test/resources/InputSplitAllocatorDFSTestData.txt"), new Path("test/inpath"));
    OrbConfiguration orbConf = new OrbConfiguration(true);
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.setJobNumber("0");
    orbConf.setFileInputPath("test/inpath");
    orbConf.setNameNode("hdfs://localhost:" + cluster.getNameNodePort());
    String hostname = OrbDNS.getDefaultHost(orbConf);
    if (hostname.endsWith(".")) {
        hostname = hostname.substring(0, hostname.length() - 1);
    }
    OrbPartitionMember opm1 = new OrbPartitionMember();
    opm1.setHostname(hostname);
    opm1.setPort(0);
    OrbPartitionMember opm2 = new OrbPartitionMember();
    opm2.setHostname(hostname);
    opm2.setPort(1);
    OrbPartitionMember opm3 = new OrbPartitionMember();
    opm3.setHostname(hostname);
    opm3.setPort(2);
    OrbPartitionMember opm4 = new OrbPartitionMember();
    opm4.setHostname(hostname);
    opm4.setPort(3);
    OrbPartitionMember opm5 = new OrbPartitionMember();
    opm5.setHostname(hostname);
    opm5.setPort(4);
    OrbPartitionMember opm6 = new OrbPartitionMember();
    opm6.setHostname(hostname);
    opm6.setPort(5);
    List<OrbPartitionMember> orbPartitionMembers = new ArrayList<OrbPartitionMember>();
    orbPartitionMembers.add(opm1);
    orbPartitionMembers.add(opm2);
    orbPartitionMembers.add(opm3);
    orbPartitionMembers.add(opm4);
    orbPartitionMembers.add(opm5);
    orbPartitionMembers.add(opm6);
    InputSplitAllocator isa = new InputSplitAllocator(orbConf, orbPartitionMembers);
    Map<OrbPartitionMember, List<RawSplit>> inputSplitAssignments = isa.assignInputSplits();
    long totalFileSize = 0;
    for (OrbPartitionMember orbPartitionMember : inputSplitAssignments.keySet()) {
        long rawSplitSize = 0;
        for (RawSplit rSplit : inputSplitAssignments.get(orbPartitionMember)) {
            rawSplitSize += rSplit.getDataLength();
        }
        totalFileSize += rawSplitSize;
        LOG.info(orbPartitionMember.getHostname() + ":" + orbPartitionMember.getPort() + " | RawSplits count: " + inputSplitAssignments.get(orbPartitionMember).size() + " | RawSplits size: " + rawSplitSize);
        assertTrue(inputSplitAssignments.get(orbPartitionMember).size() <= 5);
    }
    File testFile = new File("src/test/resources/InputSplitAllocatorDFSTestData.txt");
    assertTrue(totalFileSize == testFile.length());
}

Example 5 with OrbConfiguration

use of org.goldenorb.conf.OrbConfiguration in project goldenorb by jzachr.

the class OrbPartitionManagerTest method setUpTest.

@BeforeClass
public static void setUpTest() throws IOException, InterruptedException, OrbZKFailure {
    orbConf = new OrbConfiguration(true);
    orbConf.setOrbZooKeeperQuorum("localhost:21810");
    ZK = ZookeeperUtils.connect(orbConf.getOrbZooKeeperQuorum());
}