Examples with InputSplitAllocator org.goldenorb.io.InputSplitAllocator used on opensource projects

Example 1 with InputSplitAllocator

use of org.goldenorb.io.InputSplitAllocator in project goldenorb by jzachr.

the class TestInputSplitAllocatorDFS method testInputSplitAllocator.

@Test
public void testInputSplitAllocator() throws Exception {
    LOG = LoggerFactory.getLogger(TestInputSplitAllocatorDFS.class);
    fs.copyFromLocalFile(new Path("src/test/resources/InputSplitAllocatorDFSTestData.txt"), new Path("test/inpath"));
    OrbConfiguration orbConf = new OrbConfiguration(true);
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.setJobNumber("0");
    orbConf.setFileInputPath("test/inpath");
    orbConf.setNameNode("hdfs://localhost:" + cluster.getNameNodePort());
    String hostname = OrbDNS.getDefaultHost(orbConf);
    if (hostname.endsWith(".")) {
        hostname = hostname.substring(0, hostname.length() - 1);
    }
    OrbPartitionMember opm1 = new OrbPartitionMember();
    opm1.setHostname(hostname);
    opm1.setPort(0);
    OrbPartitionMember opm2 = new OrbPartitionMember();
    opm2.setHostname(hostname);
    opm2.setPort(1);
    OrbPartitionMember opm3 = new OrbPartitionMember();
    opm3.setHostname(hostname);
    opm3.setPort(2);
    OrbPartitionMember opm4 = new OrbPartitionMember();
    opm4.setHostname(hostname);
    opm4.setPort(3);
    OrbPartitionMember opm5 = new OrbPartitionMember();
    opm5.setHostname(hostname);
    opm5.setPort(4);
    OrbPartitionMember opm6 = new OrbPartitionMember();
    opm6.setHostname(hostname);
    opm6.setPort(5);
    List<OrbPartitionMember> orbPartitionMembers = new ArrayList<OrbPartitionMember>();
    orbPartitionMembers.add(opm1);
    orbPartitionMembers.add(opm2);
    orbPartitionMembers.add(opm3);
    orbPartitionMembers.add(opm4);
    orbPartitionMembers.add(opm5);
    orbPartitionMembers.add(opm6);
    InputSplitAllocator isa = new InputSplitAllocator(orbConf, orbPartitionMembers);
    Map<OrbPartitionMember, List<RawSplit>> inputSplitAssignments = isa.assignInputSplits();
    long totalFileSize = 0;
    for (OrbPartitionMember orbPartitionMember : inputSplitAssignments.keySet()) {
        long rawSplitSize = 0;
        for (RawSplit rSplit : inputSplitAssignments.get(orbPartitionMember)) {
            rawSplitSize += rSplit.getDataLength();
        }
        totalFileSize += rawSplitSize;
        LOG.info(orbPartitionMember.getHostname() + ":" + orbPartitionMember.getPort() + " | RawSplits count: " + inputSplitAssignments.get(orbPartitionMember).size() + " | RawSplits size: " + rawSplitSize);
        assertTrue(inputSplitAssignments.get(orbPartitionMember).size() <= 5);
    }
    File testFile = new File("src/test/resources/InputSplitAllocatorDFSTestData.txt");
    assertTrue(totalFileSize == testFile.length());
}

Example 2 with InputSplitAllocator

use of org.goldenorb.io.InputSplitAllocator in project goldenorb by jzachr.

the class OrbPartition method loadVerticesLeader.

/**
 */
private void loadVerticesLeader() {
    enterBarrier("startLoadVerticesBarrier");
    synchronized (leaderGroupMembers) {
        // Here InputSplits are sent to their constituent partitions for loading
        InputSplitAllocator inputSplitAllocator = new InputSplitAllocator(getOrbConf(), leaderGroupMembers);
        Map<OrbPartitionMember, List<RawSplit>> inputSplitAssignments = inputSplitAllocator.assignInputSplits();
        for (OrbPartitionMember orbPartitionMember : inputSplitAssignments.keySet()) {
            for (RawSplit rawSplit : inputSplitAssignments.get(orbPartitionMember)) {
                orbPartitionMember.loadVerticesFromInputSplit(rawSplit);
            }
        }
    }
    enterBarrier("sentInputSplitsBarrier");
    // just like the slave we have to wait for the InputSplitHandlers to finish loading and sending vertices
    while (!inputSplitLoaderHandlers.isEmpty()) {
        synchronized (this) {
            try {
                wait(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    enterBarrier("inputSplitHandlersCompleteBarrier");
    // them into vertices
    while (!loadVerticesHandlers.isEmpty()) {
        synchronized (this) {
            try {
                wait(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    enterBarrier("loadVerticesIntoPartitionBarrier");
    LOG.debug("Completed Loading vertices!!!");
    if (standby) {
        waitForActivate();
    }
    process();
// try {
// ZookeeperUtils.tryToCreateNode(zk, jobInProgressPath + "/messages/complete");
// } catch (OrbZKFailure e) {
// e.printStackTrace();
// }
// System.exit(1);
}

Example 3 with InputSplitAllocator

use of org.goldenorb.io.InputSplitAllocator in project goldenorb by jzachr.

the class TestInputSplitAllocator method inputSplitAllocatorTest.

/**
 */
@Test
public void inputSplitAllocatorTest() {
    LOG = LoggerFactory.getLogger(TestInputSplitAllocator.class);
    String[] rs1l = { "A", "B" };
    RawSplitWithID rs1 = new RawSplitWithID("rs1", rs1l);
    String[] rs2l = { "B", "C" };
    RawSplitWithID rs2 = new RawSplitWithID("rs2", rs2l);
    String[] rs3l = { "E", "F" };
    RawSplitWithID rs3 = new RawSplitWithID("rs3", rs3l);
    String[] rs4l = { "C" };
    RawSplitWithID rs4 = new RawSplitWithID("rs4", rs4l);
    List<RawSplit> rawSplits = new ArrayList<RawSplit>();
    rawSplits.add(rs1);
    rawSplits.add(rs2);
    rawSplits.add(rs3);
    rawSplits.add(rs4);
    OrbPartitionMember opm1 = new OrbPartitionMember();
    opm1.setHostname("A");
    opm1.setPort(0);
    OrbPartitionMember opm2 = new OrbPartitionMember();
    opm2.setHostname("A");
    opm2.setPort(1);
    OrbPartitionMember opm3 = new OrbPartitionMember();
    opm3.setHostname("B");
    opm3.setPort(0);
    OrbPartitionMember opm4 = new OrbPartitionMember();
    opm4.setHostname("B");
    opm4.setPort(1);
    OrbPartitionMember opm5 = new OrbPartitionMember();
    opm5.setHostname("C");
    opm5.setPort(0);
    OrbPartitionMember opm6 = new OrbPartitionMember();
    opm6.setHostname("C");
    opm6.setPort(1);
    List<OrbPartitionMember> orbPartitionMembers = new ArrayList<OrbPartitionMember>();
    orbPartitionMembers.add(opm1);
    orbPartitionMembers.add(opm2);
    orbPartitionMembers.add(opm3);
    orbPartitionMembers.add(opm4);
    orbPartitionMembers.add(opm5);
    orbPartitionMembers.add(opm6);
    OrbConfiguration orbConf = new OrbConfiguration();
    InputSplitAllocator isa = new InputSplitAllocator(orbConf, orbPartitionMembers);
    Map<OrbPartitionMember, List<RawSplit>> assignedSplits = isa.assignInputSplits(rawSplits);
    for (OrbPartitionMember orbPartitionMember : assignedSplits.keySet()) {
        LOG.info(orbPartitionMember.getHostname() + ":" + orbPartitionMember.getPort() + " | " + assignedSplits.get(orbPartitionMember));
        assertTrue(assignedSplits.get(orbPartitionMember).size() < 2);
    }
}