Examples with OrbPartitionMember org.goldenorb.jet.OrbPartitionMember used on opensource projects

Example 1 with OrbPartitionMember

use of org.goldenorb.jet.OrbPartitionMember in project goldenorb by jzachr.

the class TestInputSplitAllocatorDFS method testInputSplitAllocator.

@Test
public void testInputSplitAllocator() throws Exception {
    LOG = LoggerFactory.getLogger(TestInputSplitAllocatorDFS.class);
    fs.copyFromLocalFile(new Path("src/test/resources/InputSplitAllocatorDFSTestData.txt"), new Path("test/inpath"));
    OrbConfiguration orbConf = new OrbConfiguration(true);
    orbConf.set("fs.default.name", "hdfs://localhost:" + cluster.getNameNodePort());
    orbConf.setJobNumber("0");
    orbConf.setFileInputPath("test/inpath");
    orbConf.setNameNode("hdfs://localhost:" + cluster.getNameNodePort());
    String hostname = OrbDNS.getDefaultHost(orbConf);
    if (hostname.endsWith(".")) {
        hostname = hostname.substring(0, hostname.length() - 1);
    }
    OrbPartitionMember opm1 = new OrbPartitionMember();
    opm1.setHostname(hostname);
    opm1.setPort(0);
    OrbPartitionMember opm2 = new OrbPartitionMember();
    opm2.setHostname(hostname);
    opm2.setPort(1);
    OrbPartitionMember opm3 = new OrbPartitionMember();
    opm3.setHostname(hostname);
    opm3.setPort(2);
    OrbPartitionMember opm4 = new OrbPartitionMember();
    opm4.setHostname(hostname);
    opm4.setPort(3);
    OrbPartitionMember opm5 = new OrbPartitionMember();
    opm5.setHostname(hostname);
    opm5.setPort(4);
    OrbPartitionMember opm6 = new OrbPartitionMember();
    opm6.setHostname(hostname);
    opm6.setPort(5);
    List<OrbPartitionMember> orbPartitionMembers = new ArrayList<OrbPartitionMember>();
    orbPartitionMembers.add(opm1);
    orbPartitionMembers.add(opm2);
    orbPartitionMembers.add(opm3);
    orbPartitionMembers.add(opm4);
    orbPartitionMembers.add(opm5);
    orbPartitionMembers.add(opm6);
    InputSplitAllocator isa = new InputSplitAllocator(orbConf, orbPartitionMembers);
    Map<OrbPartitionMember, List<RawSplit>> inputSplitAssignments = isa.assignInputSplits();
    long totalFileSize = 0;
    for (OrbPartitionMember orbPartitionMember : inputSplitAssignments.keySet()) {
        long rawSplitSize = 0;
        for (RawSplit rSplit : inputSplitAssignments.get(orbPartitionMember)) {
            rawSplitSize += rSplit.getDataLength();
        }
        totalFileSize += rawSplitSize;
        LOG.info(orbPartitionMember.getHostname() + ":" + orbPartitionMember.getPort() + " | RawSplits count: " + inputSplitAssignments.get(orbPartitionMember).size() + " | RawSplits size: " + rawSplitSize);
        assertTrue(inputSplitAssignments.get(orbPartitionMember).size() <= 5);
    }
    File testFile = new File("src/test/resources/InputSplitAllocatorDFSTestData.txt");
    assertTrue(totalFileSize == testFile.length());
}

Example 2 with OrbPartitionMember

use of org.goldenorb.jet.OrbPartitionMember in project goldenorb by jzachr.

the class TestOrbPartitionMember method testOrbPartitionMember.

/*
   * Start of user / non-generated code -- any code written outside of this block will be
   * removed in subsequent code generations.
   */
/* End of user / non-generated code */
@Before
public void testOrbPartitionMember() throws IOException {
    /*
     * Start of user / non-generated code -- any code written outside of this block will be
     * removed in subsequent code generations.
     */
    /* End of user / non-generated code */
    orbPartitionMember = new OrbPartitionMember();
    orbPartitionMember.setPartitionID(INT_PARTITIONID_VALUE);
    orbPartitionMember.setNumberOfVertices(INT_NUMBEROFVERTICES_VALUE);
    orbPartitionMember.setSuperStep(INT_SUPERSTEP_VALUE);
    orbPartitionMember.setMessagesSent(INT_MESSAGESSENT_VALUE);
    orbPartitionMember.setPercentComplete(FLOAT_PERCENTCOMPLETE_VALUE);
    orbPartitionMember.setHostname(STRING_HOSTNAME_VALUE);
    orbPartitionMember.setLeader(BOOLEAN_LEADER_VALUE);
    orbPartitionMember.setPort(INT_PORT_VALUE);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(baos);
    orbPartitionMember.write(out);
    DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    orbPartitionMemberOut = new OrbPartitionMember();
    orbPartitionMemberOut.readFields(in);
/*
     * Start of user / non-generated code -- any code written outside of this block will be
     * removed in subsequent code generations.
     */
/* End of user / non-generated code */
}

Example 3 with OrbPartitionMember

use of org.goldenorb.jet.OrbPartitionMember in project goldenorb by jzachr.

the class OrbPartition method initializeOrbClients.

/**
 * 
 */
private void initializeOrbClients() {
    orbClients = new HashMap<Integer, OrbPartitionCommunicationProtocol>();
    synchronized (leaderGroupMembers) {
        for (OrbPartitionMember member : leaderGroupMembers) {
            // TODO do we need the retry code?
            int count = 0;
            boolean connected = false;
            while (count < 20 && !connected) {
                try {
                    member.initProxy(getOrbConf());
                    connected = true;
                    LOG.debug("partition {} proxy initialized for {}", getPartitionID(), member.getPartitionID());
                } catch (IOException e) {
                    count++;
                    e.printStackTrace();
                }
            }
            orbClients.put(member.getPartitionID(), member);
        }
    }
}

Example 4 with OrbPartitionMember

use of org.goldenorb.jet.OrbPartitionMember in project goldenorb by jzachr.

the class OrbPartition method loadVerticesLeader.

/**
 * 
 */
private void loadVerticesLeader() {
    enterBarrier("startLoadVerticesBarrier");
    synchronized (leaderGroupMembers) {
        // Here InputSplits are sent to their constituent partitions for loading
        InputSplitAllocator inputSplitAllocator = new InputSplitAllocator(getOrbConf(), leaderGroupMembers);
        Map<OrbPartitionMember, List<RawSplit>> inputSplitAssignments = inputSplitAllocator.assignInputSplits();
        for (OrbPartitionMember orbPartitionMember : inputSplitAssignments.keySet()) {
            for (RawSplit rawSplit : inputSplitAssignments.get(orbPartitionMember)) {
                orbPartitionMember.loadVerticesFromInputSplit(rawSplit);
            }
        }
    }
    enterBarrier("sentInputSplitsBarrier");
    // just like the slave we have to wait for the InputSplitHandlers to finish loading and sending vertices
    while (!inputSplitLoaderHandlers.isEmpty()) {
        synchronized (this) {
            try {
                wait(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    enterBarrier("inputSplitHandlersCompleteBarrier");
    // them into vertices
    while (!loadVerticesHandlers.isEmpty()) {
        synchronized (this) {
            try {
                wait(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    enterBarrier("loadVerticesIntoPartitionBarrier");
    LOG.debug("Completed Loading vertices!!!");
    if (standby) {
        waitForActivate();
    }
    process();
// try {
// ZookeeperUtils.tryToCreateNode(zk, jobInProgressPath + "/messages/complete");
// } catch (OrbZKFailure e) {
// e.printStackTrace();
// }
// System.exit(1);
}

Example 5 with OrbPartitionMember

use of org.goldenorb.jet.OrbPartitionMember in project goldenorb by jzachr.

the class InputSplitAllocator method assignInputSplits.

/**
   * This method assigns raw splits to partition members given a Collection of raw splits.
   * 
   * @param rawSplits
   *          - a Collection of RawSplit objects
   * @returns Map
   */
public Map<OrbPartitionMember, List<RawSplit>> assignInputSplits(Collection<RawSplit> rawSplits) {
    Map<OrbPartitionMember, List<RawSplit>> mapOfSplitsToPartitions = new HashMap<OrbPartitionMember, List<RawSplit>>();
    List<RawSplit> notLocalRawSplits = new ArrayList<RawSplit>();
    for (RawSplit rawSplit : rawSplits) {
        List<String> viableLocations = getViableHosts(rawSplit.getLocations());
        if (viableLocations.size() < 1) {
            notLocalRawSplits.add(rawSplit);
            LOG.debug("Non-local split found: " + rawSplit);
        } else {
            String host = getLightestHost(viableLocations);
            int port = getLightestPort(host);
            List<RawSplit> rawSplitAssignedList = hostToPortToRawSplits.get(host).get(port);
            rawSplitAssignedList.add(rawSplit);
            hostToPortToRawSplits.get(host).put(port, rawSplitAssignedList);
            int count = hostToRawSplitCount.get(host);
            count++;
            hostToRawSplitCount.put(host, count);
        }
    }
    for (RawSplit rawSplit : notLocalRawSplits) {
        String host = getLightestHostAll();
        int port = getLightestPort(host);
        List<RawSplit> rawSplitAssignedList = hostToPortToRawSplits.get(host).get(port);
        rawSplitAssignedList.add(rawSplit);
        hostToPortToRawSplits.get(host).put(port, rawSplitAssignedList);
        int count = hostToRawSplitCount.get(host);
        count++;
        hostToRawSplitCount.put(host, count);
    }
    for (OrbPartitionMember orbPartitionMember : orbPartitionMembers) {
        mapOfSplitsToPartitions.put(orbPartitionMember, hostToPortToRawSplits.get(orbPartitionMember.getHostname()).get(orbPartitionMember.getPort()));
    }
    return mapOfSplitsToPartitions;
}