Examples with PartitionRequest org.goldenorb.jet.PartitionRequest used on opensource projects

Example 1 with PartitionRequest

use of org.goldenorb.jet.PartitionRequest in project goldenorb by jzachr.

the class JobManager method launchJob.

/**
 * 
 * @param  OrbJob job
 */
private void launchJob(OrbJob job) {
    try {
        ZookeeperUtils.notExistCreateNode(zk, jobsInProgressPath + "/" + job.getJobNumber());
        ZookeeperUtils.notExistCreateNode(zk, jobsInProgressPath + "/" + job.getJobNumber() + "/OrbPartitionLeaderGroup");
        ZookeeperUtils.notExistCreateNode(zk, jobsInProgressPath + "/" + job.getJobNumber() + "/messages");
        ZookeeperUtils.tryToCreateNode(zk, jobsInProgressPath + "/" + job.getJobNumber() + "/messages/heartbeat", new LongWritable(0), CreateMode.PERSISTENT);
        // allocate resources and if enough, start the job
        logger.info("checking for available OrbTracker resources");
        Map<M, Integer[]> assignments = null;
        try {
            assignments = resourceAllocator.assignResources(job.getOrbConf());
        } catch (InvalidJobConfException e) {
            logger.error(e.getMessage());
        }
        logger.info("Starting Job");
        logger.info("********** Job {} started: {}", job.getJobNumber(), new Date().getTime());
        if (assignments != null) {
            logger.info("Allocating partitions");
            int basePartitionID = 0;
            for (M tracker : orbTrackerMembers) {
                logger.debug("OrbTracker - " + tracker.getHostname() + ":" + tracker.getPort());
                Integer[] assignment = assignments.get(tracker);
                tracker.initProxy(getOrbConf());
                try {
                    logger.debug("jobConf().getHDFSdistributedFiles(): {}", job.getOrbConf().getHDFSdistributedFiles());
                    tracker.getRequiredFiles(job.getOrbConf());
                } catch (OrbZKFailure e) {
                    logger.error("EXCEPTION : An OrbTrackerMember failed to copy files from HDFS to local machine");
                    logger.error(e.getMessage());
                    throw e;
                }
                PartitionRequest request = new PartitionRequest();
                request.setActivePartitions(assignment[ResourceAllocator.TRACKER_AVAILABLE]);
                request.setReservedPartitions(assignment[ResourceAllocator.TRACKER_RESERVED]);
                request.setJobID(job.getJobNumber());
                request.setBasePartitionID(basePartitionID);
                request.setJobConf(job.getOrbConf());
                basePartitionID += assignment[ResourceAllocator.TRACKER_AVAILABLE];
                logger.debug("requesting partitions");
                tracker.requestPartitions(request);
                logger.info(request.toString());
                JobStillActiveCheck jobStillActiveCheck = new JobStillActiveCheck(job);
                job.setJobStillActiveInterface(jobStillActiveCheck);
                new Thread(jobStillActiveCheck).start();
                activeJobs.add(job.getJobNumber());
                checkForDeathComplete(job);
                heartbeat(job);
            }
        } else {
            logger.error("not enough capacity for this job");
            jobComplete(job);
        }
    } catch (OrbZKFailure e) {
        e.printStackTrace();
        logger.error(e.getMessage());
        fireEvent(new OrbExceptionEvent(e));
    }
//catch (IOException e) {
//      e.printStackTrace();
//      logger.error(e.getMessage());
//    }
}

Example 2 with PartitionRequest

use of org.goldenorb.jet.PartitionRequest in project goldenorb by jzachr.

the class OrbPartitionManagerTest method testOrbPartitionManager.

@Test
public void testOrbPartitionManager() {
    int requestedPartitions = 3;
    int reservedPartitions = 2;
    int totalPartitions = requestedPartitions + reservedPartitions;
    orbConf.setJobNumber("0");
    OrbPartitionManager<OrbPartitionProcess> OPManager = new OrbPartitionManager<OrbPartitionProcess>(orbConf, OrbPartitionProcess.class);
    assertEquals(OPManager.getOrbConf(), orbConf);
    OPManager.setOrbConf(orbConf);
    assertEquals(OPManager.getPartitionProcessClass(), OrbPartitionProcess.class);
    OPManager.setPartitionProcessClass(OrbPartitionProcess.class);
    logger.info("OrbPartitionManager IP: " + OPManager.getIpAddress());
    PartitionRequest request = new PartitionRequest();
    request.setReservedPartitions(reservedPartitions);
    request.setActivePartitions(requestedPartitions);
    request.setJobID("0");
    request.setBasePartitionID(0);
    request.setJobConf(orbConf);
    try {
        OPManager.launchPartitions(request);
        Map<String, List<OrbPartitionProcess>> mapOPP = OPManager.getProcessesByJob();
        assertTrue(mapOPP.get("0").size() == totalPartitions);
        OPManager.kill("0");
    } catch (InstantiationException e) {
        logger.error("InstantiationException.");
    } catch (IllegalAccessException e) {
        logger.error("IllegalAccessException.");
    }
}

Example 3 with PartitionRequest

use of org.goldenorb.jet.PartitionRequest in project goldenorb by jzachr.

the class TestPartitionRequest method testPartitionRequest.

/*
   * Start of user / non-generated code -- any code written outside of this block will be
   * removed in subsequent code generations.
   */
/* End of user / non-generated code */
@Before
public void testPartitionRequest() throws IOException {
    /*
     * Start of user / non-generated code -- any code written outside of this block will be
     * removed in subsequent code generations.
     */
    /* End of user / non-generated code */
    partitionRequest = new PartitionRequest();
    partitionRequest.setReservedPartitions(INT_RESERVEDPARTITIONS_VALUE);
    partitionRequest.setActivePartitions(INT_ACTIVEPARTITIONS_VALUE);
    partitionRequest.setJobID(STRING_JOBID_VALUE);
    partitionRequest.setBasePartitionID(INT_BASEPARTITIONID_VALUE);
    partitionRequest.setJobConf(new OrbConfiguration(true));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(baos);
    partitionRequest.write(out);
    DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    partitionRequestOut = new PartitionRequest();
    partitionRequestOut.readFields(in);
/*
     * Start of user / non-generated code -- any code written outside of this block will be
     * removed in subsequent code generations.
     */
/* End of user / non-generated code */
}