Examples with JobReturn org.apache.sysml.runtime.matrix.JobReturn used on opensource projects

Example 11 with JobReturn

use of org.apache.sysml.runtime.matrix.JobReturn in project incubator-systemml by apache.

the class PiggybackingWorker method getJobResult.

public synchronized JobReturn getJobResult(long instID) throws InterruptedException {
    JobReturn ret = null;
    while (ret == null) {
        // wait for new results
        wait();
        // obtain job return (if available)
        ret = _results.remove(instID);
    }
    return ret;
}

Example 12 with JobReturn

use of org.apache.sysml.runtime.matrix.JobReturn in project incubator-systemml by apache.

the class PiggybackingWorkerTimeSequential method run.

@Override
public void run() {
    long lastTime = System.currentTimeMillis();
    while (!_stop) {
        try {
            // wait until next submission
            if (SUBSTRACT_EXEC_TIME) {
                long currentTime = System.currentTimeMillis();
                if (currentTime - lastTime < _time)
                    Thread.sleep(_time - (currentTime - lastTime));
                lastTime = currentTime;
            } else
                Thread.sleep(_time);
            // pick job type with largest number of jobs
            LinkedList<Pair<Long, MRJobInstruction>> workingSet = RuntimePiggybacking.getMaxWorkingSet();
            if (workingSet == null)
                // empty pool
                continue;
            // merge jobs (if possible)
            LinkedList<MergedMRJobInstruction> mergedWorkingSet = mergeMRJobInstructions(workingSet);
            // submit all resulting jobs (currently sequential submission)
            for (MergedMRJobInstruction minst : mergedWorkingSet) {
                JobReturn mret = RunMRJobs.submitJob(minst.inst);
                Statistics.incrementNoOfExecutedMRJobs();
                // error handling
                if (!mret.successful)
                    LOG.error("Failed to run merged mr-job instruction:\n" + minst.inst.toString());
                // split job return
                LinkedList<JobReturn> ret = new LinkedList<>();
                for (Long id : minst.ids) {
                    ret.add(minst.constructJobReturn(id, mret));
                    Statistics.decrementNoOfExecutedMRJobs();
                }
                // make job returns available and notify waiting clients
                putJobResults(minst.ids, ret);
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }
}

Example 13 with JobReturn

use of org.apache.sysml.runtime.matrix.JobReturn in project incubator-systemml by apache.

the class RuntimePiggybacking method submitJob.

public static JobReturn submitJob(MRJobInstruction inst) {
    JobReturn ret = null;
    try {
        // step 1: obtain job id
        long id = _idSeq.getNextID();
        // step 2: append mr job to global pool
        synchronized (_pool) {
            // maintain job-type partitioned instruction pool
            if (!_pool.containsKey(inst.getJobType()))
                _pool.put(inst.getJobType(), new LinkedList<Long>());
            _pool.get(inst.getJobType()).add(id);
            // add actual mr job instruction
            _jobs.put(id, inst);
        }
        // step 3: wait for finished job
        ret = _worker.getJobResult(id);
        if (!ret.successful)
            throw new DMLRuntimeException("Failed to run MR job via runtime piggybacking - job unsuccessful:\n" + inst.toString());
    } catch (InterruptedException ex) {
        throw new DMLRuntimeException("Failed to submit MR job to runtime piggybacking server.", ex);
    }
    return ret;
}

Example 14 with JobReturn

use of org.apache.sysml.runtime.matrix.JobReturn in project incubator-systemml by apache.

the class MRJobInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE)
        throw new DMLRuntimeException("MapReduce jobs cannot be executed when execution mode = singlenode");
    // execute MR job
    JobReturn jb = RunMRJobs.prepareAndSubmitJob(this, ec);
    // specific post processing
    if (getJobType() == JobType.SORT && jb.getMetaData().length > 0) {
        /* Populate returned stats into symbol table of matrices */
        for (int index = 0; index < jb.getMetaData().length; index++) {
            String varname = getOutputVars()[index];
            ec.setMetaData(varname, jb.getMetaData()[index]);
        }
    } else if (jb.getMetaData().length > 0) {
        /* Populate returned stats into symbol table of matrices */
        for (int index = 0; index < jb.getMetaData().length; index++) {
            String varname = getOutputVars()[index];
            MatrixCharacteristics mc = jb.getMetaData(index).getMatrixCharacteristics();
            ec.getVariable(varname).updateMatrixCharacteristics(mc);
        }
    }
    Statistics.incrementNoOfExecutedMRJobs();
}