Examples with Twister2RuntimeException edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException used on opensource projects

Example 31 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class WorkerManager method execute.

/**
 * Execute IWorker
 * return false if IWorker fails fully after retries
 * return true if execution successful
 * throw an exception if execution fails and the worker needs to be restarted from jvm
 */
public boolean execute() {
    while (JobProgress.getWorkerExecuteCount() < maxRetries) {
        LOG.info("Waiting on the init barrier before starting IWorker: " + workerID + " with restartCount: " + workerController.workerRestartCount() + " and with re-executionCount: " + JobProgress.getWorkerExecuteCount());
        try {
            workerController.waitOnInitBarrier();
            firstInitBarrierProceeded = true;
        } catch (TimeoutException e) {
            throw new Twister2RuntimeException("Could not pass through the init barrier", e);
        }
        LOG.fine("Proceeded through INIT barrier. Starting Worker: " + workerID);
        JobProgressImpl.setJobStatus(JobProgress.JobStatus.EXECUTING);
        JobProgressImpl.increaseWorkerExecuteCount();
        JobProgressImpl.setRestartedWorkers(restartedWorkers.values());
        try {
            managedWorker.execute(config, job, workerController, persistentVolume, volatileVolume);
        } catch (JobFaultyException cue) {
            // a worker in the cluster should have failed
            // we will try to re-execute this worker
            JobProgressImpl.setJobStatus(JobProgress.JobStatus.FAULTY);
            LOG.warning("thrown JobFaultyException. Some workers should have failed.");
        }
        // we need to make sure whether that all workers finished successfully also
        if (JobProgress.isJobHealthy()) {
            try {
                // wait on the barrier indefinitely until all workers arrive
                // or the barrier is broken with with a job fault
                LOG.info("Worker completed, waiting for other workers to finish at the final barrier.");
                workerController.waitOnBarrier(Long.MAX_VALUE);
                LOG.info("Worker finished successfully");
                return true;
            } catch (TimeoutException e) {
                // this should never happen
                throw new Twister2RuntimeException("Could not pass through the final barrier", e);
            } catch (JobFaultyException e) {
                JobProgressImpl.setJobStatus(JobProgress.JobStatus.FAULTY);
                LOG.warning("thrown JobFaultyException. Some workers failed before finishing.");
            }
        }
    }
    LOG.info(String.format("Re-executed IWorker %d times and failed, we are exiting", maxRetries));
    return false;
}

Example 32 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class CDFWExecutor method submitGraph.

private void submitGraph(DataFlowGraph dataFlowgraph, Set<Integer> workerIDs) {
    if (driverState == DriverState.INITIALIZE || driverState == DriverState.JOB_FINISHED) {
        try {
            // build the schedule plan for the dataflow graph
            DataFlowGraph dataFlowGraph = buildCDFWSchedulePlan(dataFlowgraph, workerIDs);
            CDFWJobAPI.SubGraph job = buildCDFWJob(dataFlowGraph);
            // now submit the job
            submitJob(job);
            driverState = DriverState.JOB_SUBMITTED;
            // lets wait for another event
            waitForEvent(DriveEventType.FINISHED_JOB);
            driverState = DriverState.JOB_FINISHED;
        } catch (Exception e) {
            throw new Twister2RuntimeException("Driver is not initialized", e);
        }
    } else {
        throw new Twister2RuntimeException("Failed to submit job in this state: " + driverState);
    }
}

Example 33 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class CDFWExecutor method executeCDFW.

/**
 * The executeCDFW method first call the schedule method to get the schedule list of the CDFW.
 * Then, it invokes the buildCDFWJob method to build the job object for the scheduled graphs.
 */
public void executeCDFW(DataFlowGraph... graph) {
    if (!(driverState == DriverState.JOB_FINISHED || driverState == DriverState.INITIALIZE)) {
        // now we need to send messages
        throw new RuntimeException("Invalid state to execute a job: " + driverState);
    }
    CDFWScheduler cdfwScheduler = new CDFWScheduler(this.executionEnv.getWorkerInfoList());
    Map<DataFlowGraph, Set<Integer>> scheduleGraphMap = cdfwScheduler.schedule(graph);
    ScheduledExecutorService executor = Executors.newScheduledThreadPool(scheduleGraphMap.size());
    for (Map.Entry<DataFlowGraph, Set<Integer>> entry : scheduleGraphMap.entrySet()) {
        CDFWExecutorTask cdfwSchedulerTask = new CDFWExecutorTask(entry.getKey(), entry.getValue());
        executor.submit(cdfwSchedulerTask);
    }
    try {
        executor.awaitTermination(1, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        throw new Twister2RuntimeException(e);
    } finally {
        executor.shutdown();
    }
}

Example 34 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class TaskExecutor method distributeData.

/**
 * This method distributes collected {@link DataPartition}s to the
 * intended {@link Receptor}s
 */
public static void distributeData(ExecutionPlan executionPlan, Map<String, DataObject> dataMap) {
    Map<Integer, INodeInstance> nodes = executionPlan.getNodes();
    if (nodes != null) {
        nodes.forEach((id, node) -> {
            INode task = node.getNode();
            if (task instanceof Receptor) {
                Set<String> receivableNames = ((Receptor) task).getReceivableNames();
                for (String receivableName : receivableNames) {
                    DataObject dataObject = dataMap.get(receivableName);
                    if (dataObject == null) {
                        throw new Twister2RuntimeException("Couldn't find input data" + receivableName + " for task " + node.getId());
                    }
                    DataPartition partition = dataObject.getPartition(node.getIndex());
                    if (partition == null) {
                        throw new Twister2RuntimeException("Couldn't find input data" + receivableName + " for task index " + node.getIndex() + " of task" + node.getId());
                    }
                    ((Receptor) task).add(receivableName, dataObject);
                    ((Receptor) task).add(receivableName, partition);
                }
            }
        });
    }
}

Example 35 with Twister2RuntimeException

use of edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException in project twister2 by DSC-SPIDAL.

the class RowItrComputeCollectorOp method prepare.

@Override
public void prepare(Config cfg, TaskContext ctx) {
    super.prepare(cfg, ctx);
    runtime = WorkerEnvironment.getSharedValue(TableRuntime.TABLE_RUNTIME_CONF, TableRuntime.class);
    if (runtime == null) {
        throw new Twister2RuntimeException("Table runtime must be set");
    }
    schema = (RowSchema) ctx.getConfig(TSetConstants.OUTPUT_SCHEMA_KEY);
    tableMaxSize = cfg.getLongValue("twister2.table.max.size", tableMaxSize);
    builder = new ArrowTableBuilder(schema.toArrowSchema(), runtime.getRootAllocator());
    collectorImp = new CollectorImp();
}