Examples with IParallelOperation edu.iu.dsc.tws.api.compute.executor.IParallelOperation used on opensource projects

Example 6 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class TaskStreamingInstance method execute.

/**
 * Executing compute task
 */
public boolean execute() {
    // execute if there are incoming messages
    while (!inQueue.isEmpty() && outQueue.size() < lowWaterMark) {
        IMessage m = inQueue.poll();
        if (m != null) {
            task.execute(m);
        }
    }
    // set the initial nothing to execute
    boolean nothingToProcess = inQueue.isEmpty();
    // now check the output queue
    while (!outQueue.isEmpty()) {
        IMessage message = outQueue.peek();
        if (message != null) {
            String edge = message.edge();
            boolean barrierMessage = (message.getFlag() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER;
            // invoke the communication operation
            IParallelOperation op = outParOps.get(edge);
            // if we successfully send remove
            if (barrierMessage ? op.sendBarrier(globalTaskId, (byte[]) message.getContent()) : op.send(globalTaskId, message, message.getFlag())) {
                outQueue.poll();
            } else {
                nothingToProcess = false;
                break;
            }
        }
    }
    for (int i = 0; i < outOpArray.length; i++) {
        boolean needProgress = outOpArray[i].progress();
        if (needProgress) {
            nothingToProcess = false;
        }
    }
    for (int i = 0; i < intOpArray.length; i++) {
        boolean needProgress = intOpArray[i].progress();
        if (needProgress) {
            nothingToProcess = false;
        }
    }
    if (this.checkpointable && this.inQueue.isEmpty() && this.outQueue.isEmpty()) {
        long checkpointedBarrierId = this.pendingCheckpoint.execute();
        if (checkpointedBarrierId != -1) {
            ((CheckpointableTask) this.task).onCheckpointPropagated(this.snapshot);
            taskContext.write(CheckpointingSGatherSink.FT_GATHER_EDGE, checkpointedBarrierId);
            this.scheduleBarriers(checkpointedBarrierId);
            nothingToProcess = false;
        }
    }
    return !nothingToProcess;
}

Example 7 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class TaskStreamingInstance method prepare.

/**
 * Preparing the task
 *
 * @param cfg configuration
 */
public void prepare(Config cfg) {
    outputCollection = new DefaultOutputCollection(outQueue);
    taskContext = new TaskContextImpl(taskIndex, taskId, globalTaskId, taskName, parallelism, workerId, outputCollection, nodeConfigs, inputEdges, outputEdges, taskSchedule, OperationMode.STREAMING);
    task.prepare(cfg, taskContext);
    // / we will use this array for iteration
    this.outOpArray = new IParallelOperation[outParOps.size()];
    int index = 0;
    for (Map.Entry<String, IParallelOperation> e : outParOps.entrySet()) {
        this.outOpArray[index++] = e.getValue();
    }
    this.outEdgeArray = new String[outputEdges.size()];
    index = 0;
    for (String e : outputEdges.keySet()) {
        this.outEdgeArray[index++] = e;
    }
    // / we will use this array for iteration
    this.intOpArray = new IParallelOperation[inParOps.size()];
    index = 0;
    for (Map.Entry<String, IParallelOperation> e : inParOps.entrySet()) {
        this.intOpArray[index++] = e.getValue();
    }
    this.inEdgeArray = new String[inputEdges.size()];
    index = 0;
    for (String e : inputEdges.keySet()) {
        this.inEdgeArray[index++] = e;
    }
    if (this.checkpointable) {
        this.stateStore = CheckpointUtils.getStateStore(config);
        this.stateStore.init(config, this.taskGraphName, String.valueOf(globalTaskId));
        this.pendingCheckpoint = new PendingCheckpoint(this.taskGraphName, (CheckpointableTask) this.task, this.globalTaskId, this.intOpArray, this.inEdgeArray.length, this.checkpointingClient, this.stateStore, this.snapshot);
        TaskCheckpointUtils.restore((CheckpointableTask) this.task, this.snapshot, this.stateStore, this.tasksVersion, globalTaskId);
    }
}

Example 8 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class StreamingAllSharingExecutor2 method close.

private void close(ExecutionPlan executionPlan, Map<Integer, INodeInstance> nodes) {
    // lets wait for thread to finish
    try {
        doneSignal.await();
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted", e);
    }
    List<IParallelOperation> ops = executionPlan.getParallelOperations();
    resetNodes(nodes, ops);
    // clean up the instances
    for (INodeInstance node : nodes.values()) {
        node.close();
    }
    // lets close the operations
    for (IParallelOperation op : ops) {
        op.close();
    }
    executionHook.onClose(this);
    // clear the finished instances
    cleanUpCalled = true;
}

Example 9 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class StreamingAllSharingExecutor method close.

private void close(ExecutionPlan executionPlan, Map<Integer, INodeInstance> nodes) {
    // lets wait for thread to finish
    try {
        doneSignal.await();
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted", e);
    }
    List<IParallelOperation> ops = executionPlan.getParallelOperations();
    resetNodes(nodes, ops);
    // clean up the instances
    for (INodeInstance node : nodes.values()) {
        node.close();
    }
    // lets close the operations
    for (IParallelOperation op : ops) {
        op.close();
    }
    executionHook.onClose(this);
    // clear the finished instances
    cleanUpCalled = true;
}

Example 10 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class StreamingSharingExecutor method cleanUp.

private void cleanUp(Map<Integer, INodeInstance> nodes) {
    // lets wait for thread to finish
    try {
        doneSignal.await();
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted", e);
    }
    // clean up the instances
    for (INodeInstance node : nodes.values()) {
        node.close();
    }
    // lets close the operations
    List<IParallelOperation> ops = executionPlan.getParallelOperations();
    for (IParallelOperation op : ops) {
        op.close();
    }
    // execution hook
    executionHook.afterExecution();
    cleanUpCalled = true;
}