Examples with IParallelOperation edu.iu.dsc.tws.api.compute.executor.IParallelOperation used on opensource projects

Example 11 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class TaskBatchInstance method prepare.

public void prepare(Config cfg) {
    outputCollection = new DefaultOutputCollection(outQueue);
    taskContext = new TaskContextImpl(taskIndex, taskId, globalTaskId, taskName, parallelism, workerId, outputCollection, nodeConfigs, inputEdges, outputEdges, taskSchedule, OperationMode.BATCH);
    task.prepare(cfg, taskContext);
    // / we will use this array for iteration
    this.outOpArray = new IParallelOperation[outParOps.size()];
    int index = 0;
    for (Map.Entry<String, IParallelOperation> e : outParOps.entrySet()) {
        this.outOpArray[index++] = e.getValue();
    }
    this.outEdgeArray = new String[outputEdges.size()];
    index = 0;
    for (String e : outputEdges.keySet()) {
        this.outEdgeArray[index++] = e;
    }
    // / we will use this array for iteration
    this.intOpArray = new IParallelOperation[inParOps.size()];
    index = 0;
    for (Map.Entry<String, IParallelOperation> e : inParOps.entrySet()) {
        this.intOpArray[index++] = e.getValue();
    }
    this.inEdgeArray = new String[inputEdges.size()];
    index = 0;
    for (String e : inputEdges.keySet()) {
        this.inEdgeArray[index++] = e;
    }
}

Example 12 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class SourceBatchInstance method prepare.

public void prepare(Config cfg) {
    outputBatchCollection = new DefaultOutputCollection(outBatchQueue);
    taskContext = new TaskContextImpl(batchTaskIndex, taskId, globalTaskId, batchTaskName, parallelism, workerId, outputBatchCollection, nodeConfigs, outputEdges, taskSchedule, OperationMode.BATCH);
    batchTask.prepare(cfg, taskContext);
    // / we will use this array for iteration
    this.outOpArray = new IParallelOperation[outBatchParOps.size()];
    int index = 0;
    for (Map.Entry<String, IParallelOperation> e : outBatchParOps.entrySet()) {
        this.outOpArray[index++] = e.getValue();
    }
    this.outEdgeArray = new String[outputEdges.size()];
    index = 0;
    for (String e : outputEdges.keySet()) {
        this.outEdgeArray[index++] = e;
    }
}

Example 13 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class SourceStreamingInstance method execute.

/**
 * Execution Method calls the SourceTasks run method to get context
 */
public boolean execute() {
    if (outStreamingQueue.size() < lowWaterMark && !(this.checkpointable && this.pendingCheckpoint.isPending())) {
        // lets execute the task
        streamingTask.execute();
        if (this.checkpointable && (executions++ % this.checkPointingFrequency) == 0) {
            this.scheduleCheckpoint(checkpointVersion++);
        }
    }
    // we start with setting nothing to execute true
    boolean nothingToProcess = true;
    // now check the output queue
    while (!outStreamingQueue.isEmpty()) {
        IMessage message = outStreamingQueue.peek();
        if (message != null) {
            String edge = message.edge();
            IParallelOperation op = outStreamingParOps.get(edge);
            boolean barrierMessage = (message.getFlag() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER;
            // if we successfully send remove message
            if (barrierMessage ? op.sendBarrier(globalTaskId, (byte[]) message.getContent()) : op.send(globalTaskId, message, message.getFlag())) {
                outStreamingQueue.poll();
            } else {
                nothingToProcess = false;
                // we need to break
                break;
            }
        } else {
            break;
        }
    }
    for (int i = 0; i < outOpArray.length; i++) {
        boolean needProgress = outOpArray[i].progress();
        if (needProgress) {
            nothingToProcess = false;
        }
    }
    if (this.checkpointable && outStreamingQueue.isEmpty() && this.pendingCheckpoint.isPending()) {
        long barrier = this.pendingCheckpoint.execute();
        if (barrier != -1) {
            ((CheckpointableTask) this.streamingTask).onCheckpointPropagated(this.snapshot);
            this.scheduleBarriers(barrier);
            taskContext.write(CheckpointingSGatherSink.FT_GATHER_EDGE, barrier);
            nothingToProcess = false;
        }
    }
    return !nothingToProcess;
}

Example 14 with IParallelOperation

use of edu.iu.dsc.tws.api.compute.executor.IParallelOperation in project twister2 by DSC-SPIDAL.

the class BatchSharingExecutor2 method close.

private void close(ExecutionPlan executionPlan, Map<Integer, INodeInstance> nodes) {
    // lets wait for thread to finish
    try {
        doneSignal.await();
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted", e);
    }
    List<IParallelOperation> ops = executionPlan.getParallelOperations();
    resetNodes(nodes, ops);
    // clean up the instances
    for (INodeInstance node : nodes.values()) {
        node.close();
    }
    // lets close the operations
    for (IParallelOperation op : ops) {
        op.close();
    }
    // clear the finished instances
    finishedInstances.set(0);
    cleanUpCalled = true;
}