Examples with IMessage edu.iu.dsc.tws.api.compute.IMessage used on opensource projects

Example 1 with IMessage

use of edu.iu.dsc.tws.api.compute.IMessage in project twister2 by DSC-SPIDAL.

the class WindowManager method bundleNonExpiredWindowIMessage.

/**
 * This method bundles data into a IWindowMessage for creating non-expired IWindowMessages
 *
 * @param events list of elements that need to be passed into a window
 * @return a bundled IWindowMessage considering a list of IMessages of a given data type
 */
public IWindowMessage<T> bundleNonExpiredWindowIMessage(List<IMessage<T>> events) {
    WindowMessageImpl winMessage = null;
    List<IMessage<T>> messages = new ArrayList<>();
    for (IMessage<T> m : events) {
        messages.add(m);
    }
    winMessage = new WindowMessageImpl(messages);
    return winMessage;
}

Example 2 with IMessage

use of edu.iu.dsc.tws.api.compute.IMessage in project twister2 by DSC-SPIDAL.

the class WindowManager method scanEvents.

public List<Event<T>> scanEvents(boolean fullScan) {
    List<IMessage<T>> eventsToExpire = new ArrayList<>();
    List<Event<T>> eventsToProcess = new ArrayList<>();
    try {
        lock.lock();
        Iterator<Event<T>> it = queue.iterator();
        while (it.hasNext()) {
            Event<T> windowEvent = it.next();
            Action action = evictionPolicy.evict(windowEvent);
            if (action == Action.EXPIRE) {
                eventsToExpire.add(windowEvent.get());
                it.remove();
            } else if (!fullScan || action == Action.STOP) {
                break;
            } else if (action == Action.PROCESS) {
                eventsToProcess.add(windowEvent);
            }
        }
        expiredEvents.addAll(eventsToExpire);
    } finally {
        lock.unlock();
    }
    eventsSinceLastExpiration.set(0);
    if (!eventsToExpire.isEmpty()) {
        if (debug) {
            LOG.severe(String.format("OnExpiry called on WindowLifeCycleListener"));
        }
        IWindowMessage<T> eventsToExpireIWindow = bundleExpiredWindowIMessage(eventsToExpire);
        windowLifeCycleListener.onExpiry(eventsToExpireIWindow);
    }
    return eventsToProcess;
}

Example 3 with IMessage

use of edu.iu.dsc.tws.api.compute.IMessage in project twister2 by DSC-SPIDAL.

the class TaskBatchInstance method execute.

@Override
public boolean execute() {
    // we started the executio
    if (state.isSet(InstanceState.INIT) && state.isNotSet(InstanceState.EXECUTION_DONE)) {
        while (!inQueue.isEmpty() && outQueue.size() < lowWaterMark) {
            IMessage m = inQueue.poll();
            task.execute(m);
            state.addState(InstanceState.EXECUTING);
        }
        // for compute we don't have to have the context done as when the inputs finish and execution
        // is done, we are done executing
        // progress in communication
        boolean complete = isComplete(intOpArray);
        // if we no longer needs to progress comm and input is empty
        if (inQueue.isEmpty() && state.isSet(InstanceState.SYNCED) && complete) {
            task.endExecute();
            state.addState(InstanceState.EXECUTION_DONE);
        }
    }
    // now check the output queue
    while (!outQueue.isEmpty()) {
        IMessage message = outQueue.peek();
        if (message != null) {
            String edge = message.edge();
            // invoke the communication operation
            IParallelOperation op = outParOps.get(edge);
            int flags = 0;
            if (op.send(globalTaskId, message, flags)) {
                outQueue.poll();
            } else {
                // no point progressing further
                break;
            }
        }
    }
    // if execution is done and outqueue is emput, we have put everything to communication
    if (state.isSet(InstanceState.EXECUTION_DONE) && outQueue.isEmpty() && state.isNotSet(InstanceState.OUT_COMPLETE)) {
        for (IParallelOperation op : outParOps.values()) {
            op.finish(globalTaskId);
        }
        state.addState(InstanceState.OUT_COMPLETE);
    }
    // lets progress the communication
    boolean complete = isComplete(outOpArray);
    // after we have put everything to communication and no progress is required, lets finish
    if (state.isSet(InstanceState.OUT_COMPLETE) && complete) {
        state.addState(InstanceState.SENDING_DONE);
    }
    return !state.isSet(InstanceState.SENDING_DONE);
}

Example 4 with IMessage

use of edu.iu.dsc.tws.api.compute.IMessage in project twister2 by DSC-SPIDAL.

the class SourceBatchInstance method execute.

/**
 * Execution Method calls the SourceTasks run method to get context
 */
public boolean execute() {
    // we started the execution
    if (state.isEqual(InstanceState.INIT)) {
        state.addState(InstanceState.EXECUTING);
    }
    if (state.isSet(InstanceState.EXECUTING) && state.isNotSet(InstanceState.EXECUTION_DONE)) {
        // we loop until low watermark is reached or all edges are done
        while (outBatchQueue.size() < lowWaterMark) {
            // if we are in executing state we can run
            batchTask.execute();
            // if all the edges are done
            if (taskContext.isCompleted()) {
                state.addState(InstanceState.EXECUTION_DONE);
                break;
            }
        }
    }
    // now check the output queue
    while (!outBatchQueue.isEmpty()) {
        IMessage message = outBatchQueue.peek();
        if (message != null) {
            String edge = message.edge();
            IParallelOperation op = outBatchParOps.get(edge);
            if (op.send(globalTaskId, message, 0)) {
                outBatchQueue.poll();
            } else {
                // no point in progressing further
                break;
            }
        }
    }
    // if execution is done and outqueue is emput, we have put everything to communication
    if (state.isSet(InstanceState.EXECUTION_DONE) && outBatchQueue.isEmpty() && state.isNotSet(InstanceState.OUT_COMPLETE)) {
        for (IParallelOperation op : outBatchParOps.values()) {
            op.finish(globalTaskId);
        }
        state.addState(InstanceState.OUT_COMPLETE);
    }
    // lets progress the communication
    boolean complete = isComplete();
    // after we have put everything to communication and no progress is required, lets finish
    if (state.isSet(InstanceState.OUT_COMPLETE) && complete) {
        state.addState(InstanceState.SENDING_DONE);
    }
    boolean equal = state.isEqual(InstanceState.FINISH);
    return !equal;
}

Example 5 with IMessage

use of edu.iu.dsc.tws.api.compute.IMessage in project twister2 by DSC-SPIDAL.

the class TaskStreamingInstance method execute.

/**
 * Executing compute task
 */
public boolean execute() {
    // execute if there are incoming messages
    while (!inQueue.isEmpty() && outQueue.size() < lowWaterMark) {
        IMessage m = inQueue.poll();
        if (m != null) {
            task.execute(m);
        }
    }
    // set the initial nothing to execute
    boolean nothingToProcess = inQueue.isEmpty();
    // now check the output queue
    while (!outQueue.isEmpty()) {
        IMessage message = outQueue.peek();
        if (message != null) {
            String edge = message.edge();
            boolean barrierMessage = (message.getFlag() & MessageFlags.SYNC_BARRIER) == MessageFlags.SYNC_BARRIER;
            // invoke the communication operation
            IParallelOperation op = outParOps.get(edge);
            // if we successfully send remove
            if (barrierMessage ? op.sendBarrier(globalTaskId, (byte[]) message.getContent()) : op.send(globalTaskId, message, message.getFlag())) {
                outQueue.poll();
            } else {
                nothingToProcess = false;
                break;
            }
        }
    }
    for (int i = 0; i < outOpArray.length; i++) {
        boolean needProgress = outOpArray[i].progress();
        if (needProgress) {
            nothingToProcess = false;
        }
    }
    for (int i = 0; i < intOpArray.length; i++) {
        boolean needProgress = intOpArray[i].progress();
        if (needProgress) {
            nothingToProcess = false;
        }
    }
    if (this.checkpointable && this.inQueue.isEmpty() && this.outQueue.isEmpty()) {
        long checkpointedBarrierId = this.pendingCheckpoint.execute();
        if (checkpointedBarrierId != -1) {
            ((CheckpointableTask) this.task).onCheckpointPropagated(this.snapshot);
            taskContext.write(CheckpointingSGatherSink.FT_GATHER_EDGE, checkpointedBarrierId);
            this.scheduleBarriers(checkpointedBarrierId);
            nothingToProcess = false;
        }
    }
    return !nothingToProcess;
}