Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 16 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceStreamingReceiver method offerMessage.

/**
 * The reduce operation overrides the offer method because the reduce operation
 * does not save all the incoming messages, it rather reduces messages with the same key and
 * saves only the reduced values. So the messages data structure will only have a single
 * entry for each target, key pair
 *
 * @param target target for which the messages are to be added
 * @param object the message/messages to be added
 * @return true if the message was added or false otherwise
 */
@Override
@SuppressWarnings("rawtypes")
protected boolean offerMessage(int target, Object object) {
    Map<Object, Queue<Object>> messagesPerTarget = messages.get(target);
    if (messagesPerTarget.size() > keyLimit) {
        LOG.fine(String.format("Executor %d Partial cannot add any further keys needs flush ", executor));
        moveMessagesToSendQueue(target, messagesPerTarget);
        return false;
    }
    if (object instanceof List) {
        List dataList = (List) object;
        for (Object dataEntry : dataList) {
            Tuple tuple = (Tuple) dataEntry;
            if (!reduceAndInsert(messagesPerTarget, tuple)) {
                throw new RuntimeException("Reduce operation should not fail to insert key");
            }
            this.localWindowCount++;
        }
    } else {
        Tuple tuple = (Tuple) object;
        if (!reduceAndInsert(messagesPerTarget, tuple)) {
            throw new RuntimeException("Reduce operation should not fail to insert key");
        }
        this.localWindowCount++;
    }
    if (localWindowCount > windowSize) {
        if (moveMessagesToSendQueue(target, messagesPerTarget)) {
            // TODO: what if the move returns false, do we still set the localWindowCount to zero?
            localWindowCount = 0;
        }
    }
    return true;
}

Example 17 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceStreamingReceiver method moveMessagesToSendQueue.

/**
 * moves all the buffered messages into the sendQueue for the given target, this method assumes
 * that for each target that there is only one object in the queue. This is required when working
 * with reduce operations
 *
 * @param target target for which the move needs to be done
 * @return true if the messagesPerTarget is not empty at the end of the moving process or false
 * otherwise
 */
@Override
protected boolean moveMessagesToSendQueue(int target, Map<Object, Queue<Object>> messagesPerTarget) {
    Queue<Object> targetSendQueue = sendQueue.get(target);
    messagesPerTarget.entrySet().removeIf(entry -> {
        Tuple send = new Tuple(entry.getKey(), entry.getValue().peek());
        return targetSendQueue.offer(send);
    });
    return messagesPerTarget.isEmpty();
}

Example 18 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class DPartitionBatchFinalReceiver method onMessage.

@Override
@SuppressWarnings("unchecked")
public boolean onMessage(int source, int path, int target, int flags, Object object) {
    if (lock.tryLock()) {
        try {
            Shuffle sortedMerger = sortedMergers.get(target);
            if (sortedMerger == null) {
                throw new RuntimeException("Un-expected target id: " + target);
            }
            if (targetStates.get(target) == ReceiverState.INIT) {
                targetStates.put(target, ReceiverState.RECEIVING);
            }
            if ((flags & MessageFlags.SYNC_EMPTY) == MessageFlags.SYNC_EMPTY) {
                Set<Integer> finished = finishedSources.get(target);
                if (finished.contains(source)) {
                    LOG.log(Level.FINE, String.format("%d Duplicate finish from source id %d -> %d", this.thisWorker, source, target));
                } else {
                    finished.add(source);
                }
                if (finished.size() == partition.getSources().size()) {
                    if (!finishedTargets.contains(target)) {
                        finishedTargets.add(target);
                    }
                    targetStates.put(target, ReceiverState.ALL_SYNCS_RECEIVED);
                }
                return true;
            }
            if (targetStates.get(target) == ReceiverState.ALL_SYNCS_RECEIVED || targetStates.get(target) == ReceiverState.SYNCED) {
                return false;
            }
            // add the object to the map
            if (keyed) {
                List<Tuple> tuples = (List<Tuple>) object;
                for (Tuple kc : tuples) {
                    Object data = kc.getValue();
                    byte[] d;
                    if (partition.getReceiveDataType() != MessageTypes.BYTE_ARRAY || !(data instanceof byte[]) || ((flags & MessageFlags.ORIGIN_SENDER) == MessageFlags.ORIGIN_SENDER && partition.getDataType() == MessageTypes.OBJECT)) {
                        // 3rd case handles, when user use Object data type, but send a byte[]
                        d = partition.getDataType().getDataPacker().packToByteArray(data);
                        kc.setValue(d);
                    }
                    sortedMerger.add(kc);
                }
            } else {
                List<Object> contents = (List<Object>) object;
                for (Object kc : contents) {
                    byte[] d;
                    if (partition.getReceiveDataType() != MessageTypes.BYTE_ARRAY || !(kc instanceof byte[]) || ((flags & MessageFlags.ORIGIN_SENDER) == MessageFlags.ORIGIN_SENDER && partition.getDataType() == MessageTypes.OBJECT)) {
                        d = partition.getDataType().getDataPacker().packToByteArray(kc);
                    } else {
                        d = (byte[]) kc;
                    }
                    sortedMerger.add(d, d.length);
                }
            }
            return true;
        } finally {
            lock.unlock();
        }
    }
    return false;
}

Example 19 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger method add.

/**
 * Add the data to the file
 */
public void add(Object key, byte[] data, int length) {
    if (status == FSStatus.READING) {
        throw new RuntimeException("Cannot add after switching to reading");
    }
    lock.lock();
    try {
        recordsInMemory.add(new Tuple(key, data));
        bytesLength.add(length);
        numOfBytesInMemory += length;
    } finally {
        lock.unlock();
    }
}

Example 20 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger method readIterator.

/**
 * This method gives the values
 */
public Iterator<Object> readIterator() {
    return new Iterator<Object>() {

        private FSIterator fsIterator = new FSIterator();

        private Tuple nextTuple = fsIterator.hasNext() ? fsIterator.next() : null;

        private Iterator itOfCurrentKey = null;

        private void skipKeys() {
            // user is trying to skip keys. For now, we are iterating over them internally
            if (this.itOfCurrentKey != null) {
                // todo replace with an alternative approach
                while (this.itOfCurrentKey.hasNext()) {
                    this.itOfCurrentKey.next();
                }
            }
        }

        @Override
        public boolean hasNext() {
            this.skipKeys();
            return nextTuple != null;
        }

        @Override
        public Tuple<Object, Iterator> next() {
            if (!hasNext()) {
                throw new NoSuchElementException("There are no more keys to iterate");
            }
            final Object currentKey = nextTuple.getKey();
            this.itOfCurrentKey = new Iterator<Object>() {

                @Override
                public boolean hasNext() {
                    return nextTuple != null && nextTuple.getKey().equals(currentKey);
                }

                @Override
                public Object next() {
                    if (this.hasNext()) {
                        Object returnValue = nextTuple.getValue();
                        if (fsIterator.hasNext()) {
                            nextTuple = fsIterator.next();
                        } else {
                            nextTuple = null;
                        }
                        return returnValue;
                    } else {
                        throw new NoSuchElementException("There are no more values for key " + currentKey);
                    }
                }
            };
            Tuple<Object, Iterator> nextValueSet = new Tuple<>();
            nextValueSet.setKey(currentKey);
            nextValueSet.setValue(this.itOfCurrentKey);
            return nextValueSet;
        }
    };
}