Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 56 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KeyedReceiver method offerMessage.

/**
 * saves the given message (or messages if the object is a list) into the messages data structure
 * if possible and rejects the message if the whole message cannot be added to the messages
 * data structure.
 *
 * @param target target for which the messages are to be added
 * @param object the message/messages to be added
 * @return true if the message was added or false otherwise
 */
@SuppressWarnings("rawtypes")
protected boolean offerMessage(int target, Object object) {
    Map<Object, Queue<Object>> messagesPerTarget = messages.get(target);
    if (!isFinalBatchReceiver && messagesPerTarget.size() > keyLimit) {
        LOG.fine(String.format("Executor %d Partial cannot add any further keys needs flush ", executor));
        moveMessagesToSendQueue(target, messagesPerTarget);
        return false;
    }
    if (object instanceof AggregatedObjects) {
        List dataList = (List) object;
        Map<Object, List<Object>> tempList = new HashMap<>();
        for (Object dataEntry : dataList) {
            Tuple tuple = (Tuple) dataEntry;
            // If any of the keys are full the method returns false because partial objects cannot be
            // added to the messages data structure
            Object key = tuple.getKey();
            if (!isFinalBatchReceiver && messagesPerTarget.containsKey(key) && messagesPerTarget.get(key).size() >= limitPerKey) {
                moveMessageToSendQueue(target, messagesPerTarget, tuple.getKey());
                LOG.fine(String.format("Executor %d Partial cannot add any further values for key " + "needs flush ", executor));
                return false;
            }
            if (tempList.containsKey(key)) {
                tempList.get(key).add(tuple.getValue());
            } else {
                tempList.put(key, new AggregatedObjects<>());
                tempList.get(key).add(tuple.getValue());
            }
        }
        boolean offerDone = true;
        for (Object key : tempList.keySet()) {
            if (messagesPerTarget.containsKey(key)) {
                List<Object> values = tempList.get(key);
                for (Object value : values) {
                    offerDone &= messagesPerTarget.get(key).offer(value);
                }
            } else {
                ArrayDeque<Object> messagesPerKey = new ArrayDeque<>();
                List<Object> values = tempList.get(key);
                for (Object value : values) {
                    offerDone &= messagesPerKey.offer(value);
                }
                messagesPerTarget.put(key, messagesPerKey);
            }
        }
        // cannot be recovered
        if (!offerDone) {
            throw new RuntimeException("Message lost during processing");
        }
    } else {
        Tuple tuple = (Tuple) object;
        if (messagesPerTarget.containsKey(tuple.getKey())) {
            if (messagesPerTarget.get(tuple.getKey()).size() < limitPerKey) {
                return messagesPerTarget.get(tuple.getKey()).offer(tuple.getValue());
            } else {
                LOG.fine(String.format("Executor %d Partial cannot add any further values for key " + "needs flush ", executor));
                moveMessageToSendQueue(target, messagesPerTarget, tuple.getKey());
                return false;
            }
        } else {
            ArrayDeque<Object> messagesPerKey = new ArrayDeque<>();
            messagesPerKey.add(tuple.getValue());
            messagesPerTarget.put(tuple.getKey(), messagesPerKey);
        }
    }
    return true;
}

Example 57 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class ControlledMemoryReader method next.

@Override
public Tuple next() {
    if (!this.hasNext()) {
        throw new NoSuchElementException("No more keys to iterate");
    }
    Tuple tuple = inMemoryMessages.get(readIndex);
    readIndex++;
    return tuple;
}

Example 58 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedMerger method deserializeObjects.

private void deserializeObjects() {
    for (int i = 0; i < recordsInMemory.size(); i++) {
        Tuple kv = recordsInMemory.get(i);
        Object o = dataType.getDataPacker().unpackFromByteArray((byte[]) kv.getValue());
        objectsInMemory.add(new Tuple(kv.getKey(), o));
    }
}

Example 59 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger2 method deserializeObjects.

private void deserializeObjects() {
    // this thread is also counted
    int threads = CommonThreadPool.getThreadCount() + 1;
    List<Future<Boolean>> deserializeFutures = new ArrayList<>();
    int chunkSize = this.recordsInMemory.size() / threads;
    if (this.recordsInMemory.size() % threads != 0) {
        chunkSize++;
    }
    for (int i = 0; i < this.recordsInMemory.size(); i += chunkSize) {
        final int start = i;
        final int end = Math.min(this.recordsInMemory.size(), i + chunkSize);
        // last chunk will be processed in this thread
        if (end == this.recordsInMemory.size()) {
            Iterator<Tuple> itr = recordsInMemory.listIterator(start);
            int count = 0;
            int elements = end - start;
            while (itr.hasNext() && count < elements) {
                Tuple tuple = itr.next();
                Object o = dataType.getDataPacker().unpackFromByteArray((byte[]) tuple.getValue());
                tuple.setValue(o);
                count++;
            }
        } else {
            deserializeFutures.add(CommonThreadPool.getExecutor().submit(() -> {
                Iterator<Tuple> itr = recordsInMemory.listIterator(start);
                int count = 0;
                int elements = end - start;
                while (itr.hasNext() && count < elements) {
                    Tuple tuple = itr.next();
                    Object o = dataType.getDataPacker().unpackFromByteArray((byte[]) tuple.getValue());
                    tuple.setValue(o);
                    count++;
                }
                return true;
            }));
        }
    }
    for (int i = 0; i < deserializeFutures.size(); i++) {
        try {
            deserializeFutures.get(i).get();
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException("Error in deserializing records in memory", e);
        }
    }
}

Example 60 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger2 method add.

/**
 * Add the data to the file
 */
public synchronized void add(Object key, byte[] data, int length) {
    if (status == FSStatus.READING) {
        throw new RuntimeException("Cannot add after switching to reading");
    }
    if (status == FSStatus.WRITING_MEMORY) {
        this.recordsInMemory.add(new Tuple(key, data));
        this.numOfBytesInMemory += data.length;
        // we switch to disk
        if (numOfBytesInMemory >= maxBytesToKeepInMemory) {
            LOG.info(String.format("Switching to write to disk memory %d >= maxMemory %d", numOfBytesInMemory, maxBytesToKeepInMemory));
            status = FSStatus.WRITING_DISK;
            this.numOfBytesInMemory = 0;
        }
    } else {
        this.recordsToDisk.add(new Tuple(key, data));
        this.numOfBytesInMemory += data.length;
    }
}