Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 1 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchFinalReceiver method merge.

@Override
protected void merge(int dest, List<Object> dests) {
    Map<Object, Object> targetValues = reduced.get(dest);
    for (int i = 0; i < dests.size(); i++) {
        Object val = dests.get(i);
        Tuple t;
        if (val instanceof Tuple) {
            t = (Tuple) val;
        } else {
            throw new RuntimeException("Un-expected type: " + val.getClass());
        }
        Object currentVal = targetValues.get(t.getKey());
        if (currentVal != null) {
            Object newVal = reduceFunction.reduce(currentVal, t.getValue());
            targetValues.put(t.getKey(), newVal);
        } else {
            targetValues.put(t.getKey(), t.getValue());
        }
    }
    dests.clear();
}

Example 2 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchReceiver method moveMessagesToSendQueue.

/**
 * moves all the buffered messages into the sendQueue for the given target, this method assumes
 * that for each target that there is only one object in the queue. This is required when working
 * with reduce operations
 *
 * @param target target for which the move needs to be done
 * @return true if the messagesPerTarget is not empty at the end of the moving process or false
 * otherwise
 */
@Override
protected boolean moveMessagesToSendQueue(int target, Map<Object, Queue<Object>> messagesPerTarget) {
    Queue<Object> targetSendQueue = sendQueue.get(target);
    messagesPerTarget.entrySet().removeIf(entry -> {
        Tuple send = new Tuple(entry.getKey(), entry.getValue().peek());
        return targetSendQueue.offer(send);
    });
    return messagesPerTarget.isEmpty();
}

Example 3 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchReceiver method offerMessage.

/**
 * The reduce operation overrides the offer method because the reduce operation
 * does not save all the incoming messages, it rather reduces messages with the same key and
 * saves only the reduced values. So the messages data structure will only have a single
 * entry for each target, key pair
 *
 * @param target target for which the messages are to be added
 * @param object the message/messages to be added
 * @return true if the message was added or false otherwise
 */
@Override
@SuppressWarnings("rawtypes")
protected boolean offerMessage(int target, Object object) {
    Map<Object, Queue<Object>> messagesPerTarget = messages.get(target);
    if (!isFinalBatchReceiver && messagesPerTarget.size() > keyLimit) {
        LOG.fine(String.format("Executor %d Partial cannot add any further keys needs flush ", executor));
        moveMessagesToSendQueue(target, messagesPerTarget);
        return false;
    }
    if (object instanceof List) {
        List dataList = (List) object;
        for (Object dataEntry : dataList) {
            Tuple tuple = (Tuple) dataEntry;
            if (!reduceAndInsert(messagesPerTarget, tuple)) {
                throw new RuntimeException("Reduce operation should not fail to insert key");
            }
        }
    } else {
        Tuple tuple = (Tuple) object;
        if (!reduceAndInsert(messagesPerTarget, tuple)) {
            throw new RuntimeException("Reduce operation should not fail to insert key");
        }
    }
    return true;
}

Example 4 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method openPart.

/**
 * Reads a file part upto max size from start offset
 *
 * @param fileName name
 * @param startOffSet start offset in bytes
 * @param maxSize max size to read
 * @param keyType key type
 * @param dataType data type
 * @param deserializer the deserializer
 * @return OpenFilePart with read information and null if fails
 */
public static OpenFilePart openPart(String fileName, long startOffSet, long maxSize, MessageType keyType, MessageType dataType, KryoSerializer deserializer) {
    List<Tuple> keyValues = new ArrayList<>();
    String outFileName = Paths.get(fileName).toString();
    FileChannel rwChannel;
    try {
        rwChannel = new RandomAccessFile(outFileName, "rw").getChannel();
        long size = maxSize <= rwChannel.size() - startOffSet ? maxSize : rwChannel.size() - startOffSet;
        ByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_ONLY, startOffSet, size);
        long totalRead = 0;
        while (totalRead < size) {
            Object key;
            Object value;
            // for object type we have to read the length of the bytes first
            if (keyType == MessageTypes.OBJECT && totalRead + Integer.BYTES > size) {
                break;
            }
            // for object type we read the object bytes + 4
            int keySize = getKeySize(keyType, os);
            // we cannot read further
            if (totalRead + keySize > size) {
                break;
            }
            key = keyType.getDataPacker().unpackFromBuffer(os, keySize - Integer.BYTES);
            if (totalRead + keySize + Integer.BYTES > size) {
                break;
            }
            int dataSize = os.getInt();
            // we cannot read further
            if (totalRead + keySize + dataSize + Integer.BYTES > size) {
                break;
            }
            value = dataType.getDataPacker().unpackFromBuffer(os, dataSize);
            keyValues.add(new Tuple(key, value));
            totalRead += Integer.BYTES + keySize + dataSize;
        }
        long size1 = rwChannel.size();
        rwChannel.close();
        return new OpenFilePart(keyValues, totalRead + startOffSet, size1, fileName);
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Error in reading file part", e);
        throw new RuntimeException(e);
    }
}

Example 5 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method saveKeyValues.

/**
 * Save the list of records to the file system
 *
 * @param records records to be written
 * @param size total size of the records
 * @param outFileName out file name
 * @return maximum size of the tuple written to this file
 */
public static long saveKeyValues(List<Tuple> records, List<Integer> sizes, long size, String outFileName, MessageType keyType) {
    try {
        // max size of a tuple saved to this file
        long maxRecord = Long.MIN_VALUE;
        // first serialize keys
        long totalSize = 0;
        List<byte[]> byteKeys = new ArrayList<>();
        if (keyType.isPrimitive() && !keyType.isArray()) {
            totalSize += records.size() * keyType.getUnitSizeInBytes();
        } else {
            for (Tuple record : records) {
                byte[] data = keyType.getDataPacker().packToByteArray(record.getKey());
                // data + length of key
                totalSize += data.length;
                if (keyType.getDataPacker().isHeaderRequired()) {
                    totalSize += Integer.BYTES;
                }
                byteKeys.add(data);
            }
        }
        // just to check whether sizes match
        long sizeSum = 0;
        // we need to write the data lengths and key lengths
        int dataLengthSize = Integer.BYTES * records.size();
        totalSize += size + dataLengthSize;
        Files.createDirectories(Paths.get(outFileName).getParent());
        RandomAccessFile randomAccessFile = new RandomAccessFile(outFileName, "rw");
        FileChannel rwChannel = randomAccessFile.getChannel();
        MappedByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, totalSize);
        for (int i = 0; i < records.size(); i++) {
            // position of os before writing this tuple
            long positionBefore = os.position();
            Tuple keyValue = records.get(i);
            // this has been already serialized
            byte[] r = (byte[]) keyValue.getValue();
            if (keyType.isPrimitive() && !keyType.isArray()) {
                keyType.getDataPacker().packToByteBuffer(os, keyValue.getKey());
            } else {
                byte[] key = byteKeys.get(i);
                if (keyType.getDataPacker().isHeaderRequired()) {
                    os.putInt(key.length);
                }
                os.put(key);
            }
            sizeSum += sizes.get(i);
            os.putInt(sizes.get(i));
            os.put(r, 0, sizes.get(i));
            long tupleSize = os.position() - positionBefore;
            maxRecord = Math.max(maxRecord, tupleSize);
        }
        if (sizeSum != size) {
            LOG.log(Level.WARNING, "Sum doesn't equal size: " + sizeSum + " != " + size);
        }
        rwChannel.close();
        randomAccessFile.close();
        try {
            MemoryMapUtils.unMapBuffer(os);
        } catch (Exception e) {
            // ignore
            LOG.warning("Couldn't manually unmap a byte buffer");
        }
        return maxRecord;
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Failed write to disc", e);
        throw new RuntimeException(e);
    }
}