Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 11 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchReceiver method offerMessage.

/**
 * The reduce operation overrides the offer method because the reduce operation
 * does not save all the incoming messages, it rather reduces messages with the same key and
 * saves only the reduced values. So the messages data structure will only have a single
 * entry for each target, key pair
 *
 * @param target target for which the messages are to be added
 * @param object the message/messages to be added
 * @return true if the message was added or false otherwise
 */
@Override
@SuppressWarnings("rawtypes")
protected boolean offerMessage(int target, Object object) {
    Map<Object, Queue<Object>> messagesPerTarget = messages.get(target);
    if (!isFinalBatchReceiver && messagesPerTarget.size() > keyLimit) {
        LOG.fine(String.format("Executor %d Partial cannot add any further keys needs flush ", executor));
        moveMessagesToSendQueue(target, messagesPerTarget);
        return false;
    }
    if (object instanceof List) {
        List dataList = (List) object;
        for (Object dataEntry : dataList) {
            Tuple tuple = (Tuple) dataEntry;
            if (!reduceAndInsert(messagesPerTarget, tuple)) {
                throw new RuntimeException("Reduce operation should not fail to insert key");
            }
        }
    } else {
        Tuple tuple = (Tuple) object;
        if (!reduceAndInsert(messagesPerTarget, tuple)) {
            throw new RuntimeException("Reduce operation should not fail to insert key");
        }
    }
    return true;
}

Example 12 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method openPart.

/**
 * Reads a file part upto max size from start offset
 *
 * @param fileName name
 * @param startOffSet start offset in bytes
 * @param maxSize max size to read
 * @param keyType key type
 * @param dataType data type
 * @param deserializer the deserializer
 * @return OpenFilePart with read information and null if fails
 */
public static OpenFilePart openPart(String fileName, long startOffSet, long maxSize, MessageType keyType, MessageType dataType, KryoSerializer deserializer) {
    List<Tuple> keyValues = new ArrayList<>();
    String outFileName = Paths.get(fileName).toString();
    FileChannel rwChannel;
    try {
        rwChannel = new RandomAccessFile(outFileName, "rw").getChannel();
        long size = maxSize <= rwChannel.size() - startOffSet ? maxSize : rwChannel.size() - startOffSet;
        ByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_ONLY, startOffSet, size);
        long totalRead = 0;
        while (totalRead < size) {
            Object key;
            Object value;
            // for object type we have to read the length of the bytes first
            if (keyType == MessageTypes.OBJECT && totalRead + Integer.BYTES > size) {
                break;
            }
            // for object type we read the object bytes + 4
            int keySize = getKeySize(keyType, os);
            // we cannot read further
            if (totalRead + keySize > size) {
                break;
            }
            key = keyType.getDataPacker().unpackFromBuffer(os, keySize - Integer.BYTES);
            if (totalRead + keySize + Integer.BYTES > size) {
                break;
            }
            int dataSize = os.getInt();
            // we cannot read further
            if (totalRead + keySize + dataSize + Integer.BYTES > size) {
                break;
            }
            value = dataType.getDataPacker().unpackFromBuffer(os, dataSize);
            keyValues.add(new Tuple(key, value));
            totalRead += Integer.BYTES + keySize + dataSize;
        }
        long size1 = rwChannel.size();
        rwChannel.close();
        return new OpenFilePart(keyValues, totalRead + startOffSet, size1, fileName);
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Error in reading file part", e);
        throw new RuntimeException(e);
    }
}

Example 13 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method saveKeyValues.

/**
 * Save the list of records to the file system
 *
 * @param records records to be written
 * @param size total size of the records
 * @param outFileName out file name
 * @return maximum size of the tuple written to this file
 */
public static long saveKeyValues(List<Tuple> records, List<Integer> sizes, long size, String outFileName, MessageType keyType) {
    try {
        // max size of a tuple saved to this file
        long maxRecord = Long.MIN_VALUE;
        // first serialize keys
        long totalSize = 0;
        List<byte[]> byteKeys = new ArrayList<>();
        if (keyType.isPrimitive() && !keyType.isArray()) {
            totalSize += records.size() * keyType.getUnitSizeInBytes();
        } else {
            for (Tuple record : records) {
                byte[] data = keyType.getDataPacker().packToByteArray(record.getKey());
                // data + length of key
                totalSize += data.length;
                if (keyType.getDataPacker().isHeaderRequired()) {
                    totalSize += Integer.BYTES;
                }
                byteKeys.add(data);
            }
        }
        // just to check whether sizes match
        long sizeSum = 0;
        // we need to write the data lengths and key lengths
        int dataLengthSize = Integer.BYTES * records.size();
        totalSize += size + dataLengthSize;
        Files.createDirectories(Paths.get(outFileName).getParent());
        RandomAccessFile randomAccessFile = new RandomAccessFile(outFileName, "rw");
        FileChannel rwChannel = randomAccessFile.getChannel();
        MappedByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, totalSize);
        for (int i = 0; i < records.size(); i++) {
            // position of os before writing this tuple
            long positionBefore = os.position();
            Tuple keyValue = records.get(i);
            // this has been already serialized
            byte[] r = (byte[]) keyValue.getValue();
            if (keyType.isPrimitive() && !keyType.isArray()) {
                keyType.getDataPacker().packToByteBuffer(os, keyValue.getKey());
            } else {
                byte[] key = byteKeys.get(i);
                if (keyType.getDataPacker().isHeaderRequired()) {
                    os.putInt(key.length);
                }
                os.put(key);
            }
            sizeSum += sizes.get(i);
            os.putInt(sizes.get(i));
            os.put(r, 0, sizes.get(i));
            long tupleSize = os.position() - positionBefore;
            maxRecord = Math.max(maxRecord, tupleSize);
        }
        if (sizeSum != size) {
            LOG.log(Level.WARNING, "Sum doesn't equal size: " + sizeSum + " != " + size);
        }
        rwChannel.close();
        randomAccessFile.close();
        try {
            MemoryMapUtils.unMapBuffer(os);
        } catch (Exception e) {
            // ignore
            LOG.warning("Couldn't manually unmap a byte buffer");
        }
        return maxRecord;
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Failed write to disc", e);
        throw new RuntimeException(e);
    }
}

Example 14 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method openFilePart.

public static Triple<List<Tuple>, Long, Long> openFilePart(String fileName, long startOffSet, int maxSize, MessageType keyType, MessageType dataType, KryoSerializer deserializer) {
    List<Tuple> keyValues = new ArrayList<>();
    String outFileName = Paths.get(fileName).toString();
    FileChannel rwChannel;
    try {
        rwChannel = new RandomAccessFile(outFileName, "rw").getChannel();
        long size = maxSize < rwChannel.size() - startOffSet ? maxSize : rwChannel.size() - startOffSet;
        ByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_ONLY, startOffSet, size);
        long totalRead = 0;
        while (totalRead < size) {
            Object key;
            Object value;
            int keySize = os.getInt();
            key = keyType.getDataPacker().unpackFromBuffer(os, keySize);
            // we cannot read further
            if (totalRead + keySize > size) {
                break;
            }
            int dataSize = os.getInt();
            value = dataType.getDataPacker().unpackFromBuffer(os, dataSize);
            // we cannot read further
            if (totalRead + keySize + dataSize > size) {
                break;
            }
            keyValues.add(new Tuple(key, value));
            totalRead += 8 + keySize + dataSize;
        }
        rwChannel.close();
        return new ImmutableTriple<>(keyValues, totalRead + startOffSet, rwChannel.size());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 15 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedMerger method add.

/**
 * Add the data to the file
 */
public void add(Object key, byte[] data, int length) {
    if (status == FSStatus.READING) {
        throw new RuntimeException("Cannot add after switching to reading");
    }
    lock.lock();
    try {
        recordsInMemory.add(new Tuple(key, data));
        bytesLength.add(length);
        numOfBytesInMemory += length;
    } finally {
        lock.unlock();
    }
}