Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 61 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger2 method readIterator.

/**
 * This method gives the values
 */
public RestorableIterator<Object> readIterator() {
    try {
        return new RestorableIterator<Object>() {

            private static final String RP_NEXT_TUPLE = "NEXT_TUPLE";

            private static final String RP_IT_OF_CURR_KEY = "IT_OF_CURR_KEY";

            private FSIterator fsIterator = new FSIterator();

            private Tuple nextTuple = fsIterator.hasNext() ? fsIterator.next() : null;

            private Iterator itOfCurrentKey = null;

            private RestorePoint restorePoint;

            @Override
            public void createRestorePoint() {
                this.restorePoint = new RestorePoint();
                this.restorePoint.put(RP_NEXT_TUPLE, this.nextTuple);
                if (groupByKey) {
                    this.restorePoint.put(RP_IT_OF_CURR_KEY, this.itOfCurrentKey);
                }
                this.fsIterator.createRestorePoint();
            }

            @Override
            public void restore() {
                if (!this.hasRestorePoint()) {
                    throw new RuntimeException("Couldn't find a valid restore point to restore from.");
                }
                this.nextTuple = (Tuple) this.restorePoint.get(RP_NEXT_TUPLE);
                this.itOfCurrentKey = (Iterator) this.restorePoint.get(RP_IT_OF_CURR_KEY);
                this.fsIterator.restore();
            }

            @Override
            public boolean hasRestorePoint() {
                return this.restorePoint != null;
            }

            @Override
            public void clearRestorePoint() {
                this.restorePoint = null;
                this.fsIterator.clearRestorePoint();
            }

            private void skipKeys() {
                // user is trying to skip keys. For now, we are iterating over them internally
                if (this.itOfCurrentKey != null) {
                    // todo replace with an alternative approach
                    while (this.itOfCurrentKey.hasNext()) {
                        this.itOfCurrentKey.next();
                    }
                }
            }

            @Override
            public boolean hasNext() {
                this.skipKeys();
                return nextTuple != null;
            }

            @Override
            public Tuple<Object, Object> next() {
                if (!hasNext()) {
                    throw new NoSuchElementException("There are no more keys to iterate");
                }
                final Object currentKey = nextTuple.getKey();
                Tuple<Object, Object> nextValueSet = new Tuple<>();
                nextValueSet.setKey(currentKey);
                if (groupByKey) {
                    this.itOfCurrentKey = new Iterator<Object>() {

                        @Override
                        public boolean hasNext() {
                            return nextTuple != null && nextTuple.getKey().equals(currentKey);
                        }

                        @Override
                        public Object next() {
                            if (this.hasNext()) {
                                Object returnValue = nextTuple.getValue();
                                if (fsIterator.hasNext()) {
                                    nextTuple = fsIterator.next();
                                } else {
                                    nextTuple = null;
                                }
                                return returnValue;
                            } else {
                                throw new NoSuchElementException("There are no more values for key " + currentKey);
                            }
                        }
                    };
                    nextValueSet.setValue(this.itOfCurrentKey);
                } else {
                    nextValueSet.setValue(nextTuple.getValue());
                    if (fsIterator.hasNext()) {
                        nextTuple = fsIterator.next();
                    } else {
                        nextTuple = null;
                    }
                }
                return nextValueSet;
            }
        };
    } catch (Exception e) {
        LOG.log(Level.SEVERE, "Error in creating iterator", e);
        throw new RuntimeException(e);
    }
}

Example 62 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSSorter method next.

public Object next() {
    HeapNode min = heap.extractMin();
    int list = min.listNo;
    FilePart p = openList.get(list);
    List<Tuple> keyValues = p.keyValues.getLeft();
    if (keyValues.size() <= p.currentIndex) {
        String fileName = folder + "/part_" + list;
        // also if the file reached end, we don't need to do anything
        if (list < noOfFiles && p.keyValues.getMiddle() < p.keyValues.getRight()) {
            Triple<List<Tuple>, Long, Long> values = FileLoader.openFilePart(fileName, p.keyValues.getMiddle(), openBytes, keyType, dataType, deserializer);
            // set the new values to the list
            p.keyValues = values;
        }
    }
    return min.data;
}

Example 63 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class OpenFilePart method next.

public Tuple next() {
    Tuple kv = keyValues.get(keyValueIndex);
    keyValueIndex++;
    return kv;
}

Example 64 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FileLoader method saveKeyValues.

/**
 * This method accepts a Array of lists instead of a list of tuples
 */
public static long saveKeyValues(List<Tuple> records, long size, String outFileName, MessageType keyType) {
    try {
        // max size of a tuple saved to this file
        long maxRecord = Long.MIN_VALUE;
        // first serialize keys
        long totalSize = 0;
        List<byte[]> byteKeys = null;
        if (keyType.isPrimitive() && !keyType.isArray()) {
            totalSize += records.size() * keyType.getUnitSizeInBytes();
        } else {
            // only initialize if required
            byteKeys = new ArrayList<>(records.size());
            for (Tuple record : records) {
                byte[] data = keyType.getDataPacker().packToByteArray(record.getKey());
                // data + length of key
                totalSize += data.length;
                if (keyType.getDataPacker().isHeaderRequired()) {
                    totalSize += Integer.BYTES;
                }
                byteKeys.add(data);
            }
        }
        // just to check whether sizes match
        long sizeSum = 0;
        // we need to write the data lengths and key lengths
        int dataLengthSize = Integer.BYTES * records.size();
        totalSize += size + dataLengthSize;
        Files.createDirectories(Paths.get(outFileName).getParent());
        RandomAccessFile randomAccessFile = new RandomAccessFile(outFileName, "rw");
        FileChannel rwChannel = randomAccessFile.getChannel();
        MappedByteBuffer os = rwChannel.map(FileChannel.MapMode.READ_WRITE, 0, totalSize);
        int i = 0;
        for (Tuple keyValue : records) {
            // position of os before writing this tuple
            long positionBefore = os.position();
            // this has been already serialized
            byte[] r = (byte[]) keyValue.getValue();
            if (keyType.isPrimitive() && !keyType.isArray()) {
                keyType.getDataPacker().packToByteBuffer(os, keyValue.getKey());
            } else {
                byte[] key = byteKeys.get(i);
                if (keyType.getDataPacker().isHeaderRequired()) {
                    os.putInt(key.length);
                }
                os.put(key);
            }
            sizeSum += r.length;
            os.putInt(r.length);
            os.put(r, 0, r.length);
            long tupleSize = os.position() - positionBefore;
            maxRecord = Math.max(maxRecord, tupleSize);
            i++;
        }
        if (sizeSum != size) {
            LOG.log(Level.WARNING, "Sum doesn't equal size: " + sizeSum + " != " + size);
        }
        rwChannel.close();
        randomAccessFile.close();
        try {
            MemoryMapUtils.unMapBuffer(os);
        } catch (Exception e) {
            // ignore
            LOG.warning("Couldn't unmap a byte buffer manually");
        }
        return maxRecord;
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Failed write to disc", e);
        throw new RuntimeException(e);
    }
}

Example 65 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class FSKeyedSortedMerger method deserializeObjects.

private void deserializeObjects() {
    for (int i = 0; i < recordsInMemory.size(); i++) {
        Tuple kv = recordsInMemory.get(i);
        Object o = dataType.getDataPacker().unpackFromByteArray((byte[]) kv.getValue());
        objectsInMemory.add(new Tuple(kv.getKey(), o));
    }
}