Examples with MMapBuffer xerial.larray.mmap.MMapBuffer used on opensource projects

Example 1 with MMapBuffer

use of xerial.larray.mmap.MMapBuffer in project pinot by linkedin.

the class StarTreeDataTable method groupByIntColumnCount.

/**
   *
   * @param startDocId inclusive
   * @param endDocId exclusive
   * @param colIndex
   * @return start,end for each value. inclusive start, exclusive end
   */
public Int2ObjectMap<IntPair> groupByIntColumnCount(int startDocId, int endDocId, Integer colIndex) {
    MMapBuffer mappedByteBuffer = null;
    try {
        int length = endDocId - startDocId;
        Int2ObjectMap<IntPair> rangeMap = new Int2ObjectLinkedOpenHashMap<>();
        final long startOffset = startDocId * (long) totalSizeInBytes;
        mappedByteBuffer = new MMapBuffer(file, startOffset, length * (long) totalSizeInBytes, MMapMode.READ_WRITE);
        int prevValue = -1;
        int prevStart = 0;
        for (int i = 0; i < length; i++) {
            int value = flipEndiannessIfNeeded(mappedByteBuffer.getInt((i * (long) totalSizeInBytes) + (colIndex * V1Constants.Numbers.INTEGER_SIZE)));
            if (prevValue != -1 && prevValue != value) {
                rangeMap.put(prevValue, new IntPair(startDocId + prevStart, startDocId + i));
                prevStart = i;
            }
            prevValue = value;
        }
        rangeMap.put(prevValue, new IntPair(startDocId + prevStart, endDocId));
        return rangeMap;
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (mappedByteBuffer != null) {
            try {
                mappedByteBuffer.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    return EMPTY_INT_OBJECT_MAP;
}

Example 2 with MMapBuffer

use of xerial.larray.mmap.MMapBuffer in project pinot by linkedin.

the class StarTreeDataTable method iterator.

public Iterator<Pair<byte[], byte[]>> iterator(int startDocId, int endDocId) throws IOException {
    final int length = endDocId - startDocId;
    final long startOffset = startDocId * (long) totalSizeInBytes;
    final MMapBuffer mappedByteBuffer = new MMapBuffer(file, startOffset, length * (long) totalSizeInBytes, MMapMode.READ_WRITE);
    return new Iterator<Pair<byte[], byte[]>>() {

        int pointer = 0;

        @Override
        public boolean hasNext() {
            return pointer < length;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Pair<byte[], byte[]> next() {
            byte[] dimBuff = new byte[dimensionSizeInBytes];
            byte[] metBuff = new byte[metricSizeInBytes];
            mappedByteBuffer.toDirectByteBuffer(pointer * (long) totalSizeInBytes, dimensionSizeInBytes).get(dimBuff);
            if (metricSizeInBytes > 0) {
                mappedByteBuffer.toDirectByteBuffer(pointer * (long) totalSizeInBytes + dimensionSizeInBytes, metricSizeInBytes).get(metBuff);
            }
            pointer = pointer + 1;
            if (pointer == length) {
                try {
                    mappedByteBuffer.close();
                } catch (IOException e) {
                    LOGGER.error("Exception caught in record iterator", e);
                }
            }
            return Pair.of(dimBuff, metBuff);
        }
    };
}

Example 3 with MMapBuffer

use of xerial.larray.mmap.MMapBuffer in project pinot by linkedin.

the class StarTreeSerDe method writeTreeOffHeapFromOnHeap.

/**
   * Given a StarTree in on-heap format, serialize it into OFF_HEAP format and write to the
   * given file.
   * @param starTree
   * @param outputFile
   */
private static void writeTreeOffHeapFromOnHeap(StarTree starTree, File outputFile) throws IOException {
    int headerSizeInBytes = computeOffHeapHeaderSizeInBytes(starTree);
    long totalSize = headerSizeInBytes + computeOffHeapNodesSizeInBytes(starTree);
    MMapBuffer mappedByteBuffer = new MMapBuffer(outputFile, 0, totalSize, MMapMode.READ_WRITE);
    long offset = writeHeaderOffHeap(starTree, headerSizeInBytes, mappedByteBuffer);
    // Ensure that the computed offset is the same as actual offset.
    Preconditions.checkState((offset == headerSizeInBytes), "Error writing Star Tree file, header size mis-match");
    // Write the actual star tree nodes in level order.
    writeNodesOffHeap(starTree, mappedByteBuffer, offset);
    mappedByteBuffer.flush();
    mappedByteBuffer.close();
}

Example 4 with MMapBuffer

use of xerial.larray.mmap.MMapBuffer in project pinot by linkedin.

the class StarTreeDataTable method sort.

/**
   *
   * @param startRecordId inclusive
   * @param endRecordId exclusive
   */
public void sort(int startRecordId, int endRecordId) {
    final MMapBuffer mappedByteBuffer;
    try {
        int numRecords = endRecordId - startRecordId;
        final long startOffset = startRecordId * (long) totalSizeInBytes;
        // Sort the docIds without actually moving the docs themselves.
        mappedByteBuffer = new MMapBuffer(file, startOffset, numRecords * (long) totalSizeInBytes, MMapMode.READ_WRITE);
        final int[] sortedDocIds = getSortedDocIds(mappedByteBuffer, totalSizeInBytes, dimensionSizeInBytes, numRecords);
        // Re-arrange the docs as per the sorted docId order.
        sortMmapBuffer(mappedByteBuffer, totalSizeInBytes, numRecords, sortedDocIds);
    } catch (IOException e) {
        LOGGER.error("Exception caught while sorting records", e);
    }
}

Example 5 with MMapBuffer

use of xerial.larray.mmap.MMapBuffer in project pinot by linkedin.

the class PinotLByteBuffer method mapFromFileInternal.

private static MMapBuffer mapFromFileInternal(File file, long start, long length, FileChannel.MapMode openMode, String context) throws IOException {
    MMapMode mmapMode = (openMode == FileChannel.MapMode.READ_ONLY) ? MMapMode.READ_ONLY : MMapMode.READ_WRITE;
    // TODO: add memory tracking MMapUtils
    MMapBuffer buf = new MMapBuffer(file, start, length, mmapMode);
    return buf;
}