Examples with MemorySegment org.apache.flink.core.memory.MemorySegment used on opensource projects

Example 91 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SkipListUtils method helpSetNextValuePointer.

/**
 * Sets the next value pointer of the value.
 *
 * @param valuePointer the value pointer.
 * @param nextValuePointer the next value pointer to set.
 * @param spaceAllocator the space allocator.
 */
static void helpSetNextValuePointer(long valuePointer, long nextValuePointer, Allocator spaceAllocator) {
    Chunk chunk = spaceAllocator.getChunkById(SpaceUtils.getChunkIdByAddress(valuePointer));
    int offsetInChunk = SpaceUtils.getChunkOffsetByAddress(valuePointer);
    MemorySegment segment = chunk.getMemorySegment(offsetInChunk);
    int offsetInByteBuffer = chunk.getOffsetInSegment(offsetInChunk);
    putNextValuePointer(segment, offsetInByteBuffer, nextValuePointer);
}

Example 92 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SkipListUtils method findPredecessor.

/**
 * Find the predecessor node for the given node at the given level.
 *
 * @param node the node.
 * @param level the level.
 * @param levelIndexHeader the head level index.
 * @param spaceAllocator the space allocator.
 * @return node id before the key at the given level.
 */
static long findPredecessor(long node, int level, LevelIndexHeader levelIndexHeader, @Nonnull Allocator spaceAllocator) {
    Chunk chunk = spaceAllocator.getChunkById(SpaceUtils.getChunkIdByAddress(node));
    int offsetInChunk = SpaceUtils.getChunkOffsetByAddress(node);
    MemorySegment keySegment = chunk.getMemorySegment(offsetInChunk);
    int offsetInByteBuffer = chunk.getOffsetInSegment(offsetInChunk);
    int keyLevel = getLevel(keySegment, offsetInByteBuffer);
    int keyOffset = offsetInByteBuffer + getKeyDataOffset(keyLevel);
    return findPredecessor(keySegment, keyOffset, level, levelIndexHeader, spaceAllocator);
}

Example 93 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SkipListKeySerializer method getSerializedKeyAndNamespace.

/**
 * Gets serialized key and namespace from the byte buffer.
 *
 * @param memorySegment the memory segment which stores the skip list key.
 * @param offset the start position of the skip list key in the byte buffer.
 * @return tuple of serialized key and namespace.
 */
Tuple2<byte[], byte[]> getSerializedKeyAndNamespace(MemorySegment memorySegment, int offset) {
    // read namespace
    int namespaceLen = memorySegment.getInt(offset);
    MemorySegment namespaceSegment = MemorySegmentFactory.allocateUnpooledSegment(namespaceLen);
    memorySegment.copyTo(offset + Integer.BYTES, namespaceSegment, 0, namespaceLen);
    // read key
    int keyOffset = offset + Integer.BYTES + namespaceLen;
    int keyLen = memorySegment.getInt(keyOffset);
    MemorySegment keySegment = MemorySegmentFactory.allocateUnpooledSegment(keyLen);
    memorySegment.copyTo(keyOffset + Integer.BYTES, keySegment, 0, keyLen);
    return Tuple2.of(keySegment.getArray(), namespaceSegment.getArray());
}

Example 94 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SkipListKeySerializer method serializeToSegment.

/**
 * Serialize the key and namespace to bytes. The format is - int: length of serialized namespace
 * - byte[]: serialized namespace - int: length of serialized key - byte[]: serialized key
 */
MemorySegment serializeToSegment(K key, N namespace) {
    outputStream.reset();
    try {
        // serialize namespace
        outputStream.setPosition(Integer.BYTES);
        namespaceSerializer.serialize(namespace, outputView);
    } catch (IOException e) {
        throw new RuntimeException("Failed to serialize namespace", e);
    }
    int keyStartPos = outputStream.getPosition();
    try {
        // serialize key
        outputStream.setPosition(keyStartPos + Integer.BYTES);
        keySerializer.serialize(key, outputView);
    } catch (IOException e) {
        throw new RuntimeException("Failed to serialize key", e);
    }
    final byte[] result = outputStream.toByteArray();
    final MemorySegment segment = MemorySegmentFactory.wrap(result);
    // set length of namespace and key
    segment.putInt(0, keyStartPos - Integer.BYTES);
    segment.putInt(keyStartPos, result.length - keyStartPos - Integer.BYTES);
    return segment;
}

Example 95 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinaryStringData method compareMultiSegments.

/**
 * Find the boundaries of segments, and then compare MemorySegment.
 */
private int compareMultiSegments(BinaryStringData other) {
    if (binarySection.sizeInBytes == 0 || other.binarySection.sizeInBytes == 0) {
        return binarySection.sizeInBytes - other.binarySection.sizeInBytes;
    }
    int len = Math.min(binarySection.sizeInBytes, other.binarySection.sizeInBytes);
    MemorySegment seg1 = binarySection.segments[0];
    MemorySegment seg2 = other.binarySection.segments[0];
    int segmentSize = binarySection.segments[0].size();
    int otherSegmentSize = other.binarySection.segments[0].size();
    int sizeOfFirst1 = segmentSize - binarySection.offset;
    int sizeOfFirst2 = otherSegmentSize - other.binarySection.offset;
    int varSegIndex1 = 1;
    int varSegIndex2 = 1;
    // find the first segment of this string.
    while (sizeOfFirst1 <= 0) {
        sizeOfFirst1 += segmentSize;
        seg1 = binarySection.segments[varSegIndex1++];
    }
    while (sizeOfFirst2 <= 0) {
        sizeOfFirst2 += otherSegmentSize;
        seg2 = other.binarySection.segments[varSegIndex2++];
    }
    int offset1 = segmentSize - sizeOfFirst1;
    int offset2 = otherSegmentSize - sizeOfFirst2;
    int needCompare = Math.min(Math.min(sizeOfFirst1, sizeOfFirst2), len);
    while (needCompare > 0) {
        // compare in one segment.
        for (int i = 0; i < needCompare; i++) {
            int res = (seg1.get(offset1 + i) & 0xFF) - (seg2.get(offset2 + i) & 0xFF);
            if (res != 0) {
                return res;
            }
        }
        if (needCompare == len) {
            break;
        }
        len -= needCompare;
        // next segment
        if (sizeOfFirst1 < sizeOfFirst2) {
            // I am smaller
            seg1 = binarySection.segments[varSegIndex1++];
            offset1 = 0;
            offset2 += needCompare;
            sizeOfFirst1 = segmentSize;
            sizeOfFirst2 -= needCompare;
        } else if (sizeOfFirst1 > sizeOfFirst2) {
            // other is smaller
            seg2 = other.binarySection.segments[varSegIndex2++];
            offset2 = 0;
            offset1 += needCompare;
            sizeOfFirst2 = otherSegmentSize;
            sizeOfFirst1 -= needCompare;
        } else {
            // same, should go ahead both.
            seg1 = binarySection.segments[varSegIndex1++];
            seg2 = other.binarySection.segments[varSegIndex2++];
            offset1 = 0;
            offset2 = 0;
            sizeOfFirst1 = segmentSize;
            sizeOfFirst2 = otherSegmentSize;
        }
        needCompare = Math.min(Math.min(sizeOfFirst1, sizeOfFirst2), len);
    }
    checkArgument(needCompare == len);
    return binarySection.sizeInBytes - other.binarySection.sizeInBytes;
}