Examples with MemorySegment org.apache.flink.core.memory.MemorySegment used on opensource projects

Example 96 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinaryStringData method compareTo.

/**
 * Compares two strings lexicographically. Since UTF-8 uses groups of six bits, it is sometimes
 * useful to use octal notation which uses 3-bit groups. With a calculator which can convert
 * between hexadecimal and octal it can be easier to manually create or interpret UTF-8 compared
 * with using binary. So we just compare the binary.
 */
@Override
public int compareTo(@Nonnull StringData o) {
    // BinaryStringData is the only implementation of StringData
    BinaryStringData other = (BinaryStringData) o;
    if (javaObject != null && other.javaObject != null) {
        return javaObject.compareTo(other.javaObject);
    }
    ensureMaterialized();
    other.ensureMaterialized();
    if (binarySection.segments.length == 1 && other.binarySection.segments.length == 1) {
        int len = Math.min(binarySection.sizeInBytes, other.binarySection.sizeInBytes);
        MemorySegment seg1 = binarySection.segments[0];
        MemorySegment seg2 = other.binarySection.segments[0];
        for (int i = 0; i < len; i++) {
            int res = (seg1.get(binarySection.offset + i) & 0xFF) - (seg2.get(other.binarySection.offset + i) & 0xFF);
            if (res != 0) {
                return res;
            }
        }
        return binarySection.sizeInBytes - other.binarySection.sizeInBytes;
    }
    // if there are multi segments.
    return compareMultiSegments(other);
}

Example 97 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinaryStringData method substring.

/**
 * Returns a binary string that is a substring of this binary string. The substring begins at
 * the specified {@code beginIndex} and extends to the character at index {@code endIndex - 1}.
 *
 * <p>Examples:
 *
 * <blockquote>
 *
 * <pre>
 * fromString("hamburger").substring(4, 8) returns binary string "urge"
 * fromString("smiles").substring(1, 5) returns binary string "mile"
 * </pre>
 *
 * </blockquote>
 *
 * @param beginIndex the beginning index, inclusive.
 * @param endIndex the ending index, exclusive.
 * @return the specified substring, return EMPTY_UTF8 when index out of bounds instead of
 *     StringIndexOutOfBoundsException.
 */
public BinaryStringData substring(int beginIndex, int endIndex) {
    ensureMaterialized();
    if (endIndex <= beginIndex || beginIndex >= binarySection.sizeInBytes) {
        return EMPTY_UTF8;
    }
    if (inFirstSegment()) {
        MemorySegment segment = binarySection.segments[0];
        int i = 0;
        int c = 0;
        while (i < binarySection.sizeInBytes && c < beginIndex) {
            i += numBytesForFirstByte(segment.get(i + binarySection.offset));
            c += 1;
        }
        int j = i;
        while (i < binarySection.sizeInBytes && c < endIndex) {
            i += numBytesForFirstByte(segment.get(i + binarySection.offset));
            c += 1;
        }
        if (i > j) {
            byte[] bytes = new byte[i - j];
            segment.get(binarySection.offset + j, bytes, 0, i - j);
            return fromBytes(bytes);
        } else {
            return EMPTY_UTF8;
        }
    } else {
        return substringMultiSegs(beginIndex, endIndex);
    }
}

Example 98 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinarySegmentUtils method getLongSlowly.

private static long getLongSlowly(MemorySegment[] segments, int segSize, int segNum, int segOffset) {
    MemorySegment segment = segments[segNum];
    long ret = 0;
    for (int i = 0; i < 8; i++) {
        if (segOffset == segSize) {
            segment = segments[++segNum];
            segOffset = 0;
        }
        long unsignedByte = segment.get(segOffset) & 0xff;
        if (LITTLE_ENDIAN) {
            ret |= (unsignedByte << (i * 8));
        } else {
            ret |= (unsignedByte << ((7 - i) * 8));
        }
        segOffset++;
    }
    return ret;
}

Example 99 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinarySegmentUtils method getIntSlowly.

private static int getIntSlowly(MemorySegment[] segments, int segSize, int segNum, int segOffset) {
    MemorySegment segment = segments[segNum];
    int ret = 0;
    for (int i = 0; i < 4; i++) {
        if (segOffset == segSize) {
            segment = segments[++segNum];
            segOffset = 0;
        }
        int unsignedByte = segment.get(segOffset) & 0xff;
        if (LITTLE_ENDIAN) {
            ret |= (unsignedByte << (i * 8));
        } else {
            ret |= (unsignedByte << ((3 - i) * 8));
        }
        segOffset++;
    }
    return ret;
}

Example 100 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BinarySegmentUtils method bitSetMultiSegments.

private static void bitSetMultiSegments(MemorySegment[] segments, int baseOffset, int index) {
    int offset = baseOffset + byteIndex(index);
    int segSize = segments[0].size();
    int segIndex = offset / segSize;
    // equal to %
    int segOffset = offset - segIndex * segSize;
    MemorySegment segment = segments[segIndex];
    byte current = segment.get(segOffset);
    current |= (1 << (index & BIT_BYTE_INDEX_MASK));
    segment.put(segOffset, current);
}