Examples with SliceUtf8.lengthOfCodePoint io.airlift.slice.SliceUtf8.lengthOfCodePoint used on opensource projects

Example 1 with SliceUtf8.lengthOfCodePoint

use of io.airlift.slice.SliceUtf8.lengthOfCodePoint in project presto by prestodb.

the class StringFunctions method safeCountCodePoints.

private static int safeCountCodePoints(Slice slice) {
    int codePoints = 0;
    for (int position = 0; position < slice.length(); ) {
        int codePoint = tryGetCodePointAt(slice, position);
        if (codePoint < 0) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid UTF-8 encoding in characters: " + slice.toStringUtf8());
        }
        position += lengthOfCodePoint(codePoint);
        codePoints++;
    }
    return codePoints;
}

Example 2 with SliceUtf8.lengthOfCodePoint

use of io.airlift.slice.SliceUtf8.lengthOfCodePoint in project presto by prestodb.

the class StringFunctions method pad.

private static Slice pad(Slice text, long targetLength, Slice padString, int paddingOffset) {
    checkCondition(0 <= targetLength && targetLength <= Integer.MAX_VALUE, INVALID_FUNCTION_ARGUMENT, "Target length must be in the range [0.." + Integer.MAX_VALUE + "]");
    checkCondition(padString.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding string must not be empty");
    int textLength = countCodePoints(text);
    int resultLength = (int) targetLength;
    // if our target length is the same as our string then return our string
    if (textLength == resultLength) {
        return text;
    }
    // if our string is bigger than requested then truncate
    if (textLength > resultLength) {
        return SliceUtf8.substring(text, 0, resultLength);
    }
    // number of bytes in each code point
    int padStringLength = countCodePoints(padString);
    int[] padStringCounts = new int[padStringLength];
    for (int i = 0; i < padStringLength; ++i) {
        padStringCounts[i] = lengthOfCodePointSafe(padString, offsetOfCodePoint(padString, i));
    }
    // preallocate the result
    int bufferSize = text.length();
    for (int i = 0; i < resultLength - textLength; ++i) {
        bufferSize += padStringCounts[i % padStringLength];
    }
    Slice buffer = Slices.allocate(bufferSize);
    // fill in the existing string
    int countBytes = bufferSize - text.length();
    int startPointOfExistingText = (paddingOffset + countBytes) % bufferSize;
    buffer.setBytes(startPointOfExistingText, text);
    // assign the pad string while there's enough space for it
    int byteIndex = paddingOffset;
    for (int i = 0; i < countBytes / padString.length(); ++i) {
        buffer.setBytes(byteIndex, padString);
        byteIndex += padString.length();
    }
    // handle the tail: at most we assign padStringLength - 1 code points
    buffer.setBytes(byteIndex, padString.getBytes(0, paddingOffset + countBytes - byteIndex));
    return buffer;
}

Example 3 with SliceUtf8.lengthOfCodePoint

use of io.airlift.slice.SliceUtf8.lengthOfCodePoint in project presto by prestodb.

the class StringFunctions method fromUtf8.

@Description("decodes the UTF-8 encoded string")
@ScalarFunction
@LiteralParameters("x")
@SqlType(StandardTypes.VARCHAR)
public static Slice fromUtf8(@SqlType(StandardTypes.VARBINARY) Slice slice, @SqlType("varchar(x)") Slice replacementCharacter) {
    int count = countCodePoints(replacementCharacter);
    if (count > 1) {
        throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Replacement character string must empty or a single character");
    }
    OptionalInt replacementCodePoint;
    if (count == 1) {
        try {
            replacementCodePoint = OptionalInt.of(getCodePointAt(replacementCharacter, 0));
        } catch (InvalidUtf8Exception e) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid replacement character");
        }
    } else {
        replacementCodePoint = OptionalInt.empty();
    }
    return SliceUtf8.fixInvalidUtf8(slice, replacementCodePoint);
}

Example 4 with SliceUtf8.lengthOfCodePoint

use of io.airlift.slice.SliceUtf8.lengthOfCodePoint in project presto by prestodb.

the class StringFunctions method splitPart.

@SqlNullable
@Description("splits a string by a delimiter and returns the specified field (counting from one)")
@ScalarFunction
@LiteralParameters({ "x", "y" })
@SqlType("varchar(x)")
public static Slice splitPart(@SqlType("varchar(x)") Slice string, @SqlType("varchar(y)") Slice delimiter, @SqlType(StandardTypes.BIGINT) long index) {
    checkCondition(index > 0, INVALID_FUNCTION_ARGUMENT, "Index must be greater than zero");
    // Empty delimiter? Then every character will be a split
    if (delimiter.length() == 0) {
        int startCodePoint = toIntExact(index);
        int indexStart = offsetOfCodePoint(string, startCodePoint - 1);
        if (indexStart < 0) {
            // index too big
            return null;
        }
        int length = lengthOfCodePoint(string, indexStart);
        if (indexStart + length > string.length()) {
            throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid UTF-8 encoding");
        }
        return string.slice(indexStart, length);
    }
    int matchCount = 0;
    int previousIndex = 0;
    while (previousIndex < string.length()) {
        int matchIndex = string.indexOf(delimiter, previousIndex);
        // No match
        if (matchIndex < 0) {
            break;
        }
        // Reached the requested part?
        if (++matchCount == index) {
            return string.slice(previousIndex, matchIndex - previousIndex);
        }
        // Continue searching after the delimiter
        previousIndex = matchIndex + delimiter.length();
    }
    if (matchCount == index - 1) {
        // returns last section of the split
        return string.slice(previousIndex, string.length() - previousIndex);
    }
    // index is too big, null is returned
    return null;
}

Example 5 with SliceUtf8.lengthOfCodePoint

use of io.airlift.slice.SliceUtf8.lengthOfCodePoint in project presto by prestodb.

the class StringFunctions method castToCodePoints.

private static int[] castToCodePoints(Slice slice) {
    int[] codePoints = new int[safeCountCodePoints(slice)];
    int position = 0;
    for (int index = 0; index < codePoints.length; index++) {
        codePoints[index] = getCodePointAt(slice, position);
        position += lengthOfCodePoint(slice, position);
    }
    return codePoints;
}