Examples with TrustedByteArrayOutputStream org.apache.phoenix.util.TrustedByteArrayOutputStream used on opensource projects

Example 6 with TrustedByteArrayOutputStream

use of org.apache.phoenix.util.TrustedByteArrayOutputStream in project phoenix by apache.

the class IndexMaintainer method buildDataRowKey.

/*
     * Build the data row key from the index row key
     */
public byte[] buildDataRowKey(ImmutableBytesWritable indexRowKeyPtr, byte[][] viewConstants) {
    RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
    DataOutput output = new DataOutputStream(stream);
    // Increment dataPosOffset until all have been written
    int dataPosOffset = 0;
    int viewConstantsIndex = 0;
    try {
        int indexPosOffset = !isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0;
        int maxRowKeyOffset = indexRowKeyPtr.getOffset() + indexRowKeyPtr.getLength();
        indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
        if (isDataTableSalted) {
            dataPosOffset++;
            // will be set at end to salt byte
            output.write(0);
        }
        if (viewIndexId != null) {
            indexRowKeySchema.next(ptr, indexPosOffset++, maxRowKeyOffset);
        }
        if (isMultiTenant) {
            indexRowKeySchema.next(ptr, indexPosOffset, maxRowKeyOffset);
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
            }
            indexPosOffset++;
            dataPosOffset++;
        }
        indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
        BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
        BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
        for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
            // same for all rows in this index)
            if (viewConstantColumnBitSet.get(i)) {
                output.write(viewConstants[viewConstantsIndex++]);
            } else {
                int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
                Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, pos + indexPosOffset + 1);
                if (Boolean.TRUE.equals(hasValue)) {
                    // Write data row key value taking into account coercion and inversion
                    // if necessary
                    Field dataField = dataRowKeySchema.getField(i);
                    Field indexField = indexRowKeySchema.getField(pos + indexPosOffset);
                    PDataType indexColumnType = indexField.getDataType();
                    PDataType dataColumnType = dataField.getDataType();
                    SortOrder dataSortOrder = dataField.getSortOrder();
                    SortOrder indexSortOrder = indexField.getSortOrder();
                    boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
                    boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
                    if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(pos)) {
                        output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                    } else {
                        if (!isBytesComparable) {
                            dataColumnType.coerceBytes(ptr, indexColumnType, indexSortOrder, SortOrder.getDefault());
                        }
                        if (descIndexColumnBitSet.get(pos) != isDataColumnInverted) {
                            writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                        } else {
                            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                        }
                    }
                }
            }
            // Write separator byte if variable length unless it's the last field in the schema
            // (but we still need to write it if it's DESC to ensure sort order is correct).
            byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
            if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i + 1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
                output.writeByte(sepByte);
            }
        }
        int length = stream.size();
        int minLength = length - maxTrailingNulls;
        byte[] dataRowKey = stream.getBuffer();
        // Remove trailing nulls
        int index = dataRowKeySchema.getFieldCount() - 1;
        while (index >= 0 && !dataRowKeySchema.getField(index).getDataType().isFixedWidth() && length > minLength && dataRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
            length--;
            index--;
        }
        // there to maintain compatibility between an old client and a new server.
        if (isDataTableSalted) {
            // Set salt byte
            byte saltByte = SaltingUtil.getSaltingByte(dataRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
            dataRowKey[0] = saltByte;
        }
        return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 7 with TrustedByteArrayOutputStream

use of org.apache.phoenix.util.TrustedByteArrayOutputStream in project phoenix by apache.

the class IndexMaintainer method buildRowKey.

public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte[] regionStartKey, byte[] regionEndKey, long ts) {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    boolean prependRegionStartKey = isLocalIndex && regionStartKey != null;
    boolean isIndexSalted = !isLocalIndex && nIndexSaltBuckets > 0;
    int prefixKeyLength = prependRegionStartKey ? (regionStartKey.length != 0 ? regionStartKey.length : regionEndKey.length) : 0;
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes + (prependRegionStartKey ? prefixKeyLength : 0));
    DataOutput output = new DataOutputStream(stream);
    try {
        // For local indexes, we must prepend the row key with the start region key
        if (prependRegionStartKey) {
            if (regionStartKey.length == 0) {
                output.write(new byte[prefixKeyLength]);
            } else {
                output.write(regionStartKey);
            }
        }
        if (isIndexSalted) {
            // will be set at end to index salt byte
            output.write(0);
        }
        // The dataRowKeySchema includes the salt byte field,
        // so we must adjust for that here.
        int dataPosOffset = isDataTableSalted ? 1 : 0;
        BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
        int nIndexedColumns = getIndexPkColumnCount() - getNumViewConstants();
        int[][] dataRowKeyLocator = new int[2][nIndexedColumns];
        // Skip data table salt byte
        int maxRowKeyOffset = rowKeyPtr.getOffset() + rowKeyPtr.getLength();
        dataRowKeySchema.iterator(rowKeyPtr, ptr, dataPosOffset);
        if (viewIndexId != null) {
            output.write(viewIndexId);
        }
        if (isMultiTenant) {
            dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
            }
            dataPosOffset++;
        }
        // Write index row key
        for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
            Boolean hasValue = dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
            // same for all rows in this index)
            if (!viewConstantColumnBitSet.get(i)) {
                int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
                if (Boolean.TRUE.equals(hasValue)) {
                    dataRowKeyLocator[0][pos] = ptr.getOffset();
                    dataRowKeyLocator[1][pos] = ptr.getLength();
                } else {
                    dataRowKeyLocator[0][pos] = 0;
                    dataRowKeyLocator[1][pos] = 0;
                }
            }
        }
        BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
        Iterator<Expression> expressionIterator = indexedExpressions.iterator();
        for (int i = 0; i < nIndexedColumns; i++) {
            PDataType dataColumnType;
            boolean isNullable;
            SortOrder dataSortOrder;
            if (dataPkPosition[i] == EXPRESSION_NOT_PRESENT) {
                Expression expression = expressionIterator.next();
                dataColumnType = expression.getDataType();
                dataSortOrder = expression.getSortOrder();
                isNullable = expression.isNullable();
                expression.evaluate(new ValueGetterTuple(valueGetter, ts), ptr);
            } else {
                Field field = dataRowKeySchema.getField(dataPkPosition[i]);
                dataColumnType = field.getDataType();
                ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
                dataSortOrder = field.getSortOrder();
                isNullable = field.isNullable();
            }
            boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
            PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);
            boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
            boolean isIndexColumnDesc = descIndexColumnBitSet.get(i);
            if (isBytesComparable && isDataColumnInverted == isIndexColumnDesc) {
                output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            } else {
                if (!isBytesComparable) {
                    indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
                }
                if (isDataColumnInverted != isIndexColumnDesc) {
                    writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                } else {
                    output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                }
            }
            if (!indexColumnType.isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, isIndexColumnDesc ? SortOrder.DESC : SortOrder.ASC));
            }
        }
        int length = stream.size();
        int minLength = length - maxTrailingNulls;
        byte[] indexRowKey = stream.getBuffer();
        // Remove trailing nulls
        while (length > minLength && indexRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
            length--;
        }
        if (isIndexSalted) {
            // Set salt byte
            byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
            indexRowKey[0] = saltByte;
        }
        return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 8 with TrustedByteArrayOutputStream

use of org.apache.phoenix.util.TrustedByteArrayOutputStream in project phoenix by apache.

the class PTableImpl method newKey.

@Override
public int newKey(ImmutableBytesWritable key, byte[][] values) {
    List<PColumn> columns = getPKColumns();
    int nValues = values.length;
    while (nValues > 0 && (values[nValues - 1] == null || values[nValues - 1].length == 0)) {
        nValues--;
    }
    for (PColumn column : columns) {
        if (column.getExpressionStr() != null) {
            nValues++;
        }
    }
    int i = 0;
    TrustedByteArrayOutputStream os = new TrustedByteArrayOutputStream(SchemaUtil.estimateKeyLength(this));
    try {
        Integer bucketNum = this.getBucketNum();
        if (bucketNum != null) {
            // Write place holder for salt byte
            i++;
            os.write(QueryConstants.SEPARATOR_BYTE_ARRAY);
        }
        int nColumns = columns.size();
        PDataType type = null;
        SortOrder sortOrder = null;
        boolean wasNull = false;
        while (i < nValues && i < nColumns) {
            // Separate variable length column values in key with zero byte
            if (type != null && !type.isFixedWidth()) {
                os.write(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable(), wasNull, sortOrder));
            }
            PColumn column = columns.get(i);
            sortOrder = column.getSortOrder();
            type = column.getDataType();
            // This will throw if the value is null and the type doesn't allow null
            byte[] byteValue = values[i++];
            if (byteValue == null) {
                if (column.getExpressionStr() != null) {
                    try {
                        String url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + PhoenixRuntime.CONNECTIONLESS;
                        PhoenixConnection conn = DriverManager.getConnection(url).unwrap(PhoenixConnection.class);
                        StatementContext context = new StatementContext(new PhoenixStatement(conn));
                        ExpressionCompiler compiler = new ExpressionCompiler(context);
                        ParseNode defaultParseNode = new SQLParser(column.getExpressionStr()).parseExpression();
                        Expression defaultExpression = defaultParseNode.accept(compiler);
                        defaultExpression.evaluate(null, key);
                        column.getDataType().coerceBytes(key, null, defaultExpression.getDataType(), defaultExpression.getMaxLength(), defaultExpression.getScale(), defaultExpression.getSortOrder(), column.getMaxLength(), column.getScale(), column.getSortOrder());
                        byteValue = ByteUtil.copyKeyBytesIfNecessary(key);
                    } catch (SQLException e) {
                        // should not be possible
                        throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " failed to compile default value expression of " + column.getExpressionStr());
                    }
                } else {
                    byteValue = ByteUtil.EMPTY_BYTE_ARRAY;
                }
            }
            wasNull = byteValue.length == 0;
            // here.
            if (byteValue.length == 0 && !column.isNullable()) {
                throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
            }
            Integer maxLength = column.getMaxLength();
            Integer scale = column.getScale();
            key.set(byteValue);
            if (!type.isSizeCompatible(key, null, type, sortOrder, null, null, maxLength, scale)) {
                throw new DataExceedsCapacityException(name.getString() + "." + column.getName().getString() + " may not exceed " + maxLength + " (" + SchemaUtil.toString(type, byteValue) + ")");
            }
            key.set(byteValue);
            type.pad(key, maxLength, sortOrder);
            byteValue = ByteUtil.copyKeyBytesIfNecessary(key);
            os.write(byteValue, 0, byteValue.length);
        }
        // Need trailing byte for DESC columns
        if (type != null && !type.isFixedWidth() && SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable(), wasNull, sortOrder) == QueryConstants.DESC_SEPARATOR_BYTE) {
            os.write(QueryConstants.DESC_SEPARATOR_BYTE);
        }
        // If some non null pk values aren't set, then throw
        if (i < nColumns) {
            PColumn column = columns.get(i);
            if (column.getDataType().isFixedWidth() || !column.isNullable()) {
                throw new ConstraintViolationException(name.getString() + "." + column.getName().getString() + " may not be null");
            }
        }
        if (nValues == 0) {
            throw new ConstraintViolationException("Primary key may not be null (" + name.getString() + ")");
        }
        byte[] buf = os.getBuffer();
        int size = os.size();
        if (bucketNum != null) {
            buf[0] = SaltingUtil.getSaltingByte(buf, 1, size - 1, bucketNum);
        }
        key.set(buf, 0, size);
        return i;
    } finally {
        try {
            os.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 9 with TrustedByteArrayOutputStream

use of org.apache.phoenix.util.TrustedByteArrayOutputStream in project phoenix by apache.

the class HashCacheClient method serialize.

private void serialize(ImmutableBytesWritable ptr, ResultIterator iterator, long estimatedSize, List<Expression> onExpressions, boolean singleValueOnly, Expression keyRangeRhsExpression, List<Expression> keyRangeRhsValues) throws SQLException {
    long maxSize = serverCache.getConnection().getQueryServices().getProps().getLong(QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_SERVER_CACHE_SIZE);
    estimatedSize = Math.min(estimatedSize, maxSize);
    if (estimatedSize > Integer.MAX_VALUE) {
        throw new IllegalStateException("Estimated size(" + estimatedSize + ") must not be greater than Integer.MAX_VALUE(" + Integer.MAX_VALUE + ")");
    }
    try {
        TrustedByteArrayOutputStream baOut = new TrustedByteArrayOutputStream((int) estimatedSize);
        DataOutputStream out = new DataOutputStream(baOut);
        // Write onExpressions first, for hash key evaluation along with deserialization
        out.writeInt(onExpressions.size());
        for (Expression expression : onExpressions) {
            WritableUtils.writeVInt(out, ExpressionType.valueOf(expression).ordinal());
            expression.write(out);
        }
        int exprSize = baOut.size() + Bytes.SIZEOF_INT;
        out.writeInt(exprSize * (singleValueOnly ? -1 : 1));
        int nRows = 0;
        // In the end will be replaced with total number of rows
        out.writeInt(nRows);
        ImmutableBytesWritable tempPtr = new ImmutableBytesWritable();
        for (Tuple result = iterator.next(); result != null; result = iterator.next()) {
            TupleUtil.write(result, out);
            if (baOut.size() > maxSize) {
                throw new MaxServerCacheSizeExceededException("Size of hash cache (" + baOut.size() + " bytes) exceeds the maximum allowed size (" + maxSize + " bytes)");
            }
            // Evaluate key expressions for hash join key range optimization.
            if (keyRangeRhsExpression != null) {
                keyRangeRhsValues.add(evaluateKeyExpression(keyRangeRhsExpression, result, tempPtr));
            }
            nRows++;
        }
        TrustedByteArrayOutputStream sizeOut = new TrustedByteArrayOutputStream(Bytes.SIZEOF_INT);
        DataOutputStream dataOut = new DataOutputStream(sizeOut);
        try {
            dataOut.writeInt(nRows);
            dataOut.flush();
            byte[] cache = baOut.getBuffer();
            // Replace number of rows written above with the correct value.
            System.arraycopy(sizeOut.getBuffer(), 0, cache, exprSize, sizeOut.size());
            // Reallocate to actual size plus compressed buffer size (which is allocated below)
            int maxCompressedSize = Snappy.maxCompressedLength(baOut.size());
            // size for worst case
            byte[] compressed = new byte[maxCompressedSize];
            int compressedSize = Snappy.compress(baOut.getBuffer(), 0, baOut.size(), compressed, 0);
            // Last realloc to size of compressed buffer.
            ptr.set(compressed, 0, compressedSize);
        } finally {
            dataOut.close();
        }
    } catch (IOException e) {
        throw ServerUtil.parseServerException(e);
    } finally {
        iterator.close();
    }
}

Example 10 with TrustedByteArrayOutputStream

use of org.apache.phoenix.util.TrustedByteArrayOutputStream in project phoenix by apache.

the class RowValueConstructorExpression method evaluate.

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    if (literalExprPtr != null) {
        // if determined during construction that the row value constructor is just comprised of literal expressions,
        // let's just return the ptr we have already computed and be done with evaluation.
        ptr.set(literalExprPtr.get(), literalExprPtr.getOffset(), literalExprPtr.getLength());
        return true;
    }
    try {
        boolean isPartialEval = this.partialEvalIndex >= 0;
        int evalIndex = isPartialEval ? this.partialEvalIndex : 0;
        int expressionCount = evalIndex;
        for (; evalIndex < ptrs.length; evalIndex++) {
            final Expression expression = children.get(evalIndex);
            // TODO: handle overflow and underflow
            if (expression.evaluate(tuple, ptr)) {
                if (ptr.getLength() == 0) {
                    estimatedByteSize += getExpressionByteCount(expression);
                } else {
                    expressionCount = evalIndex + 1;
                    ptrs[evalIndex] = new ImmutableBytesWritable();
                    ptrs[evalIndex].set(ptr.get(), ptr.getOffset(), ptr.getLength());
                    // 1 extra for the separator byte.
                    estimatedByteSize += ptr.getLength() + (expression.getDataType().isFixedWidth() ? 0 : 1);
                }
            } else if (tuple == null || tuple.isImmutable()) {
                estimatedByteSize += getExpressionByteCount(expression);
            } else {
                // Cannot yet be evaluated
                return false;
            }
        }
        if (isPartialEval) {
            // Move counter forward
            this.partialEvalIndex = evalIndex;
        }
        if (evalIndex == ptrs.length) {
            if (expressionCount == 0) {
                ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
                return true;
            }
            if (expressionCount == 1) {
                ptr.set(ptrs[0].get(), ptrs[0].getOffset(), ptrs[0].getLength());
                return true;
            }
            TrustedByteArrayOutputStream output = new TrustedByteArrayOutputStream(estimatedByteSize);
            try {
                boolean previousCarryOver = false;
                for (int i = 0; i < expressionCount; i++) {
                    Expression child = getChildren().get(i);
                    PDataType childType = child.getDataType();
                    ImmutableBytesWritable tempPtr = ptrs[i];
                    if (tempPtr == null) {
                        // Since we have a null and have no representation for null,
                        // we must decrement the value of the current. Otherwise,
                        // we'd have an ambiguity if this value happened to be the
                        // min possible value.
                        previousCarryOver = childType == null || childType.isFixedWidth();
                        int bytesToWrite = getExpressionByteCount(child);
                        for (int m = 0; m < bytesToWrite; m++) {
                            output.write(QueryConstants.SEPARATOR_BYTE);
                        }
                    } else {
                        output.write(tempPtr.get(), tempPtr.getOffset(), tempPtr.getLength());
                        if (!childType.isFixedWidth()) {
                            output.write(SchemaUtil.getSeparatorByte(true, false, child));
                        }
                        if (previousCarryOver) {
                            previousCarryOver = !ByteUtil.previousKey(output.getBuffer(), output.size());
                        }
                    }
                }
                int outputSize = output.size();
                byte[] outputBytes = output.getBuffer();
                // as otherwise we need it to ensure sort order is correct
                for (int k = expressionCount - 1; k >= 0 && getChildren().get(k).getDataType() != null && !getChildren().get(k).getDataType().isFixedWidth() && outputBytes[outputSize - 1] == SchemaUtil.getSeparatorByte(true, false, getChildren().get(k)); k--) {
                    outputSize--;
                }
                ptr.set(outputBytes, 0, outputSize);
                return true;
            } finally {
                output.close();
            }
        }
        return false;
    } catch (IOException e) {
        // Impossible.
        throw new RuntimeException(e);
    }
}