Examples with BitSet org.apache.phoenix.util.BitSet used on opensource projects

Example 1 with BitSet

use of org.apache.phoenix.util.BitSet in project phoenix by apache.

the class IndexMaintainer method getNumViewConstants.

private int getNumViewConstants() {
    BitSet bitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
    int num = 0;
    for (int i = 0; i < dataRowKeySchema.getFieldCount(); i++) {
        if (bitSet.get(i))
            num++;
    }
    return num;
}

Example 2 with BitSet

use of org.apache.phoenix.util.BitSet in project phoenix by apache.

the class IndexMaintainer method buildRowKey.

public byte[] buildRowKey(ValueGetter valueGetter, ImmutableBytesWritable rowKeyPtr, byte[] regionStartKey, byte[] regionEndKey) {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    boolean prependRegionStartKey = isLocalIndex && regionStartKey != null;
    boolean isIndexSalted = !isLocalIndex && nIndexSaltBuckets > 0;
    int prefixKeyLength = prependRegionStartKey ? (regionStartKey.length != 0 ? regionStartKey.length : regionEndKey.length) : 0;
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes + (prependRegionStartKey ? prefixKeyLength : 0));
    DataOutput output = new DataOutputStream(stream);
    try {
        // For local indexes, we must prepend the row key with the start region key
        if (prependRegionStartKey) {
            if (regionStartKey.length == 0) {
                output.write(new byte[prefixKeyLength]);
            } else {
                output.write(regionStartKey);
            }
        }
        if (isIndexSalted) {
            // will be set at end to index salt byte
            output.write(0);
        }
        // The dataRowKeySchema includes the salt byte field,
        // so we must adjust for that here.
        int dataPosOffset = isDataTableSalted ? 1 : 0;
        BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
        int nIndexedColumns = getIndexPkColumnCount() - getNumViewConstants();
        int[][] dataRowKeyLocator = new int[2][nIndexedColumns];
        // Skip data table salt byte
        int maxRowKeyOffset = rowKeyPtr.getOffset() + rowKeyPtr.getLength();
        dataRowKeySchema.iterator(rowKeyPtr, ptr, dataPosOffset);
        if (viewIndexId != null) {
            output.write(viewIndexId);
        }
        if (isMultiTenant) {
            dataRowKeySchema.next(ptr, dataPosOffset, maxRowKeyOffset);
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
            }
            dataPosOffset++;
        }
        // Write index row key
        for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
            Boolean hasValue = dataRowKeySchema.next(ptr, i, maxRowKeyOffset);
            // same for all rows in this index)
            if (!viewConstantColumnBitSet.get(i)) {
                int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
                if (Boolean.TRUE.equals(hasValue)) {
                    dataRowKeyLocator[0][pos] = ptr.getOffset();
                    dataRowKeyLocator[1][pos] = ptr.getLength();
                } else {
                    dataRowKeyLocator[0][pos] = 0;
                    dataRowKeyLocator[1][pos] = 0;
                }
            }
        }
        BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
        Iterator<Expression> expressionIterator = indexedExpressions.iterator();
        for (int i = 0; i < nIndexedColumns; i++) {
            PDataType dataColumnType;
            boolean isNullable;
            SortOrder dataSortOrder;
            if (dataPkPosition[i] == EXPRESSION_NOT_PRESENT) {
                Expression expression = expressionIterator.next();
                dataColumnType = expression.getDataType();
                dataSortOrder = expression.getSortOrder();
                isNullable = expression.isNullable();
                expression.evaluate(new ValueGetterTuple(valueGetter), ptr);
            } else {
                Field field = dataRowKeySchema.getField(dataPkPosition[i]);
                dataColumnType = field.getDataType();
                ptr.set(rowKeyPtr.get(), dataRowKeyLocator[0][i], dataRowKeyLocator[1][i]);
                dataSortOrder = field.getSortOrder();
                isNullable = field.isNullable();
            }
            boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
            PDataType indexColumnType = IndexUtil.getIndexColumnDataType(isNullable, dataColumnType);
            boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
            boolean isIndexColumnDesc = descIndexColumnBitSet.get(i);
            if (isBytesComparable && isDataColumnInverted == isIndexColumnDesc) {
                output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            } else {
                if (!isBytesComparable) {
                    indexColumnType.coerceBytes(ptr, dataColumnType, dataSortOrder, SortOrder.getDefault());
                }
                if (isDataColumnInverted != isIndexColumnDesc) {
                    writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                } else {
                    output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                }
            }
            if (!indexColumnType.isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, isIndexColumnDesc ? SortOrder.DESC : SortOrder.ASC));
            }
        }
        int length = stream.size();
        int minLength = length - maxTrailingNulls;
        byte[] indexRowKey = stream.getBuffer();
        // Remove trailing nulls
        while (length > minLength && indexRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
            length--;
        }
        if (isIndexSalted) {
            // Set salt byte
            byte saltByte = SaltingUtil.getSaltingByte(indexRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
            indexRowKey[0] = saltByte;
        }
        return indexRowKey.length == length ? indexRowKey : Arrays.copyOf(indexRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 3 with BitSet

use of org.apache.phoenix.util.BitSet in project phoenix by apache.

the class IndexMaintainer method buildDataRowKey.

/*
     * Build the data row key from the index row key
     */
public byte[] buildDataRowKey(ImmutableBytesWritable indexRowKeyPtr, byte[][] viewConstants) {
    RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
    DataOutput output = new DataOutputStream(stream);
    // Increment dataPosOffset until all have been written
    int dataPosOffset = 0;
    int viewConstantsIndex = 0;
    try {
        int indexPosOffset = !isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0;
        int maxRowKeyOffset = indexRowKeyPtr.getOffset() + indexRowKeyPtr.getLength();
        indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
        if (isDataTableSalted) {
            dataPosOffset++;
            // will be set at end to salt byte
            output.write(0);
        }
        if (viewIndexId != null) {
            indexRowKeySchema.next(ptr, indexPosOffset++, maxRowKeyOffset);
        }
        if (isMultiTenant) {
            indexRowKeySchema.next(ptr, indexPosOffset, maxRowKeyOffset);
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
            }
            indexPosOffset++;
            dataPosOffset++;
        }
        indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
        BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
        BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
        for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
            // same for all rows in this index)
            if (viewConstantColumnBitSet.get(i)) {
                output.write(viewConstants[viewConstantsIndex++]);
            } else {
                int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
                Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, pos + indexPosOffset + 1);
                if (Boolean.TRUE.equals(hasValue)) {
                    // Write data row key value taking into account coercion and inversion
                    // if necessary
                    Field dataField = dataRowKeySchema.getField(i);
                    Field indexField = indexRowKeySchema.getField(pos + indexPosOffset);
                    PDataType indexColumnType = indexField.getDataType();
                    PDataType dataColumnType = dataField.getDataType();
                    SortOrder dataSortOrder = dataField.getSortOrder();
                    SortOrder indexSortOrder = indexField.getSortOrder();
                    boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
                    boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
                    if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(pos)) {
                        output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                    } else {
                        if (!isBytesComparable) {
                            dataColumnType.coerceBytes(ptr, indexColumnType, indexSortOrder, SortOrder.getDefault());
                        }
                        if (descIndexColumnBitSet.get(pos) != isDataColumnInverted) {
                            writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                        } else {
                            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                        }
                    }
                }
            }
            // Write separator byte if variable length unless it's the last field in the schema
            // (but we still need to write it if it's DESC to ensure sort order is correct).
            byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
            if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i + 1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
                output.writeByte(sepByte);
            }
        }
        int length = stream.size();
        int minLength = length - maxTrailingNulls;
        byte[] dataRowKey = stream.getBuffer();
        // Remove trailing nulls
        while (length > minLength && dataRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
            length--;
        }
        // there to maintain compatibility between an old client and a new server.
        if (isDataTableSalted) {
            // Set salt byte
            byte saltByte = SaltingUtil.getSaltingByte(dataRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
            dataRowKey[0] = saltByte;
        }
        return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 4 with BitSet

use of org.apache.phoenix.util.BitSet in project phoenix by apache.

the class IndexMaintainer method generateIndexRowKeySchema.

// We have enough information to generate the index row key schema
private RowKeySchema generateIndexRowKeySchema() {
    int nIndexedColumns = getIndexPkColumnCount() + (isMultiTenant ? 1 : 0) + (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (viewIndexId != null ? 1 : 0) - getNumViewConstants();
    RowKeySchema.RowKeySchemaBuilder builder = new RowKeySchema.RowKeySchemaBuilder(nIndexedColumns);
    builder.rowKeyOrderOptimizable(rowKeyOrderOptimizable);
    if (!isLocalIndex && nIndexSaltBuckets > 0) {
        builder.addField(SaltingUtil.SALTING_COLUMN, false, SortOrder.ASC);
        nIndexedColumns--;
    }
    int dataPosOffset = isDataTableSalted ? 1 : 0;
    if (viewIndexId != null) {
        nIndexedColumns--;
        builder.addField(new PDatum() {

            @Override
            public boolean isNullable() {
                return false;
            }

            @Override
            public PDataType getDataType() {
                return MetaDataUtil.getViewIndexIdDataType();
            }

            @Override
            public Integer getMaxLength() {
                return null;
            }

            @Override
            public Integer getScale() {
                return null;
            }

            @Override
            public SortOrder getSortOrder() {
                return SortOrder.getDefault();
            }
        }, false, SortOrder.getDefault());
    }
    if (isMultiTenant) {
        Field field = dataRowKeySchema.getField(dataPosOffset++);
        builder.addField(field, field.isNullable(), field.getSortOrder());
        nIndexedColumns--;
    }
    Field[] indexFields = new Field[nIndexedColumns];
    BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
    // Add Field for all data row pk columns
    for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
        // same for all rows in this index)
        if (!viewConstantColumnBitSet.get(i)) {
            int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
            indexFields[pos] = dataRowKeySchema.getField(i);
        }
    }
    BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
    Iterator<Expression> expressionItr = indexedExpressions.iterator();
    for (int i = 0; i < indexFields.length; i++) {
        Field indexField = indexFields[i];
        PDataType dataTypeToBe;
        SortOrder sortOrderToBe;
        boolean isNullableToBe;
        Integer maxLengthToBe;
        Integer scaleToBe;
        if (indexField == null) {
            Expression e = expressionItr.next();
            isNullableToBe = e.isNullable();
            dataTypeToBe = IndexUtil.getIndexColumnDataType(isNullableToBe, e.getDataType());
            sortOrderToBe = descIndexColumnBitSet.get(i) ? SortOrder.DESC : SortOrder.ASC;
            maxLengthToBe = e.getMaxLength();
            scaleToBe = e.getScale();
        } else {
            isNullableToBe = indexField.isNullable();
            dataTypeToBe = IndexUtil.getIndexColumnDataType(isNullableToBe, indexField.getDataType());
            sortOrderToBe = descIndexColumnBitSet.get(i) ? SortOrder.DESC : SortOrder.ASC;
            maxLengthToBe = indexField.getMaxLength();
            scaleToBe = indexField.getScale();
        }
        final PDataType dataType = dataTypeToBe;
        final SortOrder sortOrder = sortOrderToBe;
        final boolean isNullable = isNullableToBe;
        final Integer maxLength = maxLengthToBe;
        final Integer scale = scaleToBe;
        builder.addField(new PDatum() {

            @Override
            public boolean isNullable() {
                return isNullable;
            }

            @Override
            public PDataType getDataType() {
                return dataType;
            }

            @Override
            public Integer getMaxLength() {
                return maxLength;
            }

            @Override
            public Integer getScale() {
                return scale;
            }

            @Override
            public SortOrder getSortOrder() {
                return sortOrder;
            }
        }, true, sortOrder);
    }
    return builder.build();
}

Example 5 with BitSet

use of org.apache.phoenix.util.BitSet in project phoenix by apache.

the class IndexMaintainer method initCachedState.

/**
     * Init calculated state reading/creating
     */
private void initCachedState() {
    byte[] emptyKvQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(encodingScheme).getFirst();
    dataEmptyKeyValueRef = new ColumnReference(emptyKeyValueCFPtr.copyBytesIfNecessary(), emptyKvQualifier);
    this.allColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size() + coveredColumnsMap.size());
    // columns that are required to evaluate all expressions in indexedExpressions (not including columns in data row key)
    this.indexedColumns = Sets.newLinkedHashSetWithExpectedSize(indexedExpressions.size());
    for (Expression expression : indexedExpressions) {
        KeyValueExpressionVisitor visitor = new KeyValueExpressionVisitor() {

            @Override
            public Void visit(KeyValueColumnExpression expression) {
                if (indexedColumns.add(new ColumnReference(expression.getColumnFamily(), expression.getColumnQualifier()))) {
                    indexedColumnTypes.add(expression.getDataType());
                }
                return null;
            }
        };
        expression.accept(visitor);
    }
    allColumns.addAll(indexedColumns);
    for (ColumnReference colRef : coveredColumnsMap.keySet()) {
        if (immutableStorageScheme == ImmutableStorageScheme.ONE_CELL_PER_COLUMN) {
            allColumns.add(colRef);
        } else {
            allColumns.add(new ColumnReference(colRef.getFamily(), QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES));
        }
    }
    int dataPkOffset = (isDataTableSalted ? 1 : 0) + (isMultiTenant ? 1 : 0);
    int nIndexPkColumns = getIndexPkColumnCount();
    dataPkPosition = new int[nIndexPkColumns];
    Arrays.fill(dataPkPosition, EXPRESSION_NOT_PRESENT);
    int numViewConstantColumns = 0;
    BitSet viewConstantColumnBitSet = rowKeyMetaData.getViewConstantColumnBitSet();
    for (int i = dataPkOffset; i < dataRowKeySchema.getFieldCount(); i++) {
        if (!viewConstantColumnBitSet.get(i)) {
            int indexPkPosition = rowKeyMetaData.getIndexPkPosition(i - dataPkOffset);
            this.dataPkPosition[indexPkPosition] = i;
        } else {
            numViewConstantColumns++;
        }
    }
    // Calculate the max number of trailing nulls that we should get rid of after building the index row key.
    // We only get rid of nulls for variable length types, so we have to be careful to consider the type of the
    // index table, not the data type of the data table
    int expressionsPos = indexedExpressions.size();
    int indexPkPos = nIndexPkColumns - numViewConstantColumns - 1;
    while (indexPkPos >= 0) {
        int dataPkPos = dataPkPosition[indexPkPos];
        boolean isDataNullable;
        PDataType dataType;
        if (dataPkPos == EXPRESSION_NOT_PRESENT) {
            isDataNullable = true;
            dataType = indexedExpressions.get(--expressionsPos).getDataType();
        } else {
            Field dataField = dataRowKeySchema.getField(dataPkPos);
            dataType = dataField.getDataType();
            isDataNullable = dataField.isNullable();
        }
        PDataType indexDataType = IndexUtil.getIndexColumnDataType(isDataNullable, dataType);
        if (indexDataType.isFixedWidth()) {
            break;
        }
        indexPkPos--;
    }
    maxTrailingNulls = nIndexPkColumns - indexPkPos - 1;
}