Examples with RowKeySchema org.apache.phoenix.schema.RowKeySchema used on opensource projects

Example 1 with RowKeySchema

use of org.apache.phoenix.schema.RowKeySchema in project phoenix by apache.

the class WhereOptimizer method pushKeyExpressionsToScan.

// For testing so that the extractedNodes can be verified
public static Expression pushKeyExpressionsToScan(StatementContext context, FilterableStatement statement, Expression whereClause, Set<Expression> extractNodes) throws SQLException {
    PName tenantId = context.getConnection().getTenantId();
    byte[] tenantIdBytes = null;
    PTable table = context.getCurrentTable().getTable();
    Integer nBuckets = table.getBucketNum();
    boolean isSalted = nBuckets != null;
    RowKeySchema schema = table.getRowKeySchema();
    boolean isMultiTenant = tenantId != null && table.isMultiTenant();
    boolean isSharedIndex = table.getViewIndexId() != null;
    if (isMultiTenant) {
        tenantIdBytes = ScanUtil.getTenantIdBytes(schema, isSalted, tenantId, isSharedIndex);
    }
    if (whereClause == null && (tenantId == null || !table.isMultiTenant()) && table.getViewIndexId() == null) {
        context.setScanRanges(ScanRanges.EVERYTHING);
        return whereClause;
    }
    if (LiteralExpression.isBooleanFalseOrNull(whereClause)) {
        context.setScanRanges(ScanRanges.NOTHING);
        return null;
    }
    KeyExpressionVisitor visitor = new KeyExpressionVisitor(context, table);
    KeyExpressionVisitor.KeySlots keySlots = null;
    if (whereClause != null) {
        // TODO:: When we only have one where clause, the keySlots returns as a single slot object,
        // instead of an array of slots for the corresponding column. Change the behavior so it
        // becomes consistent.
        keySlots = whereClause.accept(visitor);
        if (keySlots == null && (tenantId == null || !table.isMultiTenant()) && table.getViewIndexId() == null) {
            context.setScanRanges(ScanRanges.EVERYTHING);
            return whereClause;
        }
        // for unequal lengths.
        if (keySlots == KeyExpressionVisitor.EMPTY_KEY_SLOTS) {
            context.setScanRanges(ScanRanges.NOTHING);
            return null;
        }
    }
    if (keySlots == null) {
        keySlots = KeyExpressionVisitor.EMPTY_KEY_SLOTS;
    }
    if (extractNodes == null) {
        extractNodes = new HashSet<Expression>(table.getPKColumns().size());
    }
    int pkPos = 0;
    int nPKColumns = table.getPKColumns().size();
    int[] slotSpan = new int[nPKColumns];
    List<List<KeyRange>> cnf = Lists.newArrayListWithExpectedSize(schema.getMaxFields());
    KeyRange minMaxRange = keySlots.getMinMaxRange();
    if (minMaxRange == null) {
        minMaxRange = KeyRange.EVERYTHING_RANGE;
    }
    boolean hasMinMaxRange = (minMaxRange != KeyRange.EVERYTHING_RANGE);
    int minMaxRangeOffset = 0;
    byte[] minMaxRangePrefix = null;
    boolean hasViewIndex = table.getViewIndexId() != null;
    if (hasMinMaxRange) {
        int minMaxRangeSize = (isSalted ? SaltingUtil.NUM_SALTING_BYTES : 0) + (isMultiTenant ? tenantIdBytes.length + 1 : 0) + (hasViewIndex ? MetaDataUtil.getViewIndexIdDataType().getByteSize() : 0);
        minMaxRangePrefix = new byte[minMaxRangeSize];
    }
    Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.iterator();
    // Add placeholder for salt byte ranges
    if (isSalted) {
        cnf.add(SALT_PLACEHOLDER);
        if (hasMinMaxRange) {
            System.arraycopy(SALT_PLACEHOLDER.get(0).getLowerRange(), 0, minMaxRangePrefix, minMaxRangeOffset, SaltingUtil.NUM_SALTING_BYTES);
            minMaxRangeOffset += SaltingUtil.NUM_SALTING_BYTES;
        }
        // Increment the pkPos, as the salt column is in the row schema
        // Do not increment the iterator, though, as there will never be
        // an expression in the keySlots for the salt column
        pkPos++;
    }
    // that different indexes don't interleave.
    if (hasViewIndex) {
        byte[] viewIndexBytes = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
        KeyRange indexIdKeyRange = KeyRange.getKeyRange(viewIndexBytes);
        cnf.add(singletonList(indexIdKeyRange));
        if (hasMinMaxRange) {
            System.arraycopy(viewIndexBytes, 0, minMaxRangePrefix, minMaxRangeOffset, viewIndexBytes.length);
            minMaxRangeOffset += viewIndexBytes.length;
        }
        pkPos++;
    }
    // Add tenant data isolation for tenant-specific tables
    if (isMultiTenant) {
        KeyRange tenantIdKeyRange = KeyRange.getKeyRange(tenantIdBytes);
        cnf.add(singletonList(tenantIdKeyRange));
        if (hasMinMaxRange) {
            System.arraycopy(tenantIdBytes, 0, minMaxRangePrefix, minMaxRangeOffset, tenantIdBytes.length);
            minMaxRangeOffset += tenantIdBytes.length;
            Field f = schema.getField(pkPos);
            if (!f.getDataType().isFixedWidth()) {
                minMaxRangePrefix[minMaxRangeOffset] = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), tenantIdBytes.length == 0, f);
                minMaxRangeOffset++;
            }
        }
        pkPos++;
    }
    // range with the other range.
    if (hasMinMaxRange) {
        minMaxRange = minMaxRange.prependRange(minMaxRangePrefix, 0, minMaxRangeOffset);
    }
    boolean forcedSkipScan = statement.getHint().hasHint(Hint.SKIP_SCAN);
    boolean forcedRangeScan = statement.getHint().hasHint(Hint.RANGE_SCAN);
    boolean hasUnboundedRange = false;
    boolean hasMultiRanges = false;
    boolean hasRangeKey = false;
    boolean stopExtracting = false;
    boolean useSkipScan = false;
    // Concat byte arrays of literals to form scan start key
    while (iterator.hasNext()) {
        KeyExpressionVisitor.KeySlot slot = iterator.next();
        // If the slot is null this means we have no entry for this pk position.
        if (slot == null || slot.getKeyRanges().isEmpty()) {
            continue;
        }
        if (slot.getPKPosition() != pkPos) {
            if (!forcedSkipScan) {
                stopExtracting = true;
            } else {
                useSkipScan |= !stopExtracting && !forcedRangeScan && forcedSkipScan;
            }
            for (int i = pkPos; i < slot.getPKPosition(); i++) {
                cnf.add(Collections.singletonList(KeyRange.EVERYTHING_RANGE));
            }
        }
        KeyPart keyPart = slot.getKeyPart();
        slotSpan[cnf.size()] = slot.getPKSpan() - 1;
        pkPos = slot.getPKPosition() + slot.getPKSpan();
        // Skip span-1 slots as we skip one at the top of the loop
        for (int i = 1; i < slot.getPKSpan() && iterator.hasNext(); i++) {
            iterator.next();
        }
        List<KeyRange> keyRanges = slot.getKeyRanges();
        cnf.add(keyRanges);
        // TODO: when stats are available, we may want to use a skip scan if the
        // cardinality of this slot is low.
        /*
             *  Stop extracting nodes once we encounter:
             *  1) An unbound range unless we're forcing a skip scan and havn't encountered
             *     a multi-column span. Even if we're trying to force a skip scan, we can't
             *     execute it over a multi-column span.
             *  2) A non range key as we can extract the first one, but further ones need
             *     to be evaluated in a filter.
             */
        stopExtracting |= (hasUnboundedRange && !forcedSkipScan) || (hasRangeKey && forcedRangeScan);
        useSkipScan |= !stopExtracting && !forcedRangeScan && (keyRanges.size() > 1 || hasRangeKey);
        for (int i = 0; (!hasUnboundedRange || !hasRangeKey) && i < keyRanges.size(); i++) {
            KeyRange range = keyRanges.get(i);
            if (range.isUnbound()) {
                hasUnboundedRange = hasRangeKey = true;
            } else if (!range.isSingleKey()) {
                hasRangeKey = true;
            }
        }
        hasMultiRanges |= keyRanges.size() > 1;
        // We cannot extract if we have multiple ranges and are forcing a range scan.
        stopExtracting |= forcedRangeScan && hasMultiRanges;
        // that, so must filter on the remaining conditions (see issue #467).
        if (!stopExtracting) {
            List<Expression> nodesToExtract = keyPart.getExtractNodes();
            extractNodes.addAll(nodesToExtract);
        }
    }
    // If we have fully qualified point keys with multi-column spans (i.e. RVC),
    // we can still use our skip scan. The ScanRanges.create() call will explode
    // out the keys.
    slotSpan = Arrays.copyOf(slotSpan, cnf.size());
    ScanRanges scanRanges = ScanRanges.create(schema, cnf, slotSpan, minMaxRange, nBuckets, useSkipScan, table.getRowTimestampColPos());
    context.setScanRanges(scanRanges);
    if (whereClause == null) {
        return null;
    } else {
        return whereClause.accept(new RemoveExtractedNodesVisitor(extractNodes));
    }
}

Example 2 with RowKeySchema

use of org.apache.phoenix.schema.RowKeySchema in project phoenix by apache.

the class SkipScanFilter method readFields.

@Override
public void readFields(DataInput in) throws IOException {
    RowKeySchema schema = new RowKeySchema();
    schema.readFields(in);
    int andLen = in.readInt();
    boolean includeMultipleVersions = false;
    if (andLen < 0) {
        andLen = -andLen;
        includeMultipleVersions = true;
    }
    int[] slotSpan = new int[andLen];
    List<List<KeyRange>> slots = Lists.newArrayListWithExpectedSize(andLen);
    for (int i = 0; i < andLen; i++) {
        int orLenWithSlotSpan = in.readInt();
        int orLen = orLenWithSlotSpan;
        /*
             * For 4.2+ clients, we serialize the slotSpan array. To maintain backward
             * compatibility, we encode the slotSpan values with the size of the list
             * of key ranges. We reserve 21 bits for the key range list and 10 bits
             * for the slotSpan value (up to 1024 which should be plenty).
             */
        if (orLenWithSlotSpan < 0) {
            orLenWithSlotSpan = -orLenWithSlotSpan - 1;
            slotSpan[i] = orLenWithSlotSpan >>> KEY_RANGE_LENGTH_BITS;
            orLen = (orLenWithSlotSpan << SLOT_SPAN_BITS) >>> SLOT_SPAN_BITS;
        }
        List<KeyRange> orClause = Lists.newArrayListWithExpectedSize(orLen);
        slots.add(orClause);
        for (int j = 0; j < orLen; j++) {
            KeyRange range = KeyRange.read(in);
            orClause.add(range);
        }
    }
    this.init(slots, slotSpan, schema, includeMultipleVersions);
}

Example 3 with RowKeySchema

use of org.apache.phoenix.schema.RowKeySchema in project phoenix by apache.

the class IndexMaintainer method getViewIndexIdFromIndexRowKey.

/*
     * return the view index id from the index row key
     */
public byte[] getViewIndexIdFromIndexRowKey(ImmutableBytesWritable indexRowKeyPtr) {
    assert (isLocalIndex);
    RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
    // TODO add logic to skip region start key as well because we cannot find the region startkey in indexhalfstorefilereader.
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
    DataOutput output = new DataOutputStream(stream);
    try {
        int indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
        Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
        if (Boolean.TRUE.equals(hasValue)) {
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
        }
        int length = stream.size();
        byte[] dataRowKey = stream.getBuffer();
        return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 4 with RowKeySchema

use of org.apache.phoenix.schema.RowKeySchema in project phoenix by apache.

the class IndexMaintainer method buildDataRowKey.

/*
     * Build the data row key from the index row key
     */
public byte[] buildDataRowKey(ImmutableBytesWritable indexRowKeyPtr, byte[][] viewConstants) {
    RowKeySchema indexRowKeySchema = getIndexRowKeySchema();
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(estimatedIndexRowKeyBytes);
    DataOutput output = new DataOutputStream(stream);
    // Increment dataPosOffset until all have been written
    int dataPosOffset = 0;
    int viewConstantsIndex = 0;
    try {
        int indexPosOffset = !isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0;
        int maxRowKeyOffset = indexRowKeyPtr.getOffset() + indexRowKeyPtr.getLength();
        indexRowKeySchema.iterator(indexRowKeyPtr, ptr, indexPosOffset);
        if (isDataTableSalted) {
            dataPosOffset++;
            // will be set at end to salt byte
            output.write(0);
        }
        if (viewIndexId != null) {
            indexRowKeySchema.next(ptr, indexPosOffset++, maxRowKeyOffset);
        }
        if (isMultiTenant) {
            indexRowKeySchema.next(ptr, indexPosOffset, maxRowKeyOffset);
            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
            if (!dataRowKeySchema.getField(dataPosOffset).getDataType().isFixedWidth()) {
                output.writeByte(SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(dataPosOffset)));
            }
            indexPosOffset++;
            dataPosOffset++;
        }
        indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
        BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
        BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
        for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
            // same for all rows in this index)
            if (viewConstantColumnBitSet.get(i)) {
                output.write(viewConstants[viewConstantsIndex++]);
            } else {
                int pos = rowKeyMetaData.getIndexPkPosition(i - dataPosOffset);
                Boolean hasValue = indexRowKeySchema.iterator(indexRowKeyPtr, ptr, pos + indexPosOffset + 1);
                if (Boolean.TRUE.equals(hasValue)) {
                    // Write data row key value taking into account coercion and inversion
                    // if necessary
                    Field dataField = dataRowKeySchema.getField(i);
                    Field indexField = indexRowKeySchema.getField(pos + indexPosOffset);
                    PDataType indexColumnType = indexField.getDataType();
                    PDataType dataColumnType = dataField.getDataType();
                    SortOrder dataSortOrder = dataField.getSortOrder();
                    SortOrder indexSortOrder = indexField.getSortOrder();
                    boolean isDataColumnInverted = dataSortOrder != SortOrder.ASC;
                    boolean isBytesComparable = dataColumnType.isBytesComparableWith(indexColumnType);
                    if (isBytesComparable && isDataColumnInverted == descIndexColumnBitSet.get(pos)) {
                        output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                    } else {
                        if (!isBytesComparable) {
                            dataColumnType.coerceBytes(ptr, indexColumnType, indexSortOrder, SortOrder.getDefault());
                        }
                        if (descIndexColumnBitSet.get(pos) != isDataColumnInverted) {
                            writeInverted(ptr.get(), ptr.getOffset(), ptr.getLength(), output);
                        } else {
                            output.write(ptr.get(), ptr.getOffset(), ptr.getLength());
                        }
                    }
                }
            }
            // Write separator byte if variable length unless it's the last field in the schema
            // (but we still need to write it if it's DESC to ensure sort order is correct).
            byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
            if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i + 1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
                output.writeByte(sepByte);
            }
        }
        int length = stream.size();
        int minLength = length - maxTrailingNulls;
        byte[] dataRowKey = stream.getBuffer();
        // Remove trailing nulls
        int index = dataRowKeySchema.getFieldCount() - 1;
        while (index >= 0 && !dataRowKeySchema.getField(index).getDataType().isFixedWidth() && length > minLength && dataRowKey[length - 1] == QueryConstants.SEPARATOR_BYTE) {
            length--;
            index--;
        }
        // there to maintain compatibility between an old client and a new server.
        if (isDataTableSalted) {
            // Set salt byte
            byte saltByte = SaltingUtil.getSaltingByte(dataRowKey, SaltingUtil.NUM_SALTING_BYTES, length - SaltingUtil.NUM_SALTING_BYTES, nIndexSaltBuckets);
            dataRowKey[0] = saltByte;
        }
        return dataRowKey.length == length ? dataRowKey : Arrays.copyOf(dataRowKey, length);
    } catch (IOException e) {
        // Impossible
        throw new RuntimeException(e);
    } finally {
        try {
            stream.close();
        } catch (IOException e) {
            // Impossible
            throw new RuntimeException(e);
        }
    }
}

Example 5 with RowKeySchema

use of org.apache.phoenix.schema.RowKeySchema in project phoenix by apache.

the class MutationState method getNewRowKeyWithRowTimestamp.

private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) {
    RowKeySchema schema = table.getRowKeySchema();
    int rowTimestampColPos = table.getRowTimestampColPos();
    Field rowTimestampField = schema.getField(rowTimestampColPos);
    byte[] rowTimestampBytes = PLong.INSTANCE.toBytes(rowTimestamp, rowTimestampField.getSortOrder());
    int oldOffset = ptr.getOffset();
    int oldLength = ptr.getLength();
    // Move the pointer to the start byte of the row timestamp pk
    schema.position(ptr, 0, rowTimestampColPos);
    byte[] b = ptr.get();
    int newOffset = ptr.getOffset();
    int length = ptr.getLength();
    for (int i = newOffset; i < newOffset + length; i++) {
        // modify the underlying bytes array with the bytes of the row timestamp
        b[i] = rowTimestampBytes[i - newOffset];
    }
    // move the pointer back to where it was before.
    ptr.set(ptr.get(), oldOffset, oldLength);
    return ptr;
}