Examples with KeyRange org.apache.phoenix.query.KeyRange used on opensource projects

Example 16 with KeyRange

use of org.apache.phoenix.query.KeyRange in project phoenix by apache.

the class ConcatResultIteratorTest method testMergeSort.

@Test
public void testMergeSort() throws Throwable {
    Tuple[] results1 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(C, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))) };
    Tuple[] results2 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(2))) };
    Tuple[] results3 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(3))), new SingleKeyValueTuple(new KeyValue(D, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(4))) };
    final List<PeekingResultIterator> results = new ArrayList<PeekingResultIterator>(Arrays.asList(new PeekingResultIterator[] { new MaterializedResultIterator(Arrays.asList(results1)), new MaterializedResultIterator(Arrays.asList(results2)), new MaterializedResultIterator(Arrays.asList(results3)) }));
    Tuple[] expectedResults = new Tuple[] { new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(3))), new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(2))), new SingleKeyValueTuple(new KeyValue(C, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))), new SingleKeyValueTuple(new KeyValue(D, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(4))) };
    ResultIterators iterators = new ResultIterators() {

        @Override
        public List<PeekingResultIterator> getIterators() throws SQLException {
            return results;
        }

        @Override
        public int size() {
            return results.size();
        }

        @Override
        public void explain(List<String> planSteps) {
        }

        @Override
        public List<KeyRange> getSplits() {
            return Collections.emptyList();
        }

        @Override
        public List<List<Scan>> getScans() {
            return Collections.emptyList();
        }

        @Override
        public void close() throws SQLException {
        }
    };
    ResultIterator scanner = new MergeSortRowKeyResultIterator(iterators);
    AssertResults.assertResults(scanner, expectedResults);
}

Example 17 with KeyRange

use of org.apache.phoenix.query.KeyRange in project phoenix by apache.

the class MergeSortResultIteratorTest method testReverseMergeSort.

@Test
public void testReverseMergeSort() throws Throwable {
    Tuple[] results1 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))) };
    Tuple[] results2 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))) };
    Tuple[] results3 = new Tuple[] { new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))), new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))) };
    final List<PeekingResultIterator> results = new ArrayList<PeekingResultIterator>(Arrays.asList(new PeekingResultIterator[] { new MaterializedResultIterator(Arrays.asList(results1)), new MaterializedResultIterator(Arrays.asList(results2)), new MaterializedResultIterator(Arrays.asList(results3)) }));
    Tuple[] expectedResults = new Tuple[] { new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))), new SingleKeyValueTuple(new KeyValue(B, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))), new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))), new SingleKeyValueTuple(new KeyValue(A, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, Bytes.toBytes(1))) };
    ResultIterators iterators = new ResultIterators() {

        @Override
        public List<PeekingResultIterator> getIterators() throws SQLException {
            return results;
        }

        @Override
        public int size() {
            return results.size();
        }

        @Override
        public void explain(List<String> planSteps) {
        }

        @Override
        public List<KeyRange> getSplits() {
            return Collections.emptyList();
        }

        @Override
        public List<List<Scan>> getScans() {
            return Collections.emptyList();
        }

        @Override
        public void close() throws SQLException {
        }
    };
    ResultIterator scanner = new MergeSortRowKeyResultIterator(iterators, 0, true);
    AssertResults.assertResults(scanner, expectedResults);
}

Example 18 with KeyRange

use of org.apache.phoenix.query.KeyRange in project phoenix by apache.

the class WhereOptimizer method pushKeyExpressionsToScan.

// For testing so that the extractedNodes can be verified
public static Expression pushKeyExpressionsToScan(StatementContext context, FilterableStatement statement, Expression whereClause, Set<Expression> extractNodes) throws SQLException {
    PName tenantId = context.getConnection().getTenantId();
    byte[] tenantIdBytes = null;
    PTable table = context.getCurrentTable().getTable();
    Integer nBuckets = table.getBucketNum();
    boolean isSalted = nBuckets != null;
    RowKeySchema schema = table.getRowKeySchema();
    boolean isMultiTenant = tenantId != null && table.isMultiTenant();
    boolean isSharedIndex = table.getViewIndexId() != null;
    if (isMultiTenant) {
        tenantIdBytes = ScanUtil.getTenantIdBytes(schema, isSalted, tenantId, isSharedIndex);
    }
    if (whereClause == null && (tenantId == null || !table.isMultiTenant()) && table.getViewIndexId() == null) {
        context.setScanRanges(ScanRanges.EVERYTHING);
        return whereClause;
    }
    if (LiteralExpression.isBooleanFalseOrNull(whereClause)) {
        context.setScanRanges(ScanRanges.NOTHING);
        return null;
    }
    KeyExpressionVisitor visitor = new KeyExpressionVisitor(context, table);
    KeyExpressionVisitor.KeySlots keySlots = null;
    if (whereClause != null) {
        // TODO:: When we only have one where clause, the keySlots returns as a single slot object,
        // instead of an array of slots for the corresponding column. Change the behavior so it
        // becomes consistent.
        keySlots = whereClause.accept(visitor);
        if (keySlots == null && (tenantId == null || !table.isMultiTenant()) && table.getViewIndexId() == null) {
            context.setScanRanges(ScanRanges.EVERYTHING);
            return whereClause;
        }
        // for unequal lengths.
        if (keySlots == KeyExpressionVisitor.EMPTY_KEY_SLOTS) {
            context.setScanRanges(ScanRanges.NOTHING);
            return null;
        }
    }
    if (keySlots == null) {
        keySlots = KeyExpressionVisitor.EMPTY_KEY_SLOTS;
    }
    if (extractNodes == null) {
        extractNodes = new HashSet<Expression>(table.getPKColumns().size());
    }
    int pkPos = 0;
    int nPKColumns = table.getPKColumns().size();
    int[] slotSpan = new int[nPKColumns];
    List<List<KeyRange>> cnf = Lists.newArrayListWithExpectedSize(schema.getMaxFields());
    KeyRange minMaxRange = keySlots.getMinMaxRange();
    if (minMaxRange == null) {
        minMaxRange = KeyRange.EVERYTHING_RANGE;
    }
    boolean hasMinMaxRange = (minMaxRange != KeyRange.EVERYTHING_RANGE);
    int minMaxRangeOffset = 0;
    byte[] minMaxRangePrefix = null;
    boolean hasViewIndex = table.getViewIndexId() != null;
    if (hasMinMaxRange) {
        int minMaxRangeSize = (isSalted ? SaltingUtil.NUM_SALTING_BYTES : 0) + (isMultiTenant ? tenantIdBytes.length + 1 : 0) + (hasViewIndex ? MetaDataUtil.getViewIndexIdDataType().getByteSize() : 0);
        minMaxRangePrefix = new byte[minMaxRangeSize];
    }
    Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.iterator();
    // Add placeholder for salt byte ranges
    if (isSalted) {
        cnf.add(SALT_PLACEHOLDER);
        if (hasMinMaxRange) {
            System.arraycopy(SALT_PLACEHOLDER.get(0).getLowerRange(), 0, minMaxRangePrefix, minMaxRangeOffset, SaltingUtil.NUM_SALTING_BYTES);
            minMaxRangeOffset += SaltingUtil.NUM_SALTING_BYTES;
        }
        // Increment the pkPos, as the salt column is in the row schema
        // Do not increment the iterator, though, as there will never be
        // an expression in the keySlots for the salt column
        pkPos++;
    }
    // that different indexes don't interleave.
    if (hasViewIndex) {
        byte[] viewIndexBytes = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
        KeyRange indexIdKeyRange = KeyRange.getKeyRange(viewIndexBytes);
        cnf.add(singletonList(indexIdKeyRange));
        if (hasMinMaxRange) {
            System.arraycopy(viewIndexBytes, 0, minMaxRangePrefix, minMaxRangeOffset, viewIndexBytes.length);
            minMaxRangeOffset += viewIndexBytes.length;
        }
        pkPos++;
    }
    // Add tenant data isolation for tenant-specific tables
    if (isMultiTenant) {
        KeyRange tenantIdKeyRange = KeyRange.getKeyRange(tenantIdBytes);
        cnf.add(singletonList(tenantIdKeyRange));
        if (hasMinMaxRange) {
            System.arraycopy(tenantIdBytes, 0, minMaxRangePrefix, minMaxRangeOffset, tenantIdBytes.length);
            minMaxRangeOffset += tenantIdBytes.length;
            Field f = schema.getField(pkPos);
            if (!f.getDataType().isFixedWidth()) {
                minMaxRangePrefix[minMaxRangeOffset] = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), tenantIdBytes.length == 0, f);
                minMaxRangeOffset++;
            }
        }
        pkPos++;
    }
    // range with the other range.
    if (hasMinMaxRange) {
        minMaxRange = minMaxRange.prependRange(minMaxRangePrefix, 0, minMaxRangeOffset);
    }
    boolean forcedSkipScan = statement.getHint().hasHint(Hint.SKIP_SCAN);
    boolean forcedRangeScan = statement.getHint().hasHint(Hint.RANGE_SCAN);
    boolean hasUnboundedRange = false;
    boolean hasMultiRanges = false;
    boolean hasRangeKey = false;
    boolean stopExtracting = false;
    boolean useSkipScan = false;
    // Concat byte arrays of literals to form scan start key
    while (iterator.hasNext()) {
        KeyExpressionVisitor.KeySlot slot = iterator.next();
        // If the slot is null this means we have no entry for this pk position.
        if (slot == null || slot.getKeyRanges().isEmpty()) {
            continue;
        }
        if (slot.getPKPosition() != pkPos) {
            if (!forcedSkipScan) {
                stopExtracting = true;
            } else {
                useSkipScan |= !stopExtracting && !forcedRangeScan && forcedSkipScan;
            }
            for (int i = pkPos; i < slot.getPKPosition(); i++) {
                cnf.add(Collections.singletonList(KeyRange.EVERYTHING_RANGE));
            }
        }
        KeyPart keyPart = slot.getKeyPart();
        slotSpan[cnf.size()] = slot.getPKSpan() - 1;
        pkPos = slot.getPKPosition() + slot.getPKSpan();
        // Skip span-1 slots as we skip one at the top of the loop
        for (int i = 1; i < slot.getPKSpan() && iterator.hasNext(); i++) {
            iterator.next();
        }
        List<KeyRange> keyRanges = slot.getKeyRanges();
        cnf.add(keyRanges);
        // TODO: when stats are available, we may want to use a skip scan if the
        // cardinality of this slot is low.
        /*
             *  Stop extracting nodes once we encounter:
             *  1) An unbound range unless we're forcing a skip scan and havn't encountered
             *     a multi-column span. Even if we're trying to force a skip scan, we can't
             *     execute it over a multi-column span.
             *  2) A non range key as we can extract the first one, but further ones need
             *     to be evaluated in a filter.
             */
        stopExtracting |= (hasUnboundedRange && !forcedSkipScan) || (hasRangeKey && forcedRangeScan);
        useSkipScan |= !stopExtracting && !forcedRangeScan && (keyRanges.size() > 1 || hasRangeKey);
        for (int i = 0; (!hasUnboundedRange || !hasRangeKey) && i < keyRanges.size(); i++) {
            KeyRange range = keyRanges.get(i);
            if (range.isUnbound()) {
                hasUnboundedRange = hasRangeKey = true;
            } else if (!range.isSingleKey()) {
                hasRangeKey = true;
            }
        }
        hasMultiRanges |= keyRanges.size() > 1;
        // We cannot extract if we have multiple ranges and are forcing a range scan.
        stopExtracting |= forcedRangeScan && hasMultiRanges;
        // that, so must filter on the remaining conditions (see issue #467).
        if (!stopExtracting) {
            List<Expression> nodesToExtract = keyPart.getExtractNodes();
            extractNodes.addAll(nodesToExtract);
        }
    }
    // If we have fully qualified point keys with multi-column spans (i.e. RVC),
    // we can still use our skip scan. The ScanRanges.create() call will explode
    // out the keys.
    slotSpan = Arrays.copyOf(slotSpan, cnf.size());
    ScanRanges scanRanges = ScanRanges.create(schema, cnf, slotSpan, minMaxRange, nBuckets, useSkipScan, table.getRowTimestampColPos());
    context.setScanRanges(scanRanges);
    if (whereClause == null) {
        return null;
    } else {
        return whereClause.accept(new RemoveExtractedNodesVisitor(extractNodes));
    }
}

Example 19 with KeyRange

use of org.apache.phoenix.query.KeyRange in project phoenix by apache.

the class ScanRanges method create.

public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, KeyRange minMaxRange, Integer nBuckets, boolean useSkipScan, int rowTimestampColIndex) {
    int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
    int nSlots = ranges.size();
    if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
        return EVERYTHING;
    } else if (minMaxRange == KeyRange.EMPTY_RANGE || (nSlots == 1 + offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
        return NOTHING;
    }
    TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
    boolean isPointLookup = isPointLookup(schema, ranges, slotSpan, useSkipScan);
    if (isPointLookup) {
        // TODO: consider keeping original to use for serialization as it would be smaller?
        List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
        List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
        KeyRange unsaltedMinMaxRange = minMaxRange;
        if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
            unsaltedMinMaxRange = KeyRange.getKeyRange(stripPrefix(minMaxRange.getLowerRange(), offset), minMaxRange.lowerUnbound(), stripPrefix(minMaxRange.getUpperRange(), offset), minMaxRange.upperUnbound());
        }
        // We have full keys here, so use field from our varbinary schema
        BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
        for (byte[] key : keys) {
            // Filter now based on unsalted minMaxRange and ignore the point key salt byte
            if (unsaltedMinMaxRange.compareLowerToUpperBound(key, offset, key.length - offset, true, comparator) <= 0 && unsaltedMinMaxRange.compareUpperToLowerBound(key, offset, key.length - offset, true, comparator) >= 0) {
                keyRanges.add(KeyRange.getKeyRange(key));
            }
        }
        ranges = Collections.singletonList(keyRanges);
        useSkipScan = keyRanges.size() > 1;
        // which is not part of the value.
        if (keys.size() > 1 || SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), false, schema.getField(schema.getFieldCount() - 1)) == QueryConstants.DESC_SEPARATOR_BYTE) {
            schema = SchemaUtil.VAR_BINARY_SCHEMA;
            slotSpan = ScanUtil.SINGLE_COLUMN_SLOT_SPAN;
        } else {
            // Keep original schema and don't use skip scan as it's not necessary
            // when there's a single key.
            slotSpan = new int[] { schema.getMaxFields() - 1 };
        }
    }
    List<List<KeyRange>> sortedRanges = Lists.newArrayListWithExpectedSize(ranges.size());
    for (int i = 0; i < ranges.size(); i++) {
        List<KeyRange> sorted = Lists.newArrayList(ranges.get(i));
        Collections.sort(sorted, KeyRange.COMPARATOR);
        sortedRanges.add(ImmutableList.copyOf(sorted));
    }
    // Don't set minMaxRange for point lookup because it causes issues during intersect
    // by going across region boundaries
    KeyRange scanRange = KeyRange.EVERYTHING_RANGE;
    // if (nBuckets == null || (nBuckets != null && (!isPointLookup || !useSkipScanFilter))) {
    if (nBuckets == null || !isPointLookup || !useSkipScan) {
        byte[] minKey = ScanUtil.getMinKey(schema, sortedRanges, slotSpan);
        byte[] maxKey = ScanUtil.getMaxKey(schema, sortedRanges, slotSpan);
        // have anything to filter at the upper end of the range
        if (ScanUtil.crossesPrefixBoundary(maxKey, ScanUtil.getPrefix(minKey, offset), offset)) {
            maxKey = KeyRange.UNBOUND;
        }
        // We won't filter anything at the low end of the range if we just have the salt byte
        if (minKey.length <= offset) {
            minKey = KeyRange.UNBOUND;
        }
        scanRange = KeyRange.getKeyRange(minKey, maxKey);
    }
    if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
        minMaxRange = ScanUtil.convertToInclusiveExclusiveRange(minMaxRange, schema, new ImmutableBytesWritable());
        scanRange = scanRange.intersect(minMaxRange);
    }
    if (scanRange == KeyRange.EMPTY_RANGE) {
        return NOTHING;
    }
    return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, minMaxRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange);
}

Example 20 with KeyRange

use of org.apache.phoenix.query.KeyRange in project phoenix by apache.

the class SkipScanFilter method allTrailingNulls.

private boolean allTrailingNulls(int i) {
    for (; i < slots.size(); i++) {
        List<KeyRange> keyRanges = slots.get(i);
        if (keyRanges.size() != 1) {
            return false;
        }
        KeyRange keyRange = keyRanges.get(0);
        if (!keyRange.isSingleKey()) {
            return false;
        }
        if (keyRange.getLowerRange().length != 0) {
            return false;
        }
    }
    return true;
}