Search in sources :

Example 6 with SkipScanFilter

use of org.apache.phoenix.filter.SkipScanFilter in project phoenix by apache.

the class ScanRangesIntersectTest method assertIntersect.

private static void assertIntersect(ScanRanges ranges, String lowerRange, String upperRange, String... expectedPoints) {
    List<KeyRange> expectedKeys = points(expectedPoints);
    Collections.sort(expectedKeys, KeyRange.COMPARATOR);
    Scan scan = new Scan();
    scan.setFilter(ranges.getSkipScanFilter());
    byte[] startKey = lowerRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(lowerRange);
    byte[] stopKey = upperRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(upperRange);
    Scan newScan = ranges.intersectScan(scan, startKey, stopKey, 0, true);
    if (expectedPoints.length == 0) {
        assertNull(newScan);
    } else {
        assertNotNull(newScan);
        SkipScanFilter filter = (SkipScanFilter) newScan.getFilter();
        assertEquals(expectedKeys, filter.getSlots().get(0));
    }
}
Also used : KeyRange(org.apache.phoenix.query.KeyRange) Scan(org.apache.hadoop.hbase.client.Scan) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter)

Example 7 with SkipScanFilter

use of org.apache.phoenix.filter.SkipScanFilter in project phoenix by apache.

the class WhereOptimizerTest method testRVCWithCompareOpsForRowKeyColumnValuesSmallerThanSchema.

@Test
public void testRVCWithCompareOpsForRowKeyColumnValuesSmallerThanSchema() throws SQLException {
    String orgId = "0000005";
    String entityId = "011";
    String orgId2 = "000005";
    String entityId2 = "11";
    // CASE 1: >=
    String query = "select * from atable where (organization_id, entity_id) >= (?,?)";
    List<Object> binds = Arrays.<Object>asList(orgId, entityId);
    StatementContext context = compileStatement(query, binds);
    Scan scan = context.getScan();
    Filter filter = scan.getFilter();
    assertNull(filter);
    assertArrayEquals(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId), 15)), scan.getStartRow());
    assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow());
    // CASE 2: >
    query = "select * from atable where (organization_id, entity_id) > (?,?)";
    binds = Arrays.<Object>asList(orgId, entityId);
    context = compileStatement(query, binds);
    scan = context.getScan();
    filter = scan.getFilter();
    assertNull(filter);
    assertArrayEquals(ByteUtil.nextKey(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId), 15))), scan.getStartRow());
    assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow());
    // CASE 3: <=
    query = "select * from atable where (organization_id, entity_id) <= (?,?)";
    binds = Arrays.<Object>asList(orgId, entityId);
    context = compileStatement(query, binds);
    scan = context.getScan();
    filter = scan.getFilter();
    assertNull(filter);
    assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStartRow());
    assertArrayEquals(ByteUtil.nextKey(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId), 15))), scan.getStopRow());
    // CASE 4: <
    query = "select * from atable where (organization_id, entity_id) < (?,?)";
    binds = Arrays.<Object>asList(orgId, entityId);
    context = compileStatement(query, binds);
    scan = context.getScan();
    filter = scan.getFilter();
    assertNull(filter);
    assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStartRow());
    assertArrayEquals(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId), 15)), scan.getStopRow());
    // CASE 5: =
    // For RVC, this will only occur if there's more than one key in the IN
    query = "select * from atable where (organization_id, entity_id) IN ((?,?),(?,?))";
    binds = Arrays.<Object>asList(orgId, entityId, orgId2, entityId2);
    context = compileStatement(query, binds);
    scan = context.getScan();
    filter = scan.getFilter();
    assertTrue(filter instanceof SkipScanFilter);
    ScanRanges scanRanges = context.getScanRanges();
    assertEquals(2, scanRanges.getPointLookupCount());
    Iterator<KeyRange> iterator = scanRanges.getPointLookupKeyIterator();
    KeyRange k1 = iterator.next();
    assertTrue(k1.isSingleKey());
    assertArrayEquals(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId), 15)), k1.getLowerRange());
    KeyRange k2 = iterator.next();
    assertTrue(k2.isSingleKey());
    assertArrayEquals(ByteUtil.concat(StringUtil.padChar(PChar.INSTANCE.toBytes(orgId2), 15), StringUtil.padChar(PChar.INSTANCE.toBytes(entityId2), 15)), k2.getLowerRange());
}
Also used : RowKeyComparisonFilter(org.apache.phoenix.filter.RowKeyComparisonFilter) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) SingleCQKeyValueComparisonFilter(org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter) Filter(org.apache.hadoop.hbase.filter.Filter) SingleKeyValueComparisonFilter(org.apache.phoenix.filter.SingleKeyValueComparisonFilter) TestUtil.rowKeyFilter(org.apache.phoenix.util.TestUtil.rowKeyFilter) KeyRange(org.apache.phoenix.query.KeyRange) Scan(org.apache.hadoop.hbase.client.Scan) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) Test(org.junit.Test) BaseConnectionlessQueryTest(org.apache.phoenix.query.BaseConnectionlessQueryTest)

Example 8 with SkipScanFilter

use of org.apache.phoenix.filter.SkipScanFilter in project phoenix by apache.

the class WhereOptimizerTest method testTrailingIsNullWithOr.

@Test
public void testTrailingIsNullWithOr() throws Exception {
    String baseTableDDL = "CREATE TABLE t(\n " + "  a VARCHAR,\n" + "  b VARCHAR,\n" + "  CONSTRAINT pk PRIMARY KEY (a, b))";
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement().execute(baseTableDDL);
    conn.close();
    String query = "SELECT * FROM t WHERE a = 'a' and (b is null or b = 'b')";
    StatementContext context = compileStatement(query, Collections.<Object>emptyList());
    Scan scan = context.getScan();
    Filter filter = scan.getFilter();
    assertTrue(filter instanceof SkipScanFilter);
    SkipScanFilter skipScan = (SkipScanFilter) filter;
    List<List<KeyRange>> slots = skipScan.getSlots();
    assertEquals(2, slots.size());
    assertEquals(1, slots.get(0).size());
    assertEquals(2, slots.get(1).size());
    assertEquals(KeyRange.getKeyRange(Bytes.toBytes("a")), slots.get(0).get(0));
    assertTrue(KeyRange.IS_NULL_RANGE == slots.get(1).get(0));
    assertEquals(KeyRange.getKeyRange(Bytes.toBytes("b")), slots.get(1).get(1));
    assertArrayEquals(Bytes.toBytes("a"), scan.getStartRow());
    assertArrayEquals(ByteUtil.concat(Bytes.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY), scan.getStopRow());
}
Also used : RowKeyComparisonFilter(org.apache.phoenix.filter.RowKeyComparisonFilter) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) SingleCQKeyValueComparisonFilter(org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter) Filter(org.apache.hadoop.hbase.filter.Filter) SingleKeyValueComparisonFilter(org.apache.phoenix.filter.SingleKeyValueComparisonFilter) TestUtil.rowKeyFilter(org.apache.phoenix.util.TestUtil.rowKeyFilter) Connection(java.sql.Connection) PhoenixConnection(org.apache.phoenix.jdbc.PhoenixConnection) Scan(org.apache.hadoop.hbase.client.Scan) List(java.util.List) FilterList(org.apache.hadoop.hbase.filter.FilterList) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) Test(org.junit.Test) BaseConnectionlessQueryTest(org.apache.phoenix.query.BaseConnectionlessQueryTest)

Example 9 with SkipScanFilter

use of org.apache.phoenix.filter.SkipScanFilter in project phoenix by apache.

the class WhereOptimizerTest method testUsingRVCNonFullyQualifiedInClause.

@Test
public void testUsingRVCNonFullyQualifiedInClause() throws Exception {
    String firstOrgId = "000000000000001";
    String secondOrgId = "000000000000009";
    String firstParentId = "000000000000011";
    String secondParentId = "000000000000021";
    String query = "select * from entity_history where (organization_id, parent_id) IN ((?, ?), (?, ?))";
    List<Object> binds = Arrays.<Object>asList(firstOrgId, firstParentId, secondOrgId, secondParentId);
    StatementContext context = compileStatement(query, binds);
    Scan scan = context.getScan();
    Filter filter = scan.getFilter();
    assertTrue(filter instanceof SkipScanFilter);
    assertArrayEquals(ByteUtil.concat(PVarchar.INSTANCE.toBytes(firstOrgId), PVarchar.INSTANCE.toBytes(firstParentId)), scan.getStartRow());
    assertArrayEquals(ByteUtil.nextKey(ByteUtil.concat(PVarchar.INSTANCE.toBytes(secondOrgId), PVarchar.INSTANCE.toBytes(secondParentId))), scan.getStopRow());
}
Also used : RowKeyComparisonFilter(org.apache.phoenix.filter.RowKeyComparisonFilter) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) SingleCQKeyValueComparisonFilter(org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter) Filter(org.apache.hadoop.hbase.filter.Filter) SingleKeyValueComparisonFilter(org.apache.phoenix.filter.SingleKeyValueComparisonFilter) TestUtil.rowKeyFilter(org.apache.phoenix.util.TestUtil.rowKeyFilter) Scan(org.apache.hadoop.hbase.client.Scan) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) Test(org.junit.Test) BaseConnectionlessQueryTest(org.apache.phoenix.query.BaseConnectionlessQueryTest)

Example 10 with SkipScanFilter

use of org.apache.phoenix.filter.SkipScanFilter in project phoenix by apache.

the class PhoenixTxIndexMutationGenerator method getIndexUpdates.

public Collection<Pair<Mutation, byte[]>> getIndexUpdates(HTableInterface htable, Iterator<Mutation> mutationIterator) throws IOException, SQLException {
    if (!mutationIterator.hasNext()) {
        return Collections.emptyList();
    }
    List<IndexMaintainer> indexMaintainers = indexMetaData.getIndexMaintainers();
    ResultScanner currentScanner = null;
    // Collect up all mutations in batch
    Map<ImmutableBytesPtr, MultiMutation> mutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    // Collect the set of mutable ColumnReferences so that we can first
    // run a scan to get the current state. We'll need this to delete
    // the existing index rows.
    int estimatedSize = indexMaintainers.size() * 10;
    Set<ColumnReference> mutableColumns = Sets.newHashSetWithExpectedSize(estimatedSize);
    for (IndexMaintainer indexMaintainer : indexMaintainers) {
        // For transactional tables, we use an index maintainer
        // to aid in rollback if there's a KeyValue column in the index. The alternative would be
        // to hold on to all uncommitted index row keys (even ones already sent to HBase) on the
        // client side.
        Set<ColumnReference> allColumns = indexMaintainer.getAllColumns();
        mutableColumns.addAll(allColumns);
    }
    Mutation m = mutationIterator.next();
    Map<String, byte[]> updateAttributes = m.getAttributesMap();
    byte[] txRollbackAttribute = updateAttributes.get(PhoenixTransactionContext.TX_ROLLBACK_ATTRIBUTE_KEY);
    boolean isRollback = txRollbackAttribute != null;
    boolean isImmutable = indexMetaData.isImmutableRows();
    Map<ImmutableBytesPtr, MultiMutation> findPriorValueMutations;
    if (isImmutable && !isRollback) {
        findPriorValueMutations = new HashMap<ImmutableBytesPtr, MultiMutation>();
    } else {
        findPriorValueMutations = mutations;
    }
    while (true) {
        // add the mutation to the batch set
        ImmutableBytesPtr row = new ImmutableBytesPtr(m.getRow());
        // if we have no non PK columns, no need to find the prior values
        if (mutations != findPriorValueMutations && indexMetaData.requiresPriorRowState(m)) {
            addMutation(findPriorValueMutations, row, m);
        }
        addMutation(mutations, row, m);
        if (!mutationIterator.hasNext()) {
            break;
        }
        m = mutationIterator.next();
    }
    Collection<Pair<Mutation, byte[]>> indexUpdates = new ArrayList<Pair<Mutation, byte[]>>(mutations.size() * 2 * indexMaintainers.size());
    // this logic will work there too.
    if (!findPriorValueMutations.isEmpty()) {
        List<KeyRange> keys = Lists.newArrayListWithExpectedSize(mutations.size());
        for (ImmutableBytesPtr ptr : findPriorValueMutations.keySet()) {
            keys.add(PVarbinary.INSTANCE.getKeyRange(ptr.copyBytesIfNecessary()));
        }
        Scan scan = new Scan();
        // Project all mutable columns
        for (ColumnReference ref : mutableColumns) {
            scan.addColumn(ref.getFamily(), ref.getQualifier());
        }
        /*
             * Indexes inherit the storage scheme of the data table which means all the indexes have the same
             * storage scheme and empty key value qualifier. Note that this assumption would be broken if we start
             * supporting new indexes over existing data tables to have a different storage scheme than the data
             * table.
             */
        byte[] emptyKeyValueQualifier = indexMaintainers.get(0).getEmptyKeyValueQualifier();
        // Project empty key value column
        scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
        ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
        scanRanges.initializeScan(scan);
        PhoenixTransactionalTable txTable = TransactionFactory.getTransactionProvider().getTransactionalTable(indexMetaData.getTransactionContext(), htable);
        // For rollback, we need to see all versions, including
        // the last committed version as there may be multiple
        // checkpointed versions.
        SkipScanFilter filter = scanRanges.getSkipScanFilter();
        if (isRollback) {
            filter = new SkipScanFilter(filter, true);
            indexMetaData.getTransactionContext().setVisibilityLevel(PhoenixVisibilityLevel.SNAPSHOT_ALL);
        }
        scan.setFilter(filter);
        currentScanner = txTable.getScanner(scan);
    }
    if (isRollback) {
        processRollback(indexMetaData, txRollbackAttribute, currentScanner, mutableColumns, indexUpdates, mutations);
    } else {
        processMutation(indexMetaData, txRollbackAttribute, currentScanner, mutableColumns, indexUpdates, mutations, findPriorValueMutations);
    }
    return indexUpdates;
}
Also used : MultiMutation(org.apache.phoenix.hbase.index.MultiMutation) HashMap(java.util.HashMap) KeyRange(org.apache.phoenix.query.KeyRange) ArrayList(java.util.ArrayList) IndexMaintainer(org.apache.phoenix.index.IndexMaintainer) PhoenixTransactionalTable(org.apache.phoenix.transaction.PhoenixTransactionalTable) SkipScanFilter(org.apache.phoenix.filter.SkipScanFilter) Pair(org.apache.hadoop.hbase.util.Pair) ResultScanner(org.apache.hadoop.hbase.client.ResultScanner) ImmutableBytesPtr(org.apache.phoenix.hbase.index.util.ImmutableBytesPtr) ScanRanges(org.apache.phoenix.compile.ScanRanges) Scan(org.apache.hadoop.hbase.client.Scan) Mutation(org.apache.hadoop.hbase.client.Mutation) MultiMutation(org.apache.phoenix.hbase.index.MultiMutation) ColumnReference(org.apache.phoenix.hbase.index.covered.update.ColumnReference)

Aggregations

SkipScanFilter (org.apache.phoenix.filter.SkipScanFilter)28 Scan (org.apache.hadoop.hbase.client.Scan)24 Filter (org.apache.hadoop.hbase.filter.Filter)19 BaseConnectionlessQueryTest (org.apache.phoenix.query.BaseConnectionlessQueryTest)19 Test (org.junit.Test)19 RowKeyComparisonFilter (org.apache.phoenix.filter.RowKeyComparisonFilter)18 PhoenixConnection (org.apache.phoenix.jdbc.PhoenixConnection)15 FilterList (org.apache.hadoop.hbase.filter.FilterList)12 List (java.util.List)10 SingleCQKeyValueComparisonFilter (org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter)10 PhoenixPreparedStatement (org.apache.phoenix.jdbc.PhoenixPreparedStatement)10 SingleKeyValueComparisonFilter (org.apache.phoenix.filter.SingleKeyValueComparisonFilter)9 TestUtil.rowKeyFilter (org.apache.phoenix.util.TestUtil.rowKeyFilter)9 TestUtil.multiEncodedKVFilter (org.apache.phoenix.util.TestUtil.multiEncodedKVFilter)8 TestUtil.singleKVFilter (org.apache.phoenix.util.TestUtil.singleKVFilter)8 Connection (java.sql.Connection)6 KeyRange (org.apache.phoenix.query.KeyRange)4 ArrayList (java.util.ArrayList)3 ScanRanges (org.apache.phoenix.compile.ScanRanges)3 ImmutableList (com.google.common.collect.ImmutableList)2