Examples with Scan org.apache.hadoop.hbase.client.Scan used on opensource projects

Example 86 with Scan

use of org.apache.hadoop.hbase.client.Scan in project hbase by apache.

the class TestFilter method testSingleColumnValueFilter.

@Test
public void testSingleColumnValueFilter() throws IOException {
    // From HBASE-1821
    // Desired action is to combine two SCVF in a FilterList
    // Want to return only rows that match both conditions
    // Need to change one of the group one columns to use group two value
    Put p = new Put(ROWS_ONE[2]);
    p.addColumn(FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]);
    this.region.put(p);
    // Now let's grab rows that have Q_ONE[0](VALUES[0]) and Q_ONE[2](VALUES[1])
    // Since group two rows don't have these qualifiers, they will pass
    // so limiting scan to group one
    List<Filter> filters = new ArrayList<>();
    filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, VALUES[0]));
    filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2], CompareOp.EQUAL, VALUES[1]));
    Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
    Scan s = new Scan(ROWS_ONE[0], ROWS_TWO[0]);
    s.addFamily(FAMILIES[0]);
    s.setFilter(f);
    // Expect only one row, all qualifiers
    KeyValue[] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]), new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]) };
    verifyScanNoEarlyOut(s, 1, 3);
    verifyScanFull(s, kvs);
    // In order to get expected behavior without limiting to group one
    // need to wrap SCVFs in SkipFilters
    filters = new ArrayList<>();
    filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, VALUES[0])));
    filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2], CompareOp.EQUAL, VALUES[1])));
    f = new FilterList(Operator.MUST_PASS_ALL, filters);
    s = new Scan(ROWS_ONE[0], ROWS_TWO[0]);
    s.addFamily(FAMILIES[0]);
    s.setFilter(f);
    // Expect same KVs
    verifyScanNoEarlyOut(s, 1, 3);
    verifyScanFull(s, kvs);
    // More tests from HBASE-1821 for Clint and filterIfMissing flag
    byte[][] ROWS_THREE = { Bytes.toBytes("rowThree-0"), Bytes.toBytes("rowThree-1"), Bytes.toBytes("rowThree-2"), Bytes.toBytes("rowThree-3") };
    // Give row 0 and 2 QUALIFIERS_ONE[0] (VALUE[0] VALUE[1])
    // Give row 1 and 3 QUALIFIERS_ONE[1] (VALUE[0] VALUE[1])
    KeyValue[] srcKVs = new KeyValue[] { new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_THREE[1], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]), new KeyValue(ROWS_THREE[2], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]), new KeyValue(ROWS_THREE[3], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[1]) };
    for (KeyValue kv : srcKVs) {
        Put put = new Put(CellUtil.cloneRow(kv)).add(kv);
        put.setDurability(Durability.SKIP_WAL);
        this.region.put(put);
    }
    // Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = false
    // Expect 3 rows (0, 2, 3)
    SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, VALUES[0]);
    s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4"));
    s.addFamily(FAMILIES[0]);
    s.setFilter(scvf);
    kvs = new KeyValue[] { srcKVs[0], srcKVs[2], srcKVs[3] };
    verifyScanFull(s, kvs);
    // Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = true
    // Expect 1 row (0)
    scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOp.EQUAL, VALUES[0]);
    scvf.setFilterIfMissing(true);
    s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4"));
    s.addFamily(FAMILIES[0]);
    s.setFilter(scvf);
    kvs = new KeyValue[] { srcKVs[0] };
    verifyScanFull(s, kvs);
    // Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true
    // Expect 1 row (3)
    scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[1], CompareOp.EQUAL, VALUES[1]);
    scvf.setFilterIfMissing(true);
    s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4"));
    s.addFamily(FAMILIES[0]);
    s.setFilter(scvf);
    kvs = new KeyValue[] { srcKVs[3] };
    verifyScanFull(s, kvs);
    // Add QUALIFIERS_ONE[1] to ROWS_THREE[0] with VALUES[0]
    KeyValue kvA = new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]);
    this.region.put(new Put(CellUtil.cloneRow(kvA)).add(kvA));
    // Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true
    // Expect 1 row (3)
    scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[1], CompareOp.EQUAL, VALUES[1]);
    scvf.setFilterIfMissing(true);
    s = new Scan(ROWS_THREE[0], Bytes.toBytes("rowThree-4"));
    s.addFamily(FAMILIES[0]);
    s.setFilter(scvf);
    kvs = new KeyValue[] { srcKVs[3] };
    verifyScanFull(s, kvs);
}

Example 87 with Scan

use of org.apache.hadoop.hbase.client.Scan in project hbase by apache.

the class TestFilterListOrOperatorWithBlkCnt method getScanResult.

private List<Cell> getScanResult(byte[] startRow, byte[] stopRow, Table ht) throws IOException {
    Scan scan = new Scan();
    scan.setMaxVersions();
    if (!Bytes.toString(startRow).isEmpty()) {
        scan.setStartRow(startRow);
    }
    if (!Bytes.toString(stopRow).isEmpty()) {
        scan.setStopRow(stopRow);
    }
    ResultScanner scanner = ht.getScanner(scan);
    List<Cell> kvList = new ArrayList<>();
    Result r;
    while ((r = scanner.next()) != null) {
        for (Cell kv : r.listCells()) {
            kvList.add(kv);
        }
    }
    return kvList;
}

Example 88 with Scan

use of org.apache.hadoop.hbase.client.Scan in project hbase by apache.

the class TestFilterListOrOperatorWithBlkCnt method testMultiRowRangeWithFilterListOrOperatorWithBlkCnt.

@Test
public void testMultiRowRangeWithFilterListOrOperatorWithBlkCnt() throws IOException {
    tableName = TableName.valueOf(name.getMethodName());
    Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
    generateRows(numRows, ht, family, qf, value);
    Scan scan = new Scan();
    scan.setMaxVersions();
    long blocksStart = getBlkAccessCount();
    List<RowRange> ranges1 = new ArrayList<>();
    ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(15), false));
    ranges1.add(new RowRange(Bytes.toBytes(9980), true, Bytes.toBytes(9985), false));
    MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1);
    List<RowRange> ranges2 = new ArrayList<>();
    ranges2.add(new RowRange(Bytes.toBytes(15), true, Bytes.toBytes(20), false));
    ranges2.add(new RowRange(Bytes.toBytes(9985), true, Bytes.toBytes(9990), false));
    MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2);
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    filterList.addFilter(filter1);
    filterList.addFilter(filter2);
    scan.setFilter(filterList);
    int resultsSize = getResultsSize(ht, scan);
    LOG.info("found " + resultsSize + " results");
    List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(20), ht);
    List<Cell> results2 = getScanResult(Bytes.toBytes(9980), Bytes.toBytes(9990), ht);
    assertEquals(results1.size() + results2.size(), resultsSize);
    long blocksEnd = getBlkAccessCount();
    long diff = blocksEnd - blocksStart;
    LOG.info("Diff in number of blocks " + diff);
    /*
     * Verify that we don't read all the blocks (8 in total).
     */
    assertEquals(4, diff);
    ht.close();
}

Example 89 with Scan

use of org.apache.hadoop.hbase.client.Scan in project hbase by apache.

the class TestFilterWrapper method testFilterWrapper.

@Test
public void testFilterWrapper() {
    int kv_number = 0;
    int row_number = 0;
    try {
        Scan scan = new Scan();
        List<Filter> fs = new ArrayList<>();
        DependentColumnFilter f1 = new DependentColumnFilter(Bytes.toBytes("f1"), Bytes.toBytes("c5"), true, CompareFilter.CompareOp.EQUAL, new SubstringComparator("c5"));
        PageFilter f2 = new PageFilter(2);
        fs.add(f1);
        fs.add(f2);
        FilterList filter = new FilterList(fs);
        scan.setFilter(filter);
        Table table = connection.getTable(name);
        ResultScanner scanner = table.getScanner(scan);
        // row2 (c1-c4) and row3(c1-c4) are returned
        for (Result result : scanner) {
            row_number++;
            for (Cell kv : result.listCells()) {
                LOG.debug(kv_number + ". kv: " + kv);
                kv_number++;
                assertEquals("Returned row is not correct", new String(CellUtil.cloneRow(kv)), "row" + (row_number + 1));
            }
        }
        scanner.close();
        table.close();
    } catch (Exception e) {
        // no correct result is expected
        assertNull("Exception happens in scan", e);
    }
    LOG.debug("check the fetched kv number");
    assertEquals("We should get 8 results returned.", 8, kv_number);
    assertEquals("We should get 2 rows returned", 2, row_number);
}

Example 90 with Scan

use of org.apache.hadoop.hbase.client.Scan in project hbase by apache.

the class TestColumnRangeFilter method TestColumnRangeFilterClient.

@Test
public void TestColumnRangeFilterClient() throws Exception {
    String family = "Family";
    Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(family), Integer.MAX_VALUE);
    List<String> rows = generateRandomWords(10, 8);
    long maxTimestamp = 2;
    List<String> columns = generateRandomWords(20000, 8);
    List<KeyValue> kvList = new ArrayList<>();
    Map<StringRange, List<KeyValue>> rangeMap = new HashMap<>();
    rangeMap.put(new StringRange(null, true, "b", false), new ArrayList<>());
    rangeMap.put(new StringRange("p", true, "q", false), new ArrayList<>());
    rangeMap.put(new StringRange("r", false, "s", true), new ArrayList<>());
    rangeMap.put(new StringRange("z", false, null, false), new ArrayList<>());
    String valueString = "ValueString";
    for (String row : rows) {
        Put p = new Put(Bytes.toBytes(row));
        p.setDurability(Durability.SKIP_WAL);
        for (String column : columns) {
            for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
                KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp, valueString);
                p.add(kv);
                kvList.add(kv);
                for (StringRange s : rangeMap.keySet()) {
                    if (s.inRange(column)) {
                        rangeMap.get(s).add(kv);
                    }
                }
            }
        }
        ht.put(p);
    }
    TEST_UTIL.flush();
    ColumnRangeFilter filter;
    Scan scan = new Scan();
    scan.setMaxVersions();
    for (StringRange s : rangeMap.keySet()) {
        filter = new ColumnRangeFilter(s.getStart() == null ? null : Bytes.toBytes(s.getStart()), s.isStartInclusive(), s.getEnd() == null ? null : Bytes.toBytes(s.getEnd()), s.isEndInclusive());
        scan.setFilter(filter);
        ResultScanner scanner = ht.getScanner(scan);
        List<Cell> results = new ArrayList<>();
        LOG.info("scan column range: " + s.toString());
        long timeBeforeScan = System.currentTimeMillis();
        Result result;
        while ((result = scanner.next()) != null) {
            for (Cell kv : result.listCells()) {
                results.add(kv);
            }
        }
        long scanTime = System.currentTimeMillis() - timeBeforeScan;
        scanner.close();
        LOG.info("scan time = " + scanTime + "ms");
        LOG.info("found " + results.size() + " results");
        LOG.info("Expecting " + rangeMap.get(s).size() + " results");
        /*
      for (KeyValue kv : results) {
        LOG.info("found row " + Bytes.toString(kv.getRow()) + ", column "
            + Bytes.toString(kv.getQualifier()));
      }
      */
        assertEquals(rangeMap.get(s).size(), results.size());
    }
    ht.close();
}