Examples with ValueFilter org.apache.hadoop.hbase.filter.ValueFilter used on opensource projects

Example 1 with ValueFilter

use of org.apache.hadoop.hbase.filter.ValueFilter in project hbase by apache.

the class TestScannersWithFilters method testFilterList.

@Test
public void testFilterList() throws Exception {
    // Test getting a single row, single key using Row, Qualifier, and Value
    // regular expression and substring filters
    // Use must pass all
    List<Filter> filters = new ArrayList<>(3);
    filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
    Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
    Scan s = new Scan();
    s.addFamily(FAMILIES[0]);
    s.setFilter(f);
    KeyValue[] kvs = { new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]) };
    verifyScanFull(s, kvs);
    // Test getting everything with a MUST_PASS_ONE filter including row, qf,
    // val, regular expression and substring filters
    filters.clear();
    filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+")));
    filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
    f = new FilterList(Operator.MUST_PASS_ONE, filters);
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, numRows, colsPerRow);
}

Example 2 with ValueFilter

use of org.apache.hadoop.hbase.filter.ValueFilter in project YCSB by brianfrankcooper.

the class HBaseClient2 method scan.

/**
 * Perform a range scan for a set of records in the database. Each field/value
 * pair from the result will be stored in a HashMap.
 *
 * @param table
 *          The name of the table
 * @param startkey
 *          The record key of the first record to read.
 * @param recordcount
 *          The number of records to read
 * @param fields
 *          The list of fields to read, or null for all of them
 * @param result
 *          A Vector of HashMaps, where each HashMap is a set field/value
 *          pairs for one record
 * @return Zero on success, a non-zero error code on error
 */
@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
    // if this is a "new" table, init HTable object. Else, use existing one
    if (!tableName.equals(table)) {
        currentTable = null;
        try {
            getHTable(table);
            tableName = table;
        } catch (IOException e) {
            System.err.println("Error accessing HBase table: " + e);
            return Status.ERROR;
        }
    }
    Scan s = new Scan(Bytes.toBytes(startkey));
    // HBase has no record limit. Here, assume recordcount is small enough to
    // bring back in one call.
    // We get back recordcount records
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    s.setCaching(recordcount);
    if (this.usePageFilter) {
        filterList.addFilter(new PageFilter(recordcount));
    }
    // add specified fields or else all fields
    if (fields == null) {
        s.addFamily(columnFamilyBytes);
    } else {
        for (String field : fields) {
            s.addColumn(columnFamilyBytes, Bytes.toBytes(field));
        }
    }
    // define value filter if needed
    if (useScanValueFiltering) {
        filterList.addFilter(new ValueFilter(scanFilterOperator, scanFilterValue));
    }
    s.setFilter(filterList);
    // get results
    ResultScanner scanner = null;
    try {
        scanner = currentTable.getScanner(s);
        int numResults = 0;
        for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
            // get row key
            String key = Bytes.toString(rr.getRow());
            if (debug) {
                System.out.println("Got scan result for key: " + key);
            }
            HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>();
            while (rr.advance()) {
                final Cell cell = rr.current();
                rowResult.put(Bytes.toString(CellUtil.cloneQualifier(cell)), new ByteArrayByteIterator(CellUtil.cloneValue(cell)));
            }
            // add rowResult to result vector
            result.add(rowResult);
            numResults++;
            // break is required.
            if (numResults >= recordcount) {
                // if hit recordcount, bail out
                break;
            }
        }
    // done with row
    } catch (IOException e) {
        if (debug) {
            System.out.println("Error in getting/parsing scan result: " + e);
        }
        return Status.ERROR;
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
    return Status.OK;
}

Example 3 with ValueFilter

use of org.apache.hadoop.hbase.filter.ValueFilter in project hbase by apache.

the class TestHRegion method testGetWithFilter.

@Test
public void testGetWithFilter() throws IOException, InterruptedException {
    byte[] row1 = Bytes.toBytes("row1");
    byte[] fam1 = Bytes.toBytes("fam1");
    byte[] col1 = Bytes.toBytes("col1");
    byte[] value1 = Bytes.toBytes("value1");
    byte[] value2 = Bytes.toBytes("value2");
    final int maxVersions = 3;
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf("testFilterAndColumnTracker")).setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam1).setMaxVersions(maxVersions).build()).build();
    ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null, MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);
    RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
    Path logDir = TEST_UTIL.getDataTestDirOnTestFS(method + ".log");
    final WAL wal = HBaseTestingUtil.createWal(TEST_UTIL.getConfiguration(), logDir, info);
    this.region = TEST_UTIL.createLocalHRegion(info, CONF, tableDescriptor, wal);
    // Put 4 version to memstore
    long ts = 0;
    Put put = new Put(row1, ts);
    put.addColumn(fam1, col1, value1);
    region.put(put);
    put = new Put(row1, ts + 1);
    put.addColumn(fam1, col1, Bytes.toBytes("filter1"));
    region.put(put);
    put = new Put(row1, ts + 2);
    put.addColumn(fam1, col1, Bytes.toBytes("filter2"));
    region.put(put);
    put = new Put(row1, ts + 3);
    put.addColumn(fam1, col1, value2);
    region.put(put);
    Get get = new Get(row1);
    get.readAllVersions();
    Result res = region.get(get);
    // Get 3 versions, the oldest version has gone from user view
    assertEquals(maxVersions, res.size());
    get.setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value")));
    res = region.get(get);
    // When use value filter, the oldest version should still gone from user view and it
    // should only return one key vaule
    assertEquals(1, res.size());
    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
    assertEquals(ts + 3, res.rawCells()[0].getTimestamp());
    region.flush(true);
    region.compact(true);
    Thread.sleep(1000);
    res = region.get(get);
    // After flush and compact, the result should be consistent with previous result
    assertEquals(1, res.size());
    assertTrue(CellUtil.matchingValue(new KeyValue(row1, fam1, col1, value2), res.rawCells()[0]));
}

Example 4 with ValueFilter

use of org.apache.hadoop.hbase.filter.ValueFilter in project hbase by apache.

the class TestScannersWithFilters method testValueFilter.

@Test
public void testValueFilter() throws Exception {
    // Match group one rows
    long expectedRows = numRows / 2;
    long expectedKeys = colsPerRow;
    Filter f = new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
    Scan s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match group two rows
    expectedRows = numRows / 2;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testValueTwo")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match all values using regex
    expectedRows = numRows;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.EQUAL, new RegexStringComparator("testValue((One)|(Two))"));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values less than
    // Expect group one rows
    expectedRows = numRows / 2;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.LESS, new BinaryComparator(Bytes.toBytes("testValueTwo")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values less than or equal
    // Expect all rows
    expectedRows = numRows;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("testValueTwo")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values less than or equal
    // Expect group one rows
    expectedRows = numRows / 2;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values not equal
    // Expect half the rows
    expectedRows = numRows / 2;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values greater or equal
    // Expect all rows
    expectedRows = numRows;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.GREATER_OR_EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values greater
    // Expect half rows
    expectedRows = numRows / 2;
    expectedKeys = colsPerRow;
    f = new ValueFilter(CompareOperator.GREATER, new BinaryComparator(Bytes.toBytes("testValueOne")));
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
    // Match values not equal to testValueOne
    // Look across rows and fully validate the keys and ordering
    // Should see all keys in all group two rows
    f = new ValueFilter(CompareOperator.NOT_EQUAL, new BinaryComparator(Bytes.toBytes("testValueOne")));
    s = new Scan();
    s.setFilter(f);
    KeyValue[] kvs = { // testRowTwo-0
    new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), // testRowTwo-2
    new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]), // testRowTwo-3
    new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) };
    verifyScanFull(s, kvs);
}

Example 5 with ValueFilter

use of org.apache.hadoop.hbase.filter.ValueFilter in project hbase by apache.

the class TestFromClientSide5 method testReadWithFilter.

/**
 * Test for HBASE-17125
 */
@Test
public void testReadWithFilter() throws Exception {
    final TableName tableName = name.getTableName();
    try (Table table = TEST_UTIL.createTable(tableName, FAMILY, 3)) {
        byte[] VALUEA = Bytes.toBytes("value-a");
        byte[] VALUEB = Bytes.toBytes("value-b");
        long[] ts = { 1000, 2000, 3000, 4000 };
        Put put = new Put(ROW);
        // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
        for (int t = 0; t <= 3; t++) {
            if (t <= 1) {
                put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA);
            } else {
                put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB);
            }
        }
        table.put(put);
        Scan scan = new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(3);
        ResultScanner scanner = table.getScanner(scan);
        Result result = scanner.next();
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
        Get get = new Get(ROW).setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(3);
        result = table.get(get);
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
        // Test with max versions 1, it should still read ts[1]
        scan = new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(1);
        scanner = table.getScanner(scan);
        result = scanner.next();
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
        // Test with max versions 1, it should still read ts[1]
        get = new Get(ROW).setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(1);
        result = table.get(get);
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
        // Test with max versions 5, it should still read ts[1]
        scan = new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(5);
        scanner = table.getScanner(scan);
        result = scanner.next();
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
        // Test with max versions 5, it should still read ts[1]
        get = new Get(ROW).setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a"))).readVersions(5);
        result = table.get(get);
        // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
        assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0, 0);
    }
}