Examples with ReturnCode org.apache.hadoop.hbase.filter.Filter.ReturnCode used on opensource projects

Example 1 with ReturnCode

use of org.apache.hadoop.hbase.filter.Filter.ReturnCode in project hbase by apache.

the class UserScanQueryMatcher method matchColumn.

protected final MatchCode matchColumn(Cell cell, long timestamp, byte typeByte) throws IOException {
    int tsCmp = tr.compare(timestamp);
    if (tsCmp > 0) {
        return MatchCode.SKIP;
    }
    if (tsCmp < 0) {
        return columns.getNextRowOrNextColumn(cell);
    }
    // STEP 1: Check if the column is part of the requested columns
    MatchCode colChecker = columns.checkColumn(cell, typeByte);
    if (colChecker != MatchCode.INCLUDE) {
        if (colChecker == MatchCode.SEEK_NEXT_ROW) {
            stickyNextRow = true;
        }
        return colChecker;
    }
    ReturnCode filterResponse = ReturnCode.SKIP;
    // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
    if (filter != null) {
        // STEP 3: Filter the key value and return if it filters out
        filterResponse = filter.filterKeyValue(cell);
        switch(filterResponse) {
            case SKIP:
                return MatchCode.SKIP;
            case NEXT_COL:
                return columns.getNextRowOrNextColumn(cell);
            case NEXT_ROW:
                stickyNextRow = true;
                return MatchCode.SEEK_NEXT_ROW;
            case SEEK_NEXT_USING_HINT:
                return MatchCode.SEEK_NEXT_USING_HINT;
            default:
                // It means it is either include or include and seek next
                break;
        }
    }
    /*
     * STEP 4: Reaching this step means the column is part of the requested columns and either
     * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
     * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
     * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
     *
     * FilterResponse            ColumnChecker               Desired behavior
     * INCLUDE                   SKIP                        row has already been included, SKIP.
     * INCLUDE                   INCLUDE                     INCLUDE
     * INCLUDE                   INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
     * INCLUDE                   INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
     * INCLUDE_AND_SEEK_NEXT_COL SKIP                        row has already been included, SKIP.
     * INCLUDE_AND_SEEK_NEXT_COL INCLUDE                     INCLUDE_AND_SEEK_NEXT_COL
     * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
     * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
     *
     * In all the above scenarios, we return the column checker return value except for
     * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
     */
    colChecker = columns.checkVersions(cell, timestamp, typeByte, false);
    // Optimize with stickyNextRow
    boolean seekNextRowFromEssential = filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW && filter.isFamilyEssential(cell.getFamilyArray());
    if (colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW || seekNextRowFromEssential) {
        stickyNextRow = true;
    }
    if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) {
        if (colChecker != MatchCode.SKIP) {
            return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
        }
        return MatchCode.SEEK_NEXT_ROW;
    }
    return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL : colChecker;
}

Example 2 with ReturnCode

use of org.apache.hadoop.hbase.filter.Filter.ReturnCode in project phoenix by apache.

the class FilteredKeyValueScanner method seekToNextUnfilteredKeyValue.

private boolean seekToNextUnfilteredKeyValue() throws IOException {
    while (true) {
        Cell peeked = delegate.peek();
        // no more key values, so we are done
        if (peeked == null) {
            return false;
        }
        // filter the peeked value to see if it should be served
        ReturnCode code = filter.filterKeyValue(peeked);
        switch(code) {
            // included, so we are done
            case INCLUDE:
            case INCLUDE_AND_NEXT_COL:
                return true;
            // not included, so we need to go to the next row
            case SKIP:
            case NEXT_COL:
            case NEXT_ROW:
                delegate.next();
                break;
            // use a seek hint to find out where we should go
            case SEEK_NEXT_USING_HINT:
                delegate.seek(KeyValueUtil.ensureKeyValue(filter.getNextCellHint(peeked)));
        }
    }
}

Example 3 with ReturnCode

use of org.apache.hadoop.hbase.filter.Filter.ReturnCode in project hbase by apache.

the class TestFilterList method testFilterKeyValue.

/**
   * Test filterKeyValue logic.
   * @throws Exception
   */
@Test
public void testFilterKeyValue() throws Exception {
    Filter includeFilter = new FilterBase() {

        @Override
        public Filter.ReturnCode filterKeyValue(Cell v) {
            return Filter.ReturnCode.INCLUDE;
        }
    };
    Filter alternateFilter = new FilterBase() {

        boolean returnInclude = true;

        @Override
        public Filter.ReturnCode filterKeyValue(Cell v) {
            Filter.ReturnCode returnCode = returnInclude ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.SKIP;
            returnInclude = !returnInclude;
            return returnCode;
        }
    };
    Filter alternateIncludeFilter = new FilterBase() {

        boolean returnIncludeOnly = false;

        @Override
        public Filter.ReturnCode filterKeyValue(Cell v) {
            Filter.ReturnCode returnCode = returnIncludeOnly ? Filter.ReturnCode.INCLUDE : Filter.ReturnCode.INCLUDE_AND_NEXT_COL;
            returnIncludeOnly = !returnIncludeOnly;
            return returnCode;
        }
    };
    // Check must pass one filter.
    FilterList mpOnefilterList = new FilterList(Operator.MUST_PASS_ONE, Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
    // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL.
    assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpOnefilterList.filterKeyValue(null));
    // INCLUDE, SKIP, INCLUDE.
    assertEquals(Filter.ReturnCode.INCLUDE, mpOnefilterList.filterKeyValue(null));
    // Check must pass all filter.
    FilterList mpAllfilterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(new Filter[] { includeFilter, alternateIncludeFilter, alternateFilter }));
    // INCLUDE, INCLUDE, INCLUDE_AND_NEXT_COL.
    assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL, mpAllfilterList.filterKeyValue(null));
    // INCLUDE, SKIP, INCLUDE.
    assertEquals(Filter.ReturnCode.SKIP, mpAllfilterList.filterKeyValue(null));
}

Example 4 with ReturnCode

use of org.apache.hadoop.hbase.filter.Filter.ReturnCode in project hbase by apache.

the class LegacyScanQueryMatcher method match.

@Override
public MatchCode match(Cell cell) throws IOException {
    if (filter != null && filter.filterAllRemaining()) {
        return MatchCode.DONE_SCAN;
    }
    MatchCode returnCode = preCheck(cell);
    if (returnCode != null) {
        return returnCode;
    }
    /*
     * The delete logic is pretty complicated now.
     * This is corroborated by the following:
     * 1. The store might be instructed to keep deleted rows around.
     * 2. A scan can optionally see past a delete marker now.
     * 3. If deleted rows are kept, we have to find out when we can
     *    remove the delete markers.
     * 4. Family delete markers are always first (regardless of their TS)
     * 5. Delete markers should not be counted as version
     * 6. Delete markers affect puts of the *same* TS
     * 7. Delete marker need to be version counted together with puts
     *    they affect
     */
    long timestamp = cell.getTimestamp();
    byte typeByte = cell.getTypeByte();
    long mvccVersion = cell.getSequenceId();
    if (CellUtil.isDelete(typeByte)) {
        if (keepDeletedCells == KeepDeletedCells.FALSE || (keepDeletedCells == KeepDeletedCells.TTL && timestamp < oldestUnexpiredTS)) {
            // first ignore delete markers if the scanner can do so, and the
            // range does not include the marker
            //
            // during flushes and compactions also ignore delete markers newer
            // than the readpoint of any open scanner, this prevents deleted
            // rows that could still be seen by a scanner from being collected
            boolean includeDeleteMarker = tr.withinOrAfterTimeRange(timestamp);
            if (includeDeleteMarker && mvccVersion <= maxReadPointToTrackVersions) {
                this.deletes.add(cell);
            }
        // Can't early out now, because DelFam come before any other keys
        }
        if (timeToPurgeDeletes > 0 && (EnvironmentEdgeManager.currentTime() - timestamp) <= timeToPurgeDeletes) {
            return MatchCode.INCLUDE;
        } else if (retainDeletesInOutput || mvccVersion > maxReadPointToTrackVersions) {
            // otherwise (i.e. a "raw" scan) we fall through to normal version and timerange checking
            return MatchCode.INCLUDE;
        } else if (keepDeletedCells == KeepDeletedCells.TRUE || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= oldestUnexpiredTS)) {
            if (timestamp < earliestPutTs) {
                // this delete in the store files.
                return columns.getNextRowOrNextColumn(cell);
            }
        // else: fall through and do version counting on the
        // delete markers
        } else {
            return MatchCode.SKIP;
        }
    // note the following next else if...
    // delete marker are not subject to other delete markers
    } else if (!this.deletes.isEmpty()) {
        DeleteResult deleteResult = deletes.isDeleted(cell);
        switch(deleteResult) {
            case FAMILY_DELETED:
            case COLUMN_DELETED:
                return columns.getNextRowOrNextColumn(cell);
            case VERSION_DELETED:
            case FAMILY_VERSION_DELETED:
                return MatchCode.SKIP;
            case NOT_DELETED:
                break;
            default:
                throw new RuntimeException("UNEXPECTED");
        }
    }
    int timestampComparison = tr.compare(timestamp);
    if (timestampComparison >= 1) {
        return MatchCode.SKIP;
    } else if (timestampComparison <= -1) {
        return columns.getNextRowOrNextColumn(cell);
    }
    // STEP 1: Check if the column is part of the requested columns
    MatchCode colChecker = columns.checkColumn(cell, typeByte);
    if (colChecker == MatchCode.INCLUDE) {
        ReturnCode filterResponse = ReturnCode.SKIP;
        // STEP 2: Yes, the column is part of the requested columns. Check if filter is present
        if (filter != null) {
            // STEP 3: Filter the key value and return if it filters out
            filterResponse = filter.filterKeyValue(cell);
            switch(filterResponse) {
                case SKIP:
                    return MatchCode.SKIP;
                case NEXT_COL:
                    return columns.getNextRowOrNextColumn(cell);
                case NEXT_ROW:
                    stickyNextRow = true;
                    return MatchCode.SEEK_NEXT_ROW;
                case SEEK_NEXT_USING_HINT:
                    return MatchCode.SEEK_NEXT_USING_HINT;
                default:
                    //It means it is either include or include and seek next
                    break;
            }
        }
        /*
       * STEP 4: Reaching this step means the column is part of the requested columns and either
       * the filter is null or the filter has returned INCLUDE or INCLUDE_AND_NEXT_COL response.
       * Now check the number of versions needed. This method call returns SKIP, INCLUDE,
       * INCLUDE_AND_SEEK_NEXT_ROW, INCLUDE_AND_SEEK_NEXT_COL.
       *
       * FilterResponse            ColumnChecker               Desired behavior
       * INCLUDE                   SKIP                        row has already been included, SKIP.
       * INCLUDE                   INCLUDE                     INCLUDE
       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
       * INCLUDE                   INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
       * INCLUDE_AND_SEEK_NEXT_COL SKIP                        row has already been included, SKIP.
       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE                     INCLUDE_AND_SEEK_NEXT_COL
       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_COL   INCLUDE_AND_SEEK_NEXT_COL
       * INCLUDE_AND_SEEK_NEXT_COL INCLUDE_AND_SEEK_NEXT_ROW   INCLUDE_AND_SEEK_NEXT_ROW
       *
       * In all the above scenarios, we return the column checker return value except for
       * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE)
       */
        colChecker = columns.checkVersions(cell, timestamp, typeByte, mvccVersion > maxReadPointToTrackVersions);
        //Optimize with stickyNextRow
        boolean seekNextRowFromEssential = filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW && filter.isFamilyEssential(cell.getFamilyArray());
        if (colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW || seekNextRowFromEssential) {
            stickyNextRow = true;
        }
        if (filterResponse == ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW) {
            if (colChecker != MatchCode.SKIP) {
                return MatchCode.INCLUDE_AND_SEEK_NEXT_ROW;
            }
            return MatchCode.SEEK_NEXT_ROW;
        }
        return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && colChecker == MatchCode.INCLUDE) ? MatchCode.INCLUDE_AND_SEEK_NEXT_COL : colChecker;
    }
    stickyNextRow = (colChecker == MatchCode.SEEK_NEXT_ROW) ? true : stickyNextRow;
    return colChecker;
}

Example 5 with ReturnCode

use of org.apache.hadoop.hbase.filter.Filter.ReturnCode in project honeycomb by altamiracorp.

the class MockHTable method getScanner.

@Override
public ResultScanner getScanner(Scan scan) throws IOException {
    final List<Result> ret = new ArrayList<Result>();
    byte[] st = scan.getStartRow();
    byte[] sp = scan.getStopRow();
    Filter filter = scan.getFilter();
    for (byte[] row : data.keySet()) {
        // happen w/o this control.
        if (st != null && st.length > 0 && Bytes.BYTES_COMPARATOR.compare(st, row) != 0) {
            // if row is before startRow do not emit, pass to next row
            if (st != null && st.length > 0 && Bytes.BYTES_COMPARATOR.compare(st, row) > 0)
                continue;
            // if row is equal to stopRow or after it do not emit, stop iteration
            if (sp != null && sp.length > 0 && Bytes.BYTES_COMPARATOR.compare(sp, row) <= 0)
                break;
        }
        List<KeyValue> kvs = null;
        if (!scan.hasFamilies()) {
            kvs = toKeyValue(row, data.get(row), scan.getTimeRange().getMin(), scan.getTimeRange().getMax(), scan.getMaxVersions());
        } else {
            kvs = new ArrayList<KeyValue>();
            for (byte[] family : scan.getFamilyMap().keySet()) {
                if (data.get(row).get(family) == null)
                    continue;
                NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(family);
                if (qualifiers == null || qualifiers.isEmpty())
                    qualifiers = data.get(row).get(family).navigableKeySet();
                for (byte[] qualifier : qualifiers) {
                    if (data.get(row).get(family).get(qualifier) == null)
                        continue;
                    for (Long timestamp : data.get(row).get(family).get(qualifier).descendingKeySet()) {
                        if (timestamp < scan.getTimeRange().getMin())
                            continue;
                        if (timestamp > scan.getTimeRange().getMax())
                            continue;
                        byte[] value = data.get(row).get(family).get(qualifier).get(timestamp);
                        kvs.add(new KeyValue(row, family, qualifier, timestamp, value));
                        if (kvs.size() == scan.getMaxVersions()) {
                            break;
                        }
                    }
                }
            }
        }
        if (filter != null) {
            filter.reset();
            List<KeyValue> nkvs = new ArrayList<KeyValue>(kvs.size());
            for (KeyValue kv : kvs) {
                if (filter.filterAllRemaining()) {
                    break;
                }
                if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) {
                    continue;
                }
                ReturnCode filterResult = filter.filterKeyValue(kv);
                if (filterResult == ReturnCode.INCLUDE) {
                    nkvs.add(kv);
                } else if (filterResult == ReturnCode.NEXT_ROW) {
                    break;
                }
            // ignoring next key hint which is a optimization to reduce file system IO
            }
            if (filter.hasFilterRow()) {
                filter.filterRow(nkvs);
            }
            kvs = nkvs;
        }
        if (!kvs.isEmpty()) {
            ret.add(new Result(kvs));
        }
    }
    return new ResultScanner() {

        private final Iterator<Result> iterator = ret.iterator();

        @Override
        public Iterator<Result> iterator() {
            return iterator;
        }

        @Override
        public Result[] next(int nbRows) throws IOException {
            ArrayList<Result> resultSets = new ArrayList<Result>(nbRows);
            for (int i = 0; i < nbRows; i++) {
                Result next = next();
                if (next != null) {
                    resultSets.add(next);
                } else {
                    break;
                }
            }
            return resultSets.toArray(new Result[resultSets.size()]);
        }

        @Override
        public Result next() throws IOException {
            try {
                return iterator().next();
            } catch (NoSuchElementException e) {
                return null;
            }
        }

        @Override
        public void close() {
        }
    };
}