Examples with ScannerContext org.apache.hadoop.hbase.regionserver.ScannerContext used on opensource projects

Example 6 with ScannerContext

use of org.apache.hadoop.hbase.regionserver.ScannerContext in project hbase by apache.

the class PartitionedMobCompactor method compactDelFilesInBatch.

/**
   * Compacts the del file in a batch.
   * @param request The compaction request.
   * @param delFiles The del files.
   * @return The path of new del file after merging.
   * @throws IOException if IO failure is encountered
   */
private Path compactDelFilesInBatch(PartitionedMobCompactionRequest request, List<StoreFile> delFiles) throws IOException {
    // create a scanner for the del files.
    StoreScanner scanner = createScanner(delFiles, ScanType.COMPACT_RETAIN_DELETES);
    StoreFileWriter writer = null;
    Path filePath = null;
    try {
        writer = MobUtils.createDelFileWriter(conf, fs, column, MobUtils.formatDate(new Date(request.selectionTime)), tempPath, Long.MAX_VALUE, column.getCompactionCompressionType(), HConstants.EMPTY_START_ROW, compactionCacheConfig, cryptoContext);
        filePath = writer.getPath();
        List<Cell> cells = new ArrayList<>();
        boolean hasMore;
        ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
        do {
            hasMore = scanner.next(cells, scannerContext);
            for (Cell cell : cells) {
                writer.append(cell);
            }
            cells.clear();
        } while (hasMore);
    } finally {
        scanner.close();
        if (writer != null) {
            try {
                writer.close();
            } catch (IOException e) {
                LOG.error("Failed to close the writer of the file " + filePath, e);
            }
        }
    }
    // commit the new del file
    Path path = MobUtils.commitFile(conf, fs, filePath, mobFamilyDir, compactionCacheConfig);
    // archive the old del files
    try {
        MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delFiles);
    } catch (IOException e) {
        LOG.error("Failed to archive the old del files " + delFiles, e);
    }
    return path;
}

Example 7 with ScannerContext

use of org.apache.hadoop.hbase.regionserver.ScannerContext in project hbase by apache.

the class AccessController method checkCoveringPermission.

/**
   * Determine if cell ACLs covered by the operation grant access. This is expensive.
   * @return false if cell ACLs failed to grant access, true otherwise
   * @throws IOException
   */
private boolean checkCoveringPermission(User user, OpType request, RegionCoprocessorEnvironment e, byte[] row, Map<byte[], ? extends Collection<?>> familyMap, long opTs, Action... actions) throws IOException {
    if (!cellFeaturesEnabled) {
        return false;
    }
    long cellGrants = 0;
    long latestCellTs = 0;
    Get get = new Get(row);
    // Only in case of Put/Delete op, consider TS within cell (if set for individual cells).
    // When every cell, within a Mutation, can be linked with diff TS we can not rely on only one
    // version. We have to get every cell version and check its TS against the TS asked for in
    // Mutation and skip those Cells which is outside this Mutation TS.In case of Put, we have to
    // consider only one such passing cell. In case of Delete we have to consider all the cell
    // versions under this passing version. When Delete Mutation contains columns which are a
    // version delete just consider only one version for those column cells.
    boolean considerCellTs = (request == OpType.PUT || request == OpType.DELETE);
    if (considerCellTs) {
        get.setMaxVersions();
    } else {
        get.setMaxVersions(1);
    }
    boolean diffCellTsFromOpTs = false;
    for (Map.Entry<byte[], ? extends Collection<?>> entry : familyMap.entrySet()) {
        byte[] col = entry.getKey();
        // maps so we would not need to do this
        if (entry.getValue() instanceof Set) {
            Set<byte[]> set = (Set<byte[]>) entry.getValue();
            if (set == null || set.isEmpty()) {
                get.addFamily(col);
            } else {
                for (byte[] qual : set) {
                    get.addColumn(col, qual);
                }
            }
        } else if (entry.getValue() instanceof List) {
            List<Cell> list = (List<Cell>) entry.getValue();
            if (list == null || list.isEmpty()) {
                get.addFamily(col);
            } else {
                // In case of family delete, a Cell will be added into the list with Qualifier as null.
                for (Cell cell : list) {
                    if (cell.getQualifierLength() == 0 && (cell.getTypeByte() == Type.DeleteFamily.getCode() || cell.getTypeByte() == Type.DeleteFamilyVersion.getCode())) {
                        get.addFamily(col);
                    } else {
                        get.addColumn(col, CellUtil.cloneQualifier(cell));
                    }
                    if (considerCellTs) {
                        long cellTs = cell.getTimestamp();
                        latestCellTs = Math.max(latestCellTs, cellTs);
                        diffCellTsFromOpTs = diffCellTsFromOpTs || (opTs != cellTs);
                    }
                }
            }
        } else if (entry.getValue() == null) {
            get.addFamily(col);
        } else {
            throw new RuntimeException("Unhandled collection type " + entry.getValue().getClass().getName());
        }
    }
    // We want to avoid looking into the future. So, if the cells of the
    // operation specify a timestamp, or the operation itself specifies a
    // timestamp, then we use the maximum ts found. Otherwise, we bound
    // the Get to the current server time. We add 1 to the timerange since
    // the upper bound of a timerange is exclusive yet we need to examine
    // any cells found there inclusively.
    long latestTs = Math.max(opTs, latestCellTs);
    if (latestTs == 0 || latestTs == HConstants.LATEST_TIMESTAMP) {
        latestTs = EnvironmentEdgeManager.currentTime();
    }
    get.setTimeRange(0, latestTs + 1);
    // case with Put. There no need to get all versions but get latest version only.
    if (!diffCellTsFromOpTs && request == OpType.PUT) {
        get.setMaxVersions(1);
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("Scanning for cells with " + get);
    }
    // This Map is identical to familyMap. The key is a BR rather than byte[].
    // It will be easy to do gets over this new Map as we can create get keys over the Cell cf by
    // new SimpleByteRange(cell.familyArray, cell.familyOffset, cell.familyLen)
    Map<ByteRange, List<Cell>> familyMap1 = new HashMap<>();
    for (Entry<byte[], ? extends Collection<?>> entry : familyMap.entrySet()) {
        if (entry.getValue() instanceof List) {
            familyMap1.put(new SimpleMutableByteRange(entry.getKey()), (List<Cell>) entry.getValue());
        }
    }
    RegionScanner scanner = getRegion(e).getScanner(new Scan(get));
    List<Cell> cells = Lists.newArrayList();
    Cell prevCell = null;
    ByteRange curFam = new SimpleMutableByteRange();
    boolean curColAllVersions = (request == OpType.DELETE);
    long curColCheckTs = opTs;
    boolean foundColumn = false;
    try {
        boolean more = false;
        ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(1).build();
        do {
            cells.clear();
            // scan with limit as 1 to hold down memory use on wide rows
            more = scanner.next(cells, scannerContext);
            for (Cell cell : cells) {
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Found cell " + cell);
                }
                boolean colChange = prevCell == null || !CellUtil.matchingColumn(prevCell, cell);
                if (colChange)
                    foundColumn = false;
                prevCell = cell;
                if (!curColAllVersions && foundColumn) {
                    continue;
                }
                if (colChange && considerCellTs) {
                    curFam.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
                    List<Cell> cols = familyMap1.get(curFam);
                    for (Cell col : cols) {
                        // why the below (col.getQualifierLength() == 0) check.
                        if ((col.getQualifierLength() == 0 && request == OpType.DELETE) || CellUtil.matchingQualifier(cell, col)) {
                            byte type = col.getTypeByte();
                            if (considerCellTs) {
                                curColCheckTs = col.getTimestamp();
                            }
                            // For a Delete op we pass allVersions as true. When a Delete Mutation contains
                            // a version delete for a column no need to check all the covering cells within
                            // that column. Check all versions when Type is DeleteColumn or DeleteFamily
                            // One version delete types are Delete/DeleteFamilyVersion
                            curColAllVersions = (KeyValue.Type.DeleteColumn.getCode() == type) || (KeyValue.Type.DeleteFamily.getCode() == type);
                            break;
                        }
                    }
                }
                if (cell.getTimestamp() > curColCheckTs) {
                    // Just ignore this cell. This is not a covering cell.
                    continue;
                }
                foundColumn = true;
                for (Action action : actions) {
                    // Are there permissions for this user for the cell?
                    if (!authManager.authorize(user, getTableName(e), cell, action)) {
                        // We can stop if the cell ACL denies access
                        return false;
                    }
                }
                cellGrants++;
            }
        } while (more);
    } catch (AccessDeniedException ex) {
        throw ex;
    } catch (IOException ex) {
        LOG.error("Exception while getting cells to calculate covering permission", ex);
    } finally {
        scanner.close();
    }
    // after no table or CF grants are found.
    return cellGrants > 0;
}