Examples with ByteArrayComparator org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator used on opensource projects

Example 1 with ByteArrayComparator

use of org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator in project hbase by apache.

the class HBaseFsck method checkRegionBoundaries.

public void checkRegionBoundaries() {
    try {
        ByteArrayComparator comparator = new ByteArrayComparator();
        List<RegionInfo> regions = MetaTableAccessor.getAllRegions(connection, true);
        final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation();
        Path hbaseRoot = CommonFSUtils.getRootDir(getConf());
        for (RegionInfo regionInfo : regions) {
            Path tableDir = CommonFSUtils.getTableDir(hbaseRoot, regionInfo.getTable());
            currentRegionBoundariesInformation.regionName = regionInfo.getRegionName();
            // For each region, get the start and stop key from the META and compare them to the
            // same information from the Stores.
            Path path = new Path(tableDir, regionInfo.getEncodedName());
            FileSystem fs = path.getFileSystem(getConf());
            FileStatus[] files = fs.listStatus(path);
            // For all the column families in this region...
            byte[] storeFirstKey = null;
            byte[] storeLastKey = null;
            for (FileStatus file : files) {
                String fileName = file.getPath().toString();
                fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
                if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) {
                    FileStatus[] storeFiles = fs.listStatus(file.getPath());
                    // For all the stores in this column family.
                    for (FileStatus storeFile : storeFiles) {
                        HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), CacheConfig.DISABLED, true, getConf());
                        if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey()) > 0))) {
                            storeFirstKey = ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey().get()).getKey();
                        }
                        if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, ((KeyValue.KeyOnlyKeyValue) reader.getLastKey().get()).getKey())) < 0)) {
                            storeLastKey = ((KeyValue.KeyOnlyKeyValue) reader.getLastKey().get()).getKey();
                        }
                        reader.close();
                    }
                }
            }
            currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey();
            currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey();
            currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey);
            currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey);
            if (currentRegionBoundariesInformation.metaFirstKey.length == 0)
                currentRegionBoundariesInformation.metaFirstKey = null;
            if (currentRegionBoundariesInformation.metaLastKey.length == 0)
                currentRegionBoundariesInformation.metaLastKey = null;
            // For a region to be correct, we need the META start key to be smaller or equal to the
            // smallest start key from all the stores, and the start key from the next META entry to
            // be bigger than the last key from all the current stores. First region start key is null;
            // Last region end key is null; some regions can be empty and not have any store.
            boolean valid = true;
            // Checking start key.
            if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) {
                valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0;
            }
            // Checking stop key.
            if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) {
                valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0;
            }
            if (!valid) {
                errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable()));
                LOG.warn("Region's boundaries not aligned between stores and META for:");
                LOG.warn(Objects.toString(currentRegionBoundariesInformation));
            }
        }
    } catch (IOException e) {
        LOG.error(e.toString(), e);
    }
}

Example 2 with ByteArrayComparator

use of org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator in project hadoop by apache.

the class FlowScanner method nextInternal.

/**
   * This method loops through the cells in a given row of the
   * {@link FlowRunTable}. It looks at the tags of each cell to figure out how
   * to process the contents. It then calculates the sum or min or max for each
   * column or returns the cell as is.
   *
   * @param cells
   * @param scannerContext
   * @return true if next row is available for the scanner, false otherwise
   * @throws IOException
   */
private boolean nextInternal(List<Cell> cells, ScannerContext scannerContext) throws IOException {
    Cell cell = null;
    startNext();
    // Loop through all the cells in this row
    // For min/max/metrics we do need to scan the entire set of cells to get the
    // right one
    // But with flush/compaction, the number of cells being scanned will go down
    // cells are grouped per column qualifier then sorted by cell timestamp
    // (latest to oldest) per column qualifier
    // So all cells in one qualifier come one after the other before we see the
    // next column qualifier
    ByteArrayComparator comp = new ByteArrayComparator();
    byte[] previousColumnQualifier = Separator.EMPTY_BYTES;
    AggregationOperation currentAggOp = null;
    SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
    Set<String> alreadySeenAggDim = new HashSet<>();
    int addedCnt = 0;
    long currentTimestamp = System.currentTimeMillis();
    ValueConverter converter = null;
    int limit = batchSize;
    while (limit <= 0 || addedCnt < limit) {
        cell = peekAtNextCell(scannerContext);
        if (cell == null) {
            break;
        }
        byte[] currentColumnQualifier = CellUtil.cloneQualifier(cell);
        if (previousColumnQualifier == null) {
            // first time in loop
            previousColumnQualifier = currentColumnQualifier;
        }
        converter = getValueConverter(currentColumnQualifier);
        if (comp.compare(previousColumnQualifier, currentColumnQualifier) != 0) {
            addedCnt += emitCells(cells, currentColumnCells, currentAggOp, converter, currentTimestamp);
            resetState(currentColumnCells, alreadySeenAggDim);
            previousColumnQualifier = currentColumnQualifier;
            currentAggOp = getCurrentAggOp(cell);
            converter = getValueConverter(currentColumnQualifier);
        }
        collectCells(currentColumnCells, currentAggOp, cell, alreadySeenAggDim, converter, scannerContext);
        nextCell(scannerContext);
    }
    if ((!currentColumnCells.isEmpty()) && ((limit <= 0 || addedCnt < limit))) {
        addedCnt += emitCells(cells, currentColumnCells, currentAggOp, converter, currentTimestamp);
        if (LOG.isDebugEnabled()) {
            if (addedCnt > 0) {
                LOG.debug("emitted cells. " + addedCnt + " for " + this.action + " rowKey=" + FlowRunRowKey.parseRowKey(CellUtil.cloneRow(cells.get(0))));
            } else {
                LOG.debug("emitted no cells for " + this.action);
            }
        }
    }
    return hasMore();
}