Examples with Cell org.apache.hadoop.hbase.Cell used on opensource projects

Example 96 with Cell

use of org.apache.hadoop.hbase.Cell in project hbase by apache.

the class HMobStore method resolve.

/**
   * Reads the cell from the mob file.
   * @param reference The cell found in the HBase, its value is a path to a mob file.
   * @param cacheBlocks Whether the scanner should cache blocks.
   * @param readPt the read point.
   * @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
   *        missing or corrupt.
   * @return The cell found in the mob file.
   * @throws IOException
   */
public Cell resolve(Cell reference, boolean cacheBlocks, long readPt, boolean readEmptyValueOnMobCellMiss) throws IOException {
    Cell result = null;
    if (MobUtils.hasValidMobRefCellValue(reference)) {
        String fileName = MobUtils.getMobFileName(reference);
        Tag tableNameTag = MobUtils.getTableNameTag(reference);
        if (tableNameTag != null) {
            String tableNameString = TagUtil.getValueAsString(tableNameTag);
            List<Path> locations = map.get(tableNameString);
            if (locations == null) {
                IdLock.Entry lockEntry = keyLock.getLockEntry(tableNameString.hashCode());
                try {
                    locations = map.get(tableNameString);
                    if (locations == null) {
                        locations = new ArrayList<>(2);
                        TableName tn = TableName.valueOf(tableNameString);
                        locations.add(MobUtils.getMobFamilyPath(conf, tn, family.getNameAsString()));
                        locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn).getEncodedName(), family.getNameAsString()));
                        map.put(tableNameString, locations);
                    }
                } finally {
                    keyLock.releaseLockEntry(lockEntry);
                }
            }
            result = readCell(locations, fileName, reference, cacheBlocks, readPt, readEmptyValueOnMobCellMiss);
        }
    }
    if (result == null) {
        LOG.warn("The KeyValue result is null, assemble a new KeyValue with the same row,family," + "qualifier,timestamp,type and tags but with an empty value to return.");
        result = new KeyValue(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength(), reference.getFamilyArray(), reference.getFamilyOffset(), reference.getFamilyLength(), reference.getQualifierArray(), reference.getQualifierOffset(), reference.getQualifierLength(), reference.getTimestamp(), Type.codeToType(reference.getTypeByte()), HConstants.EMPTY_BYTE_ARRAY, 0, 0, reference.getTagsArray(), reference.getTagsOffset(), reference.getTagsLength());
    }
    return result;
}

Example 97 with Cell

use of org.apache.hadoop.hbase.Cell in project hbase by apache.

the class HRegion method replayRecoveredEdits.

/*
   * @param edits File of recovered edits.
   * @param maxSeqIdInStores Maximum sequenceid found in each store.  Edits in wal
   * must be larger than this to be replayed for each store.
   * @param reporter
   * @return the sequence id of the last edit added to this region out of the
   * recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
   * @throws IOException
   */
private long replayRecoveredEdits(final Path edits, Map<byte[], Long> maxSeqIdInStores, final CancelableProgressable reporter) throws IOException {
    String msg = "Replaying edits from " + edits;
    LOG.info(msg);
    MonitoredTask status = TaskMonitor.get().createStatus(msg);
    FileSystem fs = this.fs.getFileSystem();
    status.setStatus("Opening recovered edits");
    WAL.Reader reader = null;
    try {
        reader = WALFactory.createReader(fs, edits, conf);
        long currentEditSeqId = -1;
        long currentReplaySeqId = -1;
        long firstSeqIdInLog = -1;
        long skippedEdits = 0;
        long editsCount = 0;
        long intervalEdits = 0;
        WAL.Entry entry;
        HStore store = null;
        boolean reported_once = false;
        ServerNonceManager ng = this.rsServices == null ? null : this.rsServices.getNonceManager();
        try {
            // How many edits seen before we check elapsed time
            int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);
            // How often to send a progress report (default 1/2 master timeout)
            int period = this.conf.getInt("hbase.hstore.report.period", 300000);
            long lastReport = EnvironmentEdgeManager.currentTime();
            if (coprocessorHost != null) {
                coprocessorHost.preReplayWALs(this.getRegionInfo(), edits);
            }
            while ((entry = reader.next()) != null) {
                WALKey key = entry.getKey();
                WALEdit val = entry.getEdit();
                if (ng != null) {
                    // some test, or nonces disabled
                    ng.reportOperationFromWal(key.getNonceGroup(), key.getNonce(), key.getWriteTime());
                }
                if (reporter != null) {
                    intervalEdits += val.size();
                    if (intervalEdits >= interval) {
                        // Number of edits interval reached
                        intervalEdits = 0;
                        long cur = EnvironmentEdgeManager.currentTime();
                        if (lastReport + period <= cur) {
                            status.setStatus("Replaying edits..." + " skipped=" + skippedEdits + " edits=" + editsCount);
                            // Timeout reached
                            if (!reporter.progress()) {
                                msg = "Progressable reporter failed, stopping replay";
                                LOG.warn(msg);
                                status.abort(msg);
                                throw new IOException(msg);
                            }
                            reported_once = true;
                            lastReport = cur;
                        }
                    }
                }
                if (firstSeqIdInLog == -1) {
                    firstSeqIdInLog = key.getLogSeqNum();
                }
                if (currentEditSeqId > key.getLogSeqNum()) {
                    // when this condition is true, it means we have a serious defect because we need to
                    // maintain increasing SeqId for WAL edits per region
                    LOG.error(getRegionInfo().getEncodedName() + " : " + "Found decreasing SeqId. PreId=" + currentEditSeqId + " key=" + key + "; edit=" + val);
                } else {
                    currentEditSeqId = key.getLogSeqNum();
                }
                currentReplaySeqId = (key.getOrigLogSeqNum() > 0) ? key.getOrigLogSeqNum() : currentEditSeqId;
                // instead of a KeyValue.
                if (coprocessorHost != null) {
                    status.setStatus("Running pre-WAL-restore hook in coprocessors");
                    if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {
                        // if bypass this wal entry, ignore it ...
                        continue;
                    }
                }
                boolean checkRowWithinBoundary = false;
                // Check this edit is for this region.
                if (!Bytes.equals(key.getEncodedRegionName(), this.getRegionInfo().getEncodedNameAsBytes())) {
                    checkRowWithinBoundary = true;
                }
                boolean flush = false;
                MemstoreSize memstoreSize = new MemstoreSize();
                for (Cell cell : val.getCells()) {
                    // METACOLUMN info such as HBASE::CACHEFLUSH entries
                    if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
                        // if region names don't match, skipp replaying compaction marker
                        if (!checkRowWithinBoundary) {
                            //this is a special edit, we should handle it
                            CompactionDescriptor compaction = WALEdit.getCompaction(cell);
                            if (compaction != null) {
                                //replay the compaction
                                replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);
                            }
                        }
                        skippedEdits++;
                        continue;
                    }
                    // Figure which store the edit is meant for.
                    if (store == null || !CellUtil.matchingFamily(cell, store.getFamily().getName())) {
                        store = getHStore(cell);
                    }
                    if (store == null) {
                        // This should never happen.  Perhaps schema was changed between
                        // crash and redeploy?
                        LOG.warn("No family for " + cell);
                        skippedEdits++;
                        continue;
                    }
                    if (checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(), cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())) {
                        LOG.warn("Row of " + cell + " is not within region boundary");
                        skippedEdits++;
                        continue;
                    }
                    // Now, figure if we should skip this edit.
                    if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily().getName())) {
                        skippedEdits++;
                        continue;
                    }
                    CellUtil.setSequenceId(cell, currentReplaySeqId);
                    restoreEdit(store, cell, memstoreSize);
                    editsCount++;
                }
                if (this.rsAccounting != null) {
                    rsAccounting.addRegionReplayEditsSize(getRegionInfo().getRegionName(), memstoreSize);
                }
                flush = isFlushSize(this.addAndGetMemstoreSize(memstoreSize));
                if (flush) {
                    internalFlushcache(null, currentEditSeqId, stores.values(), status, false);
                }
                if (coprocessorHost != null) {
                    coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);
                }
            }
            if (coprocessorHost != null) {
                coprocessorHost.postReplayWALs(this.getRegionInfo(), edits);
            }
        } catch (EOFException eof) {
            Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
            msg = "EnLongAddered EOF. Most likely due to Master failure during " + "wal splitting, so we have this data in another edit.  " + "Continuing, but renaming " + edits + " as " + p;
            LOG.warn(msg, eof);
            status.abort(msg);
        } catch (IOException ioe) {
            // then this problem is idempotent and retrying won't help
            if (ioe.getCause() instanceof ParseException) {
                Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
                msg = "File corruption enLongAddered!  " + "Continuing, but renaming " + edits + " as " + p;
                LOG.warn(msg, ioe);
                status.setStatus(msg);
            } else {
                status.abort(StringUtils.stringifyException(ioe));
                // checksum exception on one datanode, etc).  throw & retry
                throw ioe;
            }
        }
        if (reporter != null && !reported_once) {
            reporter.progress();
        }
        msg = "Applied " + editsCount + ", skipped " + skippedEdits + ", firstSequenceIdInLog=" + firstSeqIdInLog + ", maxSequenceIdInLog=" + currentEditSeqId + ", path=" + edits;
        status.markComplete(msg);
        LOG.debug(msg);
        return currentEditSeqId;
    } finally {
        status.cleanup();
        if (reader != null) {
            reader.close();
        }
    }
}

Example 98 with Cell

use of org.apache.hadoop.hbase.Cell in project hbase by apache.

the class HRegion method reckonAppend.

private Cell reckonAppend(final Cell delta, final Cell currentValue, final long now, Append mutation) throws IOException {
    // Forward any tags found on the delta.
    List<Tag> tags = TagUtil.carryForwardTags(delta);
    long ts = now;
    Cell newCell = null;
    byte[] row = mutation.getRow();
    if (currentValue != null) {
        tags = TagUtil.carryForwardTags(tags, currentValue);
        ts = Math.max(now, currentValue.getTimestamp() + 1);
        tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());
        byte[] tagBytes = TagUtil.fromList(tags);
        // Allocate an empty cell and copy in all parts.
        // TODO: This is intimate knowledge of how a KeyValue is made. Undo!!! Prevents our doing
        // other Cell types. Copying on-heap too if an off-heap Cell.
        newCell = new KeyValue(row.length, delta.getFamilyLength(), delta.getQualifierLength(), ts, KeyValue.Type.Put, delta.getValueLength() + currentValue.getValueLength(), tagBytes == null ? 0 : tagBytes.length);
        // Copy in row, family, and qualifier
        System.arraycopy(row, 0, newCell.getRowArray(), newCell.getRowOffset(), row.length);
        System.arraycopy(delta.getFamilyArray(), delta.getFamilyOffset(), newCell.getFamilyArray(), newCell.getFamilyOffset(), delta.getFamilyLength());
        System.arraycopy(delta.getQualifierArray(), delta.getQualifierOffset(), newCell.getQualifierArray(), newCell.getQualifierOffset(), delta.getQualifierLength());
        // Copy in the value
        CellUtil.copyValueTo(currentValue, newCell.getValueArray(), newCell.getValueOffset());
        System.arraycopy(delta.getValueArray(), delta.getValueOffset(), newCell.getValueArray(), newCell.getValueOffset() + currentValue.getValueLength(), delta.getValueLength());
        // Copy in tag data
        if (tagBytes != null) {
            System.arraycopy(tagBytes, 0, newCell.getTagsArray(), newCell.getTagsOffset(), tagBytes.length);
        }
    } else {
        // Append's KeyValue.Type==Put and ts==HConstants.LATEST_TIMESTAMP
        CellUtil.updateLatestStamp(delta, now);
        newCell = delta;
        tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());
        if (tags != null) {
            newCell = CellUtil.createCell(delta, tags);
        }
    }
    return newCell;
}

Example 99 with Cell

use of org.apache.hadoop.hbase.Cell in project hbase by apache.

the class HRegion method get.

/**
   * Do a specific Get on passed <code>columnFamily</code> and column qualifiers.
   * @param mutation Mutation we are doing this Get for.
   * @param store Which column family on row (TODO: Go all Gets in one go)
   * @param coordinates Cells from <code>mutation</code> used as coordinates applied to Get.
   * @return Return list of Cells found.
   */
private List<Cell> get(final Mutation mutation, final Store store, final List<Cell> coordinates, final IsolationLevel isolation, final TimeRange tr) throws IOException {
    // Sort the cells so that they match the order that they appear in the Get results. Otherwise,
    // we won't be able to find the existing values if the cells are not specified in order by the
    // client since cells are in an array list.
    // TODO: I don't get why we are sorting. St.Ack 20150107
    sort(coordinates, store.getComparator());
    Get get = new Get(mutation.getRow());
    if (isolation != null) {
        get.setIsolationLevel(isolation);
    }
    for (Cell cell : coordinates) {
        get.addColumn(store.getFamily().getName(), CellUtil.cloneQualifier(cell));
    }
    // Increments carry time range. If an Increment instance, put it on the Get.
    if (tr != null) {
        get.setTimeRange(tr.getMin(), tr.getMax());
    }
    return get(get, false);
}

Example 100 with Cell

use of org.apache.hadoop.hbase.Cell in project hbase by apache.

the class HRegionFileSystem method splitStoreFile.

/**
   * Write out a split reference. Package local so it doesnt leak out of
   * regionserver.
   * @param hri {@link HRegionInfo} of the destination
   * @param familyName Column Family Name
   * @param f File to split.
   * @param splitRow Split Row
   * @param top True if we are referring to the top half of the hfile.
   * @param splitPolicy
   * @return Path to created reference.
   * @throws IOException
   */
public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy) throws IOException {
    if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
        // If it is outside the range, return directly.
        if (f.getReader() == null) {
            f.createReader();
        }
        try {
            if (top) {
                //check if larger than last key.
                Cell splitKey = CellUtil.createFirstOnRow(splitRow);
                Cell lastKey = f.getLastKey();
                // If lastKey is null means storefile is empty.
                if (lastKey == null) {
                    return null;
                }
                if (f.getComparator().compare(splitKey, lastKey) > 0) {
                    return null;
                }
            } else {
                //check if smaller than first key
                Cell splitKey = CellUtil.createLastOnRow(splitRow);
                Cell firstKey = f.getFirstKey();
                // If firstKey is null means storefile is empty.
                if (firstKey == null) {
                    return null;
                }
                if (f.getComparator().compare(splitKey, firstKey) < 0) {
                    return null;
                }
            }
        } finally {
            f.closeReader(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
        }
    }
    Path splitDir = new Path(getSplitsDir(hri), familyName);
    // A reference to the bottom half of the hsf store file.
    Reference r = top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
    // Add the referred-to regions name as a dot separated suffix.
    // See REF_NAME_REGEX regex above.  The referred-to regions name is
    // up in the path of the passed in <code>f</code> -- parentdir is family,
    // then the directory above is the region name.
    String parentRegionName = regionInfoForFs.getEncodedName();
    // Write reference with same file id only with the other region name as
    // suffix and into the new region location (under same family).
    Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
    return r.write(fs, p);
}