use of org.apache.hadoop.hbase.Cell in project hbase by apache.
the class HMobStore method resolve.
/**
* Reads the cell from the mob file.
* @param reference The cell found in the HBase, its value is a path to a mob file.
* @param cacheBlocks Whether the scanner should cache blocks.
* @param readPt the read point.
* @param readEmptyValueOnMobCellMiss Whether return null value when the mob file is
* missing or corrupt.
* @return The cell found in the mob file.
* @throws IOException
*/
public Cell resolve(Cell reference, boolean cacheBlocks, long readPt, boolean readEmptyValueOnMobCellMiss) throws IOException {
Cell result = null;
if (MobUtils.hasValidMobRefCellValue(reference)) {
String fileName = MobUtils.getMobFileName(reference);
Tag tableNameTag = MobUtils.getTableNameTag(reference);
if (tableNameTag != null) {
String tableNameString = TagUtil.getValueAsString(tableNameTag);
List<Path> locations = map.get(tableNameString);
if (locations == null) {
IdLock.Entry lockEntry = keyLock.getLockEntry(tableNameString.hashCode());
try {
locations = map.get(tableNameString);
if (locations == null) {
locations = new ArrayList<>(2);
TableName tn = TableName.valueOf(tableNameString);
locations.add(MobUtils.getMobFamilyPath(conf, tn, family.getNameAsString()));
locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn).getEncodedName(), family.getNameAsString()));
map.put(tableNameString, locations);
}
} finally {
keyLock.releaseLockEntry(lockEntry);
}
}
result = readCell(locations, fileName, reference, cacheBlocks, readPt, readEmptyValueOnMobCellMiss);
}
}
if (result == null) {
LOG.warn("The KeyValue result is null, assemble a new KeyValue with the same row,family," + "qualifier,timestamp,type and tags but with an empty value to return.");
result = new KeyValue(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength(), reference.getFamilyArray(), reference.getFamilyOffset(), reference.getFamilyLength(), reference.getQualifierArray(), reference.getQualifierOffset(), reference.getQualifierLength(), reference.getTimestamp(), Type.codeToType(reference.getTypeByte()), HConstants.EMPTY_BYTE_ARRAY, 0, 0, reference.getTagsArray(), reference.getTagsOffset(), reference.getTagsLength());
}
return result;
}
use of org.apache.hadoop.hbase.Cell in project hbase by apache.
the class HRegion method replayRecoveredEdits.
/*
* @param edits File of recovered edits.
* @param maxSeqIdInStores Maximum sequenceid found in each store. Edits in wal
* must be larger than this to be replayed for each store.
* @param reporter
* @return the sequence id of the last edit added to this region out of the
* recovered edits log or <code>minSeqId</code> if nothing added from editlogs.
* @throws IOException
*/
private long replayRecoveredEdits(final Path edits, Map<byte[], Long> maxSeqIdInStores, final CancelableProgressable reporter) throws IOException {
String msg = "Replaying edits from " + edits;
LOG.info(msg);
MonitoredTask status = TaskMonitor.get().createStatus(msg);
FileSystem fs = this.fs.getFileSystem();
status.setStatus("Opening recovered edits");
WAL.Reader reader = null;
try {
reader = WALFactory.createReader(fs, edits, conf);
long currentEditSeqId = -1;
long currentReplaySeqId = -1;
long firstSeqIdInLog = -1;
long skippedEdits = 0;
long editsCount = 0;
long intervalEdits = 0;
WAL.Entry entry;
HStore store = null;
boolean reported_once = false;
ServerNonceManager ng = this.rsServices == null ? null : this.rsServices.getNonceManager();
try {
// How many edits seen before we check elapsed time
int interval = this.conf.getInt("hbase.hstore.report.interval.edits", 2000);
// How often to send a progress report (default 1/2 master timeout)
int period = this.conf.getInt("hbase.hstore.report.period", 300000);
long lastReport = EnvironmentEdgeManager.currentTime();
if (coprocessorHost != null) {
coprocessorHost.preReplayWALs(this.getRegionInfo(), edits);
}
while ((entry = reader.next()) != null) {
WALKey key = entry.getKey();
WALEdit val = entry.getEdit();
if (ng != null) {
// some test, or nonces disabled
ng.reportOperationFromWal(key.getNonceGroup(), key.getNonce(), key.getWriteTime());
}
if (reporter != null) {
intervalEdits += val.size();
if (intervalEdits >= interval) {
// Number of edits interval reached
intervalEdits = 0;
long cur = EnvironmentEdgeManager.currentTime();
if (lastReport + period <= cur) {
status.setStatus("Replaying edits..." + " skipped=" + skippedEdits + " edits=" + editsCount);
// Timeout reached
if (!reporter.progress()) {
msg = "Progressable reporter failed, stopping replay";
LOG.warn(msg);
status.abort(msg);
throw new IOException(msg);
}
reported_once = true;
lastReport = cur;
}
}
}
if (firstSeqIdInLog == -1) {
firstSeqIdInLog = key.getLogSeqNum();
}
if (currentEditSeqId > key.getLogSeqNum()) {
// when this condition is true, it means we have a serious defect because we need to
// maintain increasing SeqId for WAL edits per region
LOG.error(getRegionInfo().getEncodedName() + " : " + "Found decreasing SeqId. PreId=" + currentEditSeqId + " key=" + key + "; edit=" + val);
} else {
currentEditSeqId = key.getLogSeqNum();
}
currentReplaySeqId = (key.getOrigLogSeqNum() > 0) ? key.getOrigLogSeqNum() : currentEditSeqId;
// instead of a KeyValue.
if (coprocessorHost != null) {
status.setStatus("Running pre-WAL-restore hook in coprocessors");
if (coprocessorHost.preWALRestore(this.getRegionInfo(), key, val)) {
// if bypass this wal entry, ignore it ...
continue;
}
}
boolean checkRowWithinBoundary = false;
// Check this edit is for this region.
if (!Bytes.equals(key.getEncodedRegionName(), this.getRegionInfo().getEncodedNameAsBytes())) {
checkRowWithinBoundary = true;
}
boolean flush = false;
MemstoreSize memstoreSize = new MemstoreSize();
for (Cell cell : val.getCells()) {
// METACOLUMN info such as HBASE::CACHEFLUSH entries
if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
// if region names don't match, skipp replaying compaction marker
if (!checkRowWithinBoundary) {
//this is a special edit, we should handle it
CompactionDescriptor compaction = WALEdit.getCompaction(cell);
if (compaction != null) {
//replay the compaction
replayWALCompactionMarker(compaction, false, true, Long.MAX_VALUE);
}
}
skippedEdits++;
continue;
}
// Figure which store the edit is meant for.
if (store == null || !CellUtil.matchingFamily(cell, store.getFamily().getName())) {
store = getHStore(cell);
}
if (store == null) {
// This should never happen. Perhaps schema was changed between
// crash and redeploy?
LOG.warn("No family for " + cell);
skippedEdits++;
continue;
}
if (checkRowWithinBoundary && !rowIsInRange(this.getRegionInfo(), cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())) {
LOG.warn("Row of " + cell + " is not within region boundary");
skippedEdits++;
continue;
}
// Now, figure if we should skip this edit.
if (key.getLogSeqNum() <= maxSeqIdInStores.get(store.getFamily().getName())) {
skippedEdits++;
continue;
}
CellUtil.setSequenceId(cell, currentReplaySeqId);
restoreEdit(store, cell, memstoreSize);
editsCount++;
}
if (this.rsAccounting != null) {
rsAccounting.addRegionReplayEditsSize(getRegionInfo().getRegionName(), memstoreSize);
}
flush = isFlushSize(this.addAndGetMemstoreSize(memstoreSize));
if (flush) {
internalFlushcache(null, currentEditSeqId, stores.values(), status, false);
}
if (coprocessorHost != null) {
coprocessorHost.postWALRestore(this.getRegionInfo(), key, val);
}
}
if (coprocessorHost != null) {
coprocessorHost.postReplayWALs(this.getRegionInfo(), edits);
}
} catch (EOFException eof) {
Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
msg = "EnLongAddered EOF. Most likely due to Master failure during " + "wal splitting, so we have this data in another edit. " + "Continuing, but renaming " + edits + " as " + p;
LOG.warn(msg, eof);
status.abort(msg);
} catch (IOException ioe) {
// then this problem is idempotent and retrying won't help
if (ioe.getCause() instanceof ParseException) {
Path p = WALSplitter.moveAsideBadEditsFile(fs, edits);
msg = "File corruption enLongAddered! " + "Continuing, but renaming " + edits + " as " + p;
LOG.warn(msg, ioe);
status.setStatus(msg);
} else {
status.abort(StringUtils.stringifyException(ioe));
// checksum exception on one datanode, etc). throw & retry
throw ioe;
}
}
if (reporter != null && !reported_once) {
reporter.progress();
}
msg = "Applied " + editsCount + ", skipped " + skippedEdits + ", firstSequenceIdInLog=" + firstSeqIdInLog + ", maxSequenceIdInLog=" + currentEditSeqId + ", path=" + edits;
status.markComplete(msg);
LOG.debug(msg);
return currentEditSeqId;
} finally {
status.cleanup();
if (reader != null) {
reader.close();
}
}
}
use of org.apache.hadoop.hbase.Cell in project hbase by apache.
the class HRegion method reckonAppend.
private Cell reckonAppend(final Cell delta, final Cell currentValue, final long now, Append mutation) throws IOException {
// Forward any tags found on the delta.
List<Tag> tags = TagUtil.carryForwardTags(delta);
long ts = now;
Cell newCell = null;
byte[] row = mutation.getRow();
if (currentValue != null) {
tags = TagUtil.carryForwardTags(tags, currentValue);
ts = Math.max(now, currentValue.getTimestamp() + 1);
tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());
byte[] tagBytes = TagUtil.fromList(tags);
// Allocate an empty cell and copy in all parts.
// TODO: This is intimate knowledge of how a KeyValue is made. Undo!!! Prevents our doing
// other Cell types. Copying on-heap too if an off-heap Cell.
newCell = new KeyValue(row.length, delta.getFamilyLength(), delta.getQualifierLength(), ts, KeyValue.Type.Put, delta.getValueLength() + currentValue.getValueLength(), tagBytes == null ? 0 : tagBytes.length);
// Copy in row, family, and qualifier
System.arraycopy(row, 0, newCell.getRowArray(), newCell.getRowOffset(), row.length);
System.arraycopy(delta.getFamilyArray(), delta.getFamilyOffset(), newCell.getFamilyArray(), newCell.getFamilyOffset(), delta.getFamilyLength());
System.arraycopy(delta.getQualifierArray(), delta.getQualifierOffset(), newCell.getQualifierArray(), newCell.getQualifierOffset(), delta.getQualifierLength());
// Copy in the value
CellUtil.copyValueTo(currentValue, newCell.getValueArray(), newCell.getValueOffset());
System.arraycopy(delta.getValueArray(), delta.getValueOffset(), newCell.getValueArray(), newCell.getValueOffset() + currentValue.getValueLength(), delta.getValueLength());
// Copy in tag data
if (tagBytes != null) {
System.arraycopy(tagBytes, 0, newCell.getTagsArray(), newCell.getTagsOffset(), tagBytes.length);
}
} else {
// Append's KeyValue.Type==Put and ts==HConstants.LATEST_TIMESTAMP
CellUtil.updateLatestStamp(delta, now);
newCell = delta;
tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL());
if (tags != null) {
newCell = CellUtil.createCell(delta, tags);
}
}
return newCell;
}
use of org.apache.hadoop.hbase.Cell in project hbase by apache.
the class HRegion method get.
/**
* Do a specific Get on passed <code>columnFamily</code> and column qualifiers.
* @param mutation Mutation we are doing this Get for.
* @param store Which column family on row (TODO: Go all Gets in one go)
* @param coordinates Cells from <code>mutation</code> used as coordinates applied to Get.
* @return Return list of Cells found.
*/
private List<Cell> get(final Mutation mutation, final Store store, final List<Cell> coordinates, final IsolationLevel isolation, final TimeRange tr) throws IOException {
// Sort the cells so that they match the order that they appear in the Get results. Otherwise,
// we won't be able to find the existing values if the cells are not specified in order by the
// client since cells are in an array list.
// TODO: I don't get why we are sorting. St.Ack 20150107
sort(coordinates, store.getComparator());
Get get = new Get(mutation.getRow());
if (isolation != null) {
get.setIsolationLevel(isolation);
}
for (Cell cell : coordinates) {
get.addColumn(store.getFamily().getName(), CellUtil.cloneQualifier(cell));
}
// Increments carry time range. If an Increment instance, put it on the Get.
if (tr != null) {
get.setTimeRange(tr.getMin(), tr.getMax());
}
return get(get, false);
}
use of org.apache.hadoop.hbase.Cell in project hbase by apache.
the class HRegionFileSystem method splitStoreFile.
/**
* Write out a split reference. Package local so it doesnt leak out of
* regionserver.
* @param hri {@link HRegionInfo} of the destination
* @param familyName Column Family Name
* @param f File to split.
* @param splitRow Split Row
* @param top True if we are referring to the top half of the hfile.
* @param splitPolicy
* @return Path to created reference.
* @throws IOException
*/
public Path splitStoreFile(final HRegionInfo hri, final String familyName, final StoreFile f, final byte[] splitRow, final boolean top, RegionSplitPolicy splitPolicy) throws IOException {
if (splitPolicy == null || !splitPolicy.skipStoreFileRangeCheck(familyName)) {
// If it is outside the range, return directly.
if (f.getReader() == null) {
f.createReader();
}
try {
if (top) {
//check if larger than last key.
Cell splitKey = CellUtil.createFirstOnRow(splitRow);
Cell lastKey = f.getLastKey();
// If lastKey is null means storefile is empty.
if (lastKey == null) {
return null;
}
if (f.getComparator().compare(splitKey, lastKey) > 0) {
return null;
}
} else {
//check if smaller than first key
Cell splitKey = CellUtil.createLastOnRow(splitRow);
Cell firstKey = f.getFirstKey();
// If firstKey is null means storefile is empty.
if (firstKey == null) {
return null;
}
if (f.getComparator().compare(splitKey, firstKey) < 0) {
return null;
}
}
} finally {
f.closeReader(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true);
}
}
Path splitDir = new Path(getSplitsDir(hri), familyName);
// A reference to the bottom half of the hsf store file.
Reference r = top ? Reference.createTopReference(splitRow) : Reference.createBottomReference(splitRow);
// Add the referred-to regions name as a dot separated suffix.
// See REF_NAME_REGEX regex above. The referred-to regions name is
// up in the path of the passed in <code>f</code> -- parentdir is family,
// then the directory above is the region name.
String parentRegionName = regionInfoForFs.getEncodedName();
// Write reference with same file id only with the other region name as
// suffix and into the new region location (under same family).
Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
return r.write(fs, p);
}
Aggregations