Examples with MonitoredTask org.apache.hadoop.hbase.monitoring.MonitoredTask used on opensource projects

Example 16 with MonitoredTask

use of org.apache.hadoop.hbase.monitoring.MonitoredTask in project hbase by apache.

the class HRegion method replayWALFlushCommitMarker.

@VisibleForTesting
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "NN_NAKED_NOTIFY", justification = "Intentional; post memstore flush")
void replayWALFlushCommitMarker(FlushDescriptor flush) throws IOException {
    MonitoredTask status = TaskMonitor.get().createStatus("Committing flush " + this);
    // the original seqIds.
    synchronized (writestate) {
        try {
            if (flush.getFlushSequenceNumber() < lastReplayedOpenRegionSeqId) {
                LOG.warn(getRegionInfo().getEncodedName() + " : " + "Skipping replaying flush event :" + TextFormat.shortDebugString(flush) + " because its sequence id is smaller than this regions lastReplayedOpenRegionSeqId " + " of " + lastReplayedOpenRegionSeqId);
                return;
            }
            if (writestate.flushing) {
                PrepareFlushResult prepareFlushResult = this.prepareFlushResult;
                if (flush.getFlushSequenceNumber() == prepareFlushResult.flushOpSeqId) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(getRegionInfo().getEncodedName() + " : " + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber() + " and a previous prepared snapshot was found");
                    }
                    // This is the regular case where we received commit flush after prepare flush
                    // corresponding to the same seqId.
                    replayFlushInStores(flush, prepareFlushResult, true);
                    // Set down the memstore size by amount of flush.
                    this.decrMemstoreSize(prepareFlushResult.totalFlushableSize);
                    this.prepareFlushResult = null;
                    writestate.flushing = false;
                } else if (flush.getFlushSequenceNumber() < prepareFlushResult.flushOpSeqId) {
                    // This should not happen normally. However, lets be safe and guard against these cases
                    // we received a flush commit with a smaller seqId than what we have prepared
                    // we will pick the flush file up from this commit (if we have not seen it), but we
                    // will not drop the memstore
                    LOG.warn(getRegionInfo().getEncodedName() + " : " + "Received a flush commit marker with smaller seqId: " + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: " + prepareFlushResult.flushOpSeqId + ". Picking up new file, but not dropping" + "  prepared memstore snapshot");
                    replayFlushInStores(flush, prepareFlushResult, false);
                // snapshot is not dropped, so memstore sizes should not be decremented
                // we still have the prepared snapshot, flushing should still be true
                } else {
                    // This should not happen normally. However, lets be safe and guard against these cases
                    // we received a flush commit with a larger seqId than what we have prepared
                    // we will pick the flush file for this. We will also obtain the updates lock and
                    // look for contents of the memstore to see whether we have edits after this seqId.
                    // If not, we will drop all the memstore edits and the snapshot as well.
                    LOG.warn(getRegionInfo().getEncodedName() + " : " + "Received a flush commit marker with larger seqId: " + flush.getFlushSequenceNumber() + " than what we have prepared with seqId: " + prepareFlushResult.flushOpSeqId + ". Picking up new file and dropping prepared" + " memstore snapshot");
                    replayFlushInStores(flush, prepareFlushResult, true);
                    // Set down the memstore size by amount of flush.
                    this.decrMemstoreSize(prepareFlushResult.totalFlushableSize);
                    // Inspect the memstore contents to see whether the memstore contains only edits
                    // with seqId smaller than the flush seqId. If so, we can discard those edits.
                    dropMemstoreContentsForSeqId(flush.getFlushSequenceNumber(), null);
                    this.prepareFlushResult = null;
                    writestate.flushing = false;
                }
                // If we were waiting for observing a flush or region opening event for not showing
                // partial data after a secondary region crash, we can allow reads now. We can only make
                // sure that we are not showing partial data (for example skipping some previous edits)
                // until we observe a full flush start and flush commit. So if we were not able to find
                // a previous flush we will not enable reads now.
                this.setReadsEnabled(true);
            } else {
                LOG.warn(getRegionInfo().getEncodedName() + " : " + "Received a flush commit marker with seqId:" + flush.getFlushSequenceNumber() + ", but no previous prepared snapshot was found");
                // There is no corresponding prepare snapshot from before.
                // We will pick up the new flushed file
                replayFlushInStores(flush, null, false);
                // Inspect the memstore contents to see whether the memstore contains only edits
                // with seqId smaller than the flush seqId. If so, we can discard those edits.
                dropMemstoreContentsForSeqId(flush.getFlushSequenceNumber(), null);
            }
            status.markComplete("Flush commit successful");
            // Update the last flushed sequence id for region.
            this.maxFlushedSeqId = flush.getFlushSequenceNumber();
            // advance the mvcc read point so that the new flushed file is visible.
            mvcc.advanceTo(flush.getFlushSequenceNumber());
        } catch (FileNotFoundException ex) {
            LOG.warn(getRegionInfo().getEncodedName() + " : " + "At least one of the store files in flush: " + TextFormat.shortDebugString(flush) + " doesn't exist any more. Skip loading the file(s)", ex);
        } finally {
            status.cleanup();
            writestate.notifyAll();
        }
    }
    // e.g. checkResources().
    synchronized (this) {
        // FindBugs NN_NAKED_NOTIFY
        notifyAll();
    }
}

Example 17 with MonitoredTask

use of org.apache.hadoop.hbase.monitoring.MonitoredTask in project hbase by apache.

the class HRegion method initialize.

/**
   * Initialize this region.
   *
   * @param reporter Tickle every so often if initialize is taking a while.
   * @return What the next sequence (edit) id should be.
   * @throws IOException e
   */
private long initialize(final CancelableProgressable reporter) throws IOException {
    //Refuse to open the region if there is no column family in the table
    if (htableDescriptor.getColumnFamilyCount() == 0) {
        throw new DoNotRetryIOException("Table " + htableDescriptor.getNameAsString() + " should have at least one column family.");
    }
    MonitoredTask status = TaskMonitor.get().createStatus("Initializing region " + this);
    long nextSeqId = -1;
    try {
        nextSeqId = initializeRegionInternals(reporter, status);
        return nextSeqId;
    } finally {
        // At least it will be 0 otherwise.
        if (nextSeqId == -1) {
            status.abort("Exception during region " + getRegionInfo().getRegionNameAsString() + " initialization.");
        }
    }
}

Example 18 with MonitoredTask

use of org.apache.hadoop.hbase.monitoring.MonitoredTask in project hbase by apache.

the class TestHRegion method testSkipRecoveredEditsReplaySomeIgnored.

@Test
public void testSkipRecoveredEditsReplaySomeIgnored() throws Exception {
    byte[] family = Bytes.toBytes("family");
    this.region = initHRegion(tableName, method, CONF, family);
    final WALFactory wals = new WALFactory(CONF, null, method);
    try {
        Path regiondir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
        Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
        long maxSeqId = 1050;
        long minSeqId = 1000;
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
            fs.create(recoveredEdits);
            WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
            long time = System.nanoTime();
            WALEdit edit = new WALEdit();
            edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
            writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
            writer.close();
        }
        long recoverSeqId = 1030;
        MonitoredTask status = TaskMonitor.get().createStatus(method);
        Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        for (Store store : region.getStores()) {
            maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
        }
        long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
        assertEquals(maxSeqId, seqId);
        region.getMVCC().advanceTo(seqId);
        Get get = new Get(row);
        Result result = region.get(get);
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            List<Cell> kvs = result.getColumnCells(family, Bytes.toBytes(i));
            if (i < recoverSeqId) {
                assertEquals(0, kvs.size());
            } else {
                assertEquals(1, kvs.size());
                assertArrayEquals(Bytes.toBytes(i), CellUtil.cloneValue(kvs.get(0)));
            }
        }
    } finally {
        HBaseTestingUtility.closeRegionAndWAL(this.region);
        this.region = null;
        wals.close();
    }
}

Example 19 with MonitoredTask

use of org.apache.hadoop.hbase.monitoring.MonitoredTask in project hbase by apache.

the class TestHRegion method testSkipRecoveredEditsReplayTheLastFileIgnored.

@Test
public void testSkipRecoveredEditsReplayTheLastFileIgnored() throws Exception {
    byte[] family = Bytes.toBytes("family");
    this.region = initHRegion(tableName, method, CONF, family);
    final WALFactory wals = new WALFactory(CONF, null, method);
    try {
        Path regiondir = region.getRegionFileSystem().getRegionDir();
        FileSystem fs = region.getRegionFileSystem().getFileSystem();
        byte[] regionName = region.getRegionInfo().getEncodedNameAsBytes();
        byte[][] columns = region.getTableDesc().getFamiliesKeys().toArray(new byte[0][]);
        assertEquals(0, region.getStoreFileList(columns).size());
        Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
        long maxSeqId = 1050;
        long minSeqId = 1000;
        for (long i = minSeqId; i <= maxSeqId; i += 10) {
            Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i));
            fs.create(recoveredEdits);
            WALProvider.Writer writer = wals.createRecoveredEditsWriter(fs, recoveredEdits);
            long time = System.nanoTime();
            WALEdit edit = null;
            if (i == maxSeqId) {
                edit = WALEdit.createCompaction(region.getRegionInfo(), CompactionDescriptor.newBuilder().setTableName(ByteString.copyFrom(tableName.getName())).setFamilyName(ByteString.copyFrom(regionName)).setEncodedRegionName(ByteString.copyFrom(regionName)).setStoreHomeDirBytes(ByteString.copyFrom(Bytes.toBytes(regiondir.toString()))).setRegionName(ByteString.copyFrom(region.getRegionInfo().getRegionName())).build());
            } else {
                edit = new WALEdit();
                edit.add(new KeyValue(row, family, Bytes.toBytes(i), time, KeyValue.Type.Put, Bytes.toBytes(i)));
            }
            writer.append(new WAL.Entry(new WALKey(regionName, tableName, i, time, HConstants.DEFAULT_CLUSTER_ID), edit));
            writer.close();
        }
        long recoverSeqId = 1030;
        Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        MonitoredTask status = TaskMonitor.get().createStatus(method);
        for (Store store : region.getStores()) {
            maxSeqIdInStores.put(store.getColumnFamilyName().getBytes(), recoverSeqId - 1);
        }
        long seqId = region.replayRecoveredEditsIfAny(regiondir, maxSeqIdInStores, null, status);
        assertEquals(maxSeqId, seqId);
        // assert that the files are flushed
        assertEquals(1, region.getStoreFileList(columns).size());
    } finally {
        HBaseTestingUtility.closeRegionAndWAL(this.region);
        this.region = null;
        wals.close();
    }
}

Example 20 with MonitoredTask

use of org.apache.hadoop.hbase.monitoring.MonitoredTask in project hbase by apache.

the class TestRestoreSnapshotHelper method getRestoreHelper.

/**
   * Initialize the restore helper, based on the snapshot and table information provided.
   */
private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
    MonitoredTask status = Mockito.mock(MonitoredTask.class);
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
    return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status);
}