Examples with SnapshotDescription org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription used on opensource projects

Example 11 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class SnapshotReferenceUtil method visitReferencedFiles.

/**
 * Iterate over the snapshot store files
 *
 * @param conf The current {@link Configuration} instance.
 * @param fs {@link FileSystem}
 * @param snapshotDir {@link Path} to the Snapshot directory
 * @param visitor callback object to get the referenced files
 * @throws IOException if an error occurred while scanning the directory
 */
public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotVisitor visitor) throws IOException {
    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    visitReferencedFiles(conf, fs, snapshotDir, desc, visitor);
}

Example 12 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class RestoreSnapshotHelper method copySnapshotForScanner.

/**
 * Copy the snapshot files for a snapshot scanner, discards meta changes.
 * @param conf
 * @param fs
 * @param rootDir
 * @param restoreDir
 * @param snapshotName
 * @throws IOException
 */
public static RestoreMetaChanges copySnapshotForScanner(Configuration conf, FileSystem fs, Path rootDir, Path restoreDir, String snapshotName) throws IOException {
    // ensure that restore dir is not under root dir
    if (!restoreDir.getFileSystem(conf).getUri().equals(rootDir.getFileSystem(conf).getUri())) {
        throw new IllegalArgumentException("Filesystems for restore directory and HBase root " + "directory should be the same");
    }
    if (restoreDir.toUri().getPath().startsWith(rootDir.toUri().getPath() + "/")) {
        throw new IllegalArgumentException("Restore directory cannot be a sub directory of HBase " + "root directory. RootDir: " + rootDir + ", restoreDir: " + restoreDir);
    }
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
    MonitoredTask status = TaskMonitor.get().createStatus("Restoring  snapshot '" + snapshotName + "' to directory " + restoreDir);
    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher();
    // we send createBackRefs=false so that restored hfiles do not create back reference links
    // in the base hbase root dir.
    RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false);
    // TODO: parallelize.
    RestoreMetaChanges metaChanges = helper.restoreHdfsRegions();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Restored table dir:" + restoreDir);
        CommonFSUtils.logFileSystemState(fs, restoreDir, LOG);
    }
    return metaChanges;
}

Example 13 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class TestSnapshotFromMaster method testIsDoneContract.

/**
 * Test that the contract from the master for checking on a snapshot are valid.
 * <p>
 * <ol>
 * <li>If a snapshot fails with an error, we expect to get the source error.</li>
 * <li>If there is no snapshot name supplied, we should get an error.</li>
 * <li>If asking about a snapshot has hasn't occurred, you should get an error.</li>
 * </ol>
 */
@Test
public void testIsDoneContract() throws Exception {
    IsSnapshotDoneRequest.Builder builder = IsSnapshotDoneRequest.newBuilder();
    String snapshotName = "asyncExpectedFailureTest";
    // check that we get an exception when looking up snapshot where one hasn't happened
    SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), UnknownSnapshotException.class);
    // and that we get the same issue, even if we specify a name
    SnapshotDescription desc = SnapshotDescription.newBuilder().setName(snapshotName).setTable(TABLE_NAME.getNameAsString()).build();
    builder.setSnapshot(desc);
    SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), UnknownSnapshotException.class);
    // set a mock handler to simulate a snapshot
    DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
    Mockito.when(mockHandler.getException()).thenReturn(null);
    Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
    Mockito.when(mockHandler.isFinished()).thenReturn(Boolean.TRUE);
    Mockito.when(mockHandler.getCompletionTimestamp()).thenReturn(EnvironmentEdgeManager.currentTime());
    master.getSnapshotManager().setSnapshotHandlerForTesting(TABLE_NAME, mockHandler);
    // if we do a lookup without a snapshot name, we should fail - you should always know your name
    builder = IsSnapshotDoneRequest.newBuilder();
    SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), UnknownSnapshotException.class);
    // then do the lookup for the snapshot that it is done
    builder.setSnapshot(desc);
    IsSnapshotDoneResponse response = master.getMasterRpcServices().isSnapshotDone(null, builder.build());
    assertTrue("Snapshot didn't complete when it should have.", response.getDone());
    // now try the case where we are looking for a snapshot we didn't take
    builder.setSnapshot(SnapshotDescription.newBuilder().setName("Not A Snapshot").build());
    SnapshotTestingUtils.expectSnapshotDoneException(master, builder.build(), UnknownSnapshotException.class);
    // then create a snapshot to the fs and make sure that we can find it when checking done
    snapshotName = "completed";
    desc = createSnapshot(snapshotName);
    builder.setSnapshot(desc);
    response = master.getMasterRpcServices().isSnapshotDone(null, builder.build());
    assertTrue("Completed, on-disk snapshot not found", response.getDone());
}

Example 14 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class TestSnapshotFromMaster method testDeleteSnapshot.

@Test
public void testDeleteSnapshot() throws Exception {
    String snapshotName = "completed";
    SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
    DeleteSnapshotRequest request = DeleteSnapshotRequest.newBuilder().setSnapshot(snapshot).build();
    try {
        master.getMasterRpcServices().deleteSnapshot(null, request);
        fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
    } catch (org.apache.hbase.thirdparty.com.google.protobuf.ServiceException e) {
    // Expected
    }
    // write one snapshot to the fs
    createSnapshot(snapshotName);
    // then delete the existing snapshot,which shouldn't cause an exception to be thrown
    master.getMasterRpcServices().deleteSnapshot(null, request);
}

Example 15 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method takeSnapshotInternal.

private void takeSnapshotInternal(SnapshotDescription snapshot) throws IOException {
    // check to see if we already completed the snapshot
    if (isSnapshotCompleted(snapshot)) {
        throw new SnapshotExistsException("Snapshot '" + snapshot.getName() + "' already stored on the filesystem.", ProtobufUtil.createSnapshotDesc(snapshot));
    }
    LOG.debug("No existing snapshot, attempting snapshot...");
    // stop tracking "abandoned" handlers
    cleanupSentinels();
    // check to see if the table exists
    TableDescriptor desc = null;
    try {
        desc = master.getTableDescriptors().get(TableName.valueOf(snapshot.getTable()));
    } catch (FileNotFoundException e) {
        String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
        LOG.error(msg);
        throw new SnapshotCreationException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
    } catch (IOException e) {
        throw new SnapshotCreationException("Error while geting table description for table " + snapshot.getTable(), e, ProtobufUtil.createSnapshotDesc(snapshot));
    }
    if (desc == null) {
        throw new SnapshotCreationException("Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.", ProtobufUtil.createSnapshotDesc(snapshot));
    }
    SnapshotDescription.Builder builder = snapshot.toBuilder();
    // if not specified, set the snapshot format
    if (!snapshot.hasVersion()) {
        builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
    }
    RpcServer.getRequestUser().ifPresent(user -> {
        if (AccessChecker.isAuthorizationSupported(master.getConfiguration())) {
            builder.setOwner(user.getShortName());
        }
    });
    snapshot = builder.build();
    // call pre coproc hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
    if (cpHost != null) {
        snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
        cpHost.preSnapshot(snapshotPOJO, desc);
    }
    // if the table is enabled, then have the RS run actually the snapshot work
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.ENABLED)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Table enabled, starting distributed snapshots for {}", ClientSnapshotDescriptionUtils.toString(snapshot));
        }
        snapshotEnabledTable(snapshot);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
        }
    } else // For disabled table, snapshot is created by the master
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.DISABLED)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Table is disabled, running snapshot entirely on master for {}", ClientSnapshotDescriptionUtils.toString(snapshot));
        }
        snapshotDisabledTable(snapshot);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Started snapshot: {}", ClientSnapshotDescriptionUtils.toString(snapshot));
        }
    } else {
        LOG.error("Can't snapshot table '" + snapshot.getTable() + "', isn't open or closed, we don't know what to do!");
        TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable() + " isn't fully open.");
        throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, ProtobufUtil.createSnapshotDesc(snapshot));
    }
    // call post coproc hook
    if (cpHost != null) {
        cpHost.postSnapshot(snapshotPOJO, desc);
    }
}