Examples with SnapshotDescription org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription used on opensource projects

Example 21 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method restoreOrCloneSnapshot.

/**
   * Restore or Clone the specified snapshot
   * @param reqSnapshot
   * @param nonceKey unique identifier to prevent duplicated RPC
   * @throws IOException
   */
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey) throws IOException {
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
    // check if the snapshot exists
    if (!fs.exists(snapshotDir)) {
        LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
        throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(reqSnapshot));
    }
    // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
    // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
    // information.
    SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs, snapshotDir, snapshot);
    HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
    TableName tableName = TableName.valueOf(reqSnapshot.getTable());
    // stop tracking "abandoned" handlers
    cleanupSentinels();
    // Verify snapshot validity
    SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
    // Execute the restore/clone operation
    long procId;
    if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
        procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey);
    } else {
        procId = cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey);
    }
    return procId;
}

Example 22 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class TestWithDisabledAuthorization method testPassiveMasterOperations.

/** Test master observer */
@Test
public void testPassiveMasterOperations() throws Exception {
    // preCreateTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
            htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
            ACCESS_CONTROLLER.preCreateTable(ObserverContext.createAndPrepare(CP_ENV, null), htd, null);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preModifyTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
            htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
            htd.addFamily(new HColumnDescriptor(TEST_FAMILY2));
            ACCESS_CONTROLLER.preModifyTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), htd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preDeleteTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName());
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preTruncateTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preTruncateTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName());
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preAddColumnFamily
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
            ACCESS_CONTROLLER.preAddColumnFamily(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), hcd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preModifyColumnFamily
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
            ACCESS_CONTROLLER.preModifyColumnFamily(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), hcd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preDeleteColumnFamily
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preDeleteColumnFamily(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), TEST_FAMILY2);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preEnableTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName());
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preDisableTable
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName());
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preMove
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HRegionInfo region = new HRegionInfo(TEST_TABLE.getTableName());
            ServerName srcServer = ServerName.valueOf("1.1.1.1", 1, 0);
            ServerName destServer = ServerName.valueOf("2.2.2.2", 2, 0);
            ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), region, srcServer, destServer);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preAssign
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HRegionInfo region = new HRegionInfo(TEST_TABLE.getTableName());
            ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null), region);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preUnassign
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            HRegionInfo region = new HRegionInfo(TEST_TABLE.getTableName());
            ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), region, true);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preBalance
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preBalance(ObserverContext.createAndPrepare(CP_ENV, null));
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preBalanceSwitch
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preBalanceSwitch(ObserverContext.createAndPrepare(CP_ENV, null), true);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preSnapshot
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("foo").build();
            HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
            ACCESS_CONTROLLER.preSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), snapshot, htd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preListSnapshot
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("foo").build();
            ACCESS_CONTROLLER.preListSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), snapshot);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preCloneSnapshot
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("foo").build();
            HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
            ACCESS_CONTROLLER.preCloneSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), snapshot, htd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preRestoreSnapshot
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("foo").build();
            HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
            ACCESS_CONTROLLER.preRestoreSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), snapshot, htd);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preDeleteSnapshot
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("foo").build();
            ACCESS_CONTROLLER.preDeleteSnapshot(ObserverContext.createAndPrepare(CP_ENV, null), snapshot);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preGetTableDescriptors
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            List<TableName> tableNamesList = Lists.newArrayList();
            tableNamesList.add(TEST_TABLE.getTableName());
            List<HTableDescriptor> descriptors = Lists.newArrayList();
            ACCESS_CONTROLLER.preGetTableDescriptors(ObserverContext.createAndPrepare(CP_ENV, null), tableNamesList, descriptors, ".+");
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preGetTableNames
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            List<HTableDescriptor> descriptors = Lists.newArrayList();
            ACCESS_CONTROLLER.preGetTableNames(ObserverContext.createAndPrepare(CP_ENV, null), descriptors, ".+");
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preCreateNamespace
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            NamespaceDescriptor ns = NamespaceDescriptor.create("test").build();
            ACCESS_CONTROLLER.preCreateNamespace(ObserverContext.createAndPrepare(CP_ENV, null), ns);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preDeleteNamespace
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preDeleteNamespace(ObserverContext.createAndPrepare(CP_ENV, null), "test");
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preModifyNamespace
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            NamespaceDescriptor ns = NamespaceDescriptor.create("test").build();
            ACCESS_CONTROLLER.preModifyNamespace(ObserverContext.createAndPrepare(CP_ENV, null), ns);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preGetNamespaceDescriptor
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preGetNamespaceDescriptor(ObserverContext.createAndPrepare(CP_ENV, null), "test");
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preListNamespaceDescriptors
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            List<NamespaceDescriptor> descriptors = Lists.newArrayList();
            ACCESS_CONTROLLER.preListNamespaceDescriptors(ObserverContext.createAndPrepare(CP_ENV, null), descriptors);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preSplit
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            ACCESS_CONTROLLER.preSplitRegion(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), Bytes.toBytes("ss"));
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preSetUserQuota
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            Quotas quotas = Quotas.newBuilder().build();
            ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), "testuser", quotas);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preSetTableQuota
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            Quotas quotas = Quotas.newBuilder().build();
            ACCESS_CONTROLLER.preSetTableQuota(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), quotas);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
    // preSetNamespaceQuota
    verifyAllowed(new AccessTestAction() {

        @Override
        public Object run() throws Exception {
            Quotas quotas = Quotas.newBuilder().build();
            ACCESS_CONTROLLER.preSetNamespaceQuota(ObserverContext.createAndPrepare(CP_ENV, null), "test", quotas);
            return null;
        }
    }, SUPERUSER, USER_ADMIN, USER_RW, USER_RO, USER_OWNER, USER_CREATE, USER_QUAL, USER_NONE);
}

Example 23 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class TestSnapshotFromMaster method testGetCompletedSnapshots.

@Test(timeout = 300000)
public void testGetCompletedSnapshots() throws Exception {
    // first check when there are no snapshots
    GetCompletedSnapshotsRequest request = GetCompletedSnapshotsRequest.newBuilder().build();
    GetCompletedSnapshotsResponse response = master.getMasterRpcServices().getCompletedSnapshots(null, request);
    assertEquals("Found unexpected number of snapshots", 0, response.getSnapshotsCount());
    // write one snapshot to the fs
    String snapshotName = "completed";
    SnapshotDescription snapshot = createSnapshot(snapshotName);
    // check that we get one snapshot
    response = master.getMasterRpcServices().getCompletedSnapshots(null, request);
    assertEquals("Found unexpected number of snapshots", 1, response.getSnapshotsCount());
    List<SnapshotDescription> snapshots = response.getSnapshotsList();
    List<SnapshotDescription> expected = Lists.newArrayList(snapshot);
    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
    // write a second snapshot
    snapshotName = "completed_two";
    snapshot = createSnapshot(snapshotName);
    expected.add(snapshot);
    // check that we get one snapshot
    response = master.getMasterRpcServices().getCompletedSnapshots(null, request);
    assertEquals("Found unexpected number of snapshots", 2, response.getSnapshotsCount());
    snapshots = response.getSnapshotsList();
    assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
}

Example 24 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class TestExportSnapshot method verifySnapshot.

/*
   * Verify if the files exists
   */
protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, final Path rootDir, final TableName tableName, final String snapshotName, final RegionPredicate bypassregionPredicate) throws IOException {
    final Path exportedSnapshot = new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
    final Set<String> snapshotFiles = new HashSet<>();
    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, new SnapshotReferenceUtil.SnapshotVisitor() {

        @Override
        public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo))
                return;
            String hfile = storeFile.getName();
            snapshotFiles.add(hfile);
            if (storeFile.hasReference()) {
            // Nothing to do here, we have already the reference embedded
            } else {
                verifyNonEmptyFile(new Path(exportedArchive, new Path(FSUtils.getTableDir(new Path("./"), tableName), new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
            }
        }

        private void verifyNonEmptyFile(final Path path) throws IOException {
            assertTrue(path + " should exists", fs.exists(path));
            assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
        }
    });
    // Verify Snapshot description
    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
    assertTrue(desc.getName().equals(snapshotName));
    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
    return snapshotFiles;
}

Example 25 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class TableSnapshotInputFormatImpl method getSnapshotManifest.

public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, Path rootDir, FileSystem fs) throws IOException {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}