Examples with SnapshotDescription org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription used on opensource projects

Example 16 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotReferenceUtil method verifySnapshot.

/**
   * Verify the validity of the snapshot
   *
   * @param conf The current {@link Configuration} instance.
   * @param fs {@link FileSystem}
   * @param manifest snapshot manifest to inspect
   * @throws CorruptedSnapshotException if the snapshot is corrupted
   * @throws IOException if an error occurred while scanning the directory
   */
public static void verifySnapshot(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest) throws IOException {
    final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
    final Path snapshotDir = manifest.getSnapshotDir();
    concurrentVisitReferencedFiles(conf, fs, manifest, "VerifySnapshot", new StoreFileVisitor() {

        @Override
        public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile);
        }
    });
}

Example 17 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotReferenceUtil method concurrentVisitReferencedFiles.

public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final ExecutorService exec, final StoreFileVisitor visitor) throws IOException {
    final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
    final Path snapshotDir = manifest.getSnapshotDir();
    List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null || regionManifests.isEmpty()) {
        LOG.debug("No manifest files present: " + snapshotDir);
        return;
    }
    final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<>(exec);
    for (final SnapshotRegionManifest regionManifest : regionManifests) {
        completionService.submit(new Callable<Void>() {

            @Override
            public Void call() throws IOException {
                visitRegionStoreFiles(regionManifest, visitor);
                return null;
            }
        });
    }
    try {
        for (int i = 0; i < regionManifests.size(); ++i) {
            completionService.take().get();
        }
    } catch (InterruptedException e) {
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        if (e.getCause() instanceof CorruptedSnapshotException) {
            throw new CorruptedSnapshotException(e.getCause().getMessage(), ProtobufUtil.createSnapshotDesc(snapshotDesc));
        } else {
            IOException ex = new IOException();
            ex.initCause(e.getCause());
            throw ex;
        }
    }
}

Example 18 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method checkSnapshotSupport.

/**
   * Called at startup, to verify if snapshot operation is supported, and to avoid
   * starting the master if there're snapshots present but the cleaners needed are missing.
   * Otherwise we can end up with snapshot data loss.
   * @param conf The {@link Configuration} object to use
   * @param mfs The MasterFileSystem to use
   * @throws IOException in case of file-system operation failure
   * @throws UnsupportedOperationException in case cleaners are missing and
   *         there're snapshot in the system
   */
private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs) throws IOException, UnsupportedOperationException {
    // Verify if snapshot is disabled by the user
    String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
    boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
    boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
    // Extract cleaners from conf
    Set<String> hfileCleaners = new HashSet<>();
    String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
    if (cleaners != null)
        Collections.addAll(hfileCleaners, cleaners);
    Set<String> logCleaners = new HashSet<>();
    cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
    if (cleaners != null)
        Collections.addAll(logCleaners, cleaners);
    // check if an older version of snapshot directory was present
    Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
    FileSystem fs = mfs.getFileSystem();
    List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
    if (ss != null && !ss.isEmpty()) {
        LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
        LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
    }
    // that there're no snapshot in the .snapshot folder.
    if (snapshotEnabled) {
        // Inject snapshot cleaners, if snapshot.enable is true
        hfileCleaners.add(SnapshotHFileCleaner.class.getName());
        hfileCleaners.add(HFileLinkCleaner.class.getName());
        // Set cleaners conf
        conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, hfileCleaners.toArray(new String[hfileCleaners.size()]));
        conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, logCleaners.toArray(new String[logCleaners.size()]));
    } else {
        // Verify if cleaners are present
        snapshotEnabled = hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) && hfileCleaners.contains(HFileLinkCleaner.class.getName());
        // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
        if (snapshotEnabled) {
            LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " + "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " + (userDisabled ? "is set to 'false'." : "is not set."));
        }
    }
    // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
    this.isSnapshotSupported = snapshotEnabled && !userDisabled;
    // otherwise we end up with snapshot data loss.
    if (!snapshotEnabled) {
        LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
        Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
        if (fs.exists(snapshotDir)) {
            FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
            if (snapshots != null) {
                LOG.error("Snapshots are present, but cleaners are not enabled.");
                checkSnapshotSupport();
            }
        }
    }
}

Example 19 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method getCompletedSnapshots.

/**
   * Gets the list of all completed snapshots.
   * @param snapshotDir snapshot directory
   * @return list of SnapshotDescriptions
   * @throws IOException File system exception
   */
private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
    List<SnapshotDescription> snapshotDescs = new ArrayList<>();
    // first create the snapshot root path and check to see if it exists
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    if (snapshotDir == null)
        snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
    // if there are no snapshots, return an empty list
    if (!fs.exists(snapshotDir)) {
        return snapshotDescs;
    }
    // ignore all the snapshots in progress
    FileStatus[] snapshots = fs.listStatus(snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    // loop through all the completed snapshots
    for (FileStatus snapshot : snapshots) {
        Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
        // if the snapshot is bad
        if (!fs.exists(info)) {
            LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
            continue;
        }
        FSDataInputStream in = null;
        try {
            in = fs.open(info);
            SnapshotDescription desc = SnapshotDescription.parseFrom(in);
            if (cpHost != null) {
                try {
                    cpHost.preListSnapshot(desc);
                } catch (AccessDeniedException e) {
                    LOG.warn("Current user does not have access to " + desc.getName() + " snapshot. " + "Either you should be owner of this snapshot or admin user.");
                    // Skip this and try for next snapshot
                    continue;
                }
            }
            snapshotDescs.add(desc);
            // call coproc post hook
            if (cpHost != null) {
                cpHost.postListSnapshot(desc);
            }
        } catch (IOException e) {
            LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
        } finally {
            if (in != null) {
                in.close();
            }
        }
    }
    return snapshotDescs;
}

Example 20 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method takeSnapshot.

/**
   * Take a snapshot based on the enabled/disabled state of the table.
   *
   * @param snapshot
   * @throws HBaseSnapshotException when a snapshot specific exception occurs.
   * @throws IOException when some sort of generic IO exception occurs.
   */
public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
    // check to see if we already completed the snapshot
    if (isSnapshotCompleted(snapshot)) {
        throw new SnapshotExistsException("Snapshot '" + snapshot.getName() + "' already stored on the filesystem.", ProtobufUtil.createSnapshotDesc(snapshot));
    }
    LOG.debug("No existing snapshot, attempting snapshot...");
    // stop tracking "abandoned" handlers
    cleanupSentinels();
    // check to see if the table exists
    HTableDescriptor desc = null;
    try {
        desc = master.getTableDescriptors().get(TableName.valueOf(snapshot.getTable()));
    } catch (FileNotFoundException e) {
        String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
        LOG.error(msg);
        throw new SnapshotCreationException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
    } catch (IOException e) {
        throw new SnapshotCreationException("Error while geting table description for table " + snapshot.getTable(), e, ProtobufUtil.createSnapshotDesc(snapshot));
    }
    if (desc == null) {
        throw new SnapshotCreationException("Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.", ProtobufUtil.createSnapshotDesc(snapshot));
    }
    SnapshotDescription.Builder builder = snapshot.toBuilder();
    // if not specified, set the snapshot format
    if (!snapshot.hasVersion()) {
        builder.setVersion(SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_VERSION);
    }
    User user = RpcServer.getRequestUser();
    if (User.isHBaseSecurityEnabled(master.getConfiguration()) && user != null) {
        builder.setOwner(user.getShortName());
    }
    snapshot = builder.build();
    // call pre coproc hook
    MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
    if (cpHost != null) {
        cpHost.preSnapshot(snapshot, desc);
    }
    // if the table is enabled, then have the RS run actually the snapshot work
    TableName snapshotTable = TableName.valueOf(snapshot.getTable());
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.ENABLED)) {
        LOG.debug("Table enabled, starting distributed snapshot.");
        snapshotEnabledTable(snapshot);
        LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    } else // For disabled table, snapshot is created by the master
    if (master.getTableStateManager().isTableState(snapshotTable, TableState.State.DISABLED)) {
        LOG.debug("Table is disabled, running snapshot entirely on master.");
        snapshotDisabledTable(snapshot);
        LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
    } else {
        LOG.error("Can't snapshot table '" + snapshot.getTable() + "', isn't open or closed, we don't know what to do!");
        TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable() + " isn't fully open.");
        throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, ProtobufUtil.createSnapshotDesc(snapshot));
    }
    // call post coproc hook
    if (cpHost != null) {
        cpHost.postSnapshot(snapshot, desc);
    }
}