Examples with SnapshotDescription org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription used on opensource projects

Example 16 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class SnapshotManager method checkSnapshotSupport.

/**
 * Called at startup, to verify if snapshot operation is supported, and to avoid
 * starting the master if there're snapshots present but the cleaners needed are missing.
 * Otherwise we can end up with snapshot data loss.
 * @param conf The {@link Configuration} object to use
 * @param mfs The MasterFileSystem to use
 * @throws IOException in case of file-system operation failure
 * @throws UnsupportedOperationException in case cleaners are missing and
 *         there're snapshot in the system
 */
private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs) throws IOException, UnsupportedOperationException {
    // Verify if snapshot is disabled by the user
    String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
    boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
    boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
    // Extract cleaners from conf
    Set<String> hfileCleaners = new HashSet<>();
    String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
    if (cleaners != null)
        Collections.addAll(hfileCleaners, cleaners);
    Set<String> logCleaners = new HashSet<>();
    cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
    if (cleaners != null)
        Collections.addAll(logCleaners, cleaners);
    // check if an older version of snapshot directory was present
    Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
    FileSystem fs = mfs.getFileSystem();
    List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir), false);
    if (ss != null && !ss.isEmpty()) {
        LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
        LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
    }
    // that there're no snapshot in the .snapshot folder.
    if (snapshotEnabled) {
        // Inject snapshot cleaners, if snapshot.enable is true
        hfileCleaners.add(SnapshotHFileCleaner.class.getName());
        hfileCleaners.add(HFileLinkCleaner.class.getName());
        // If sync acl to HDFS feature is enabled, then inject the cleaner
        if (SnapshotScannerHDFSAclHelper.isAclSyncToHdfsEnabled(conf)) {
            hfileCleaners.add(SnapshotScannerHDFSAclCleaner.class.getName());
        }
        // Set cleaners conf
        conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, hfileCleaners.toArray(new String[hfileCleaners.size()]));
        conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, logCleaners.toArray(new String[logCleaners.size()]));
    } else {
        // Verify if cleaners are present
        snapshotEnabled = hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) && hfileCleaners.contains(HFileLinkCleaner.class.getName());
        // Warn if the cleaners are enabled but the snapshot.enabled property is false/not set.
        if (snapshotEnabled) {
            LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " + "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " + (userDisabled ? "is set to 'false'." : "is not set."));
        }
    }
    // Mark snapshot feature as enabled if cleaners are present and user has not disabled it.
    this.isSnapshotSupported = snapshotEnabled && !userDisabled;
    // otherwise we end up with snapshot data loss.
    if (!snapshotEnabled) {
        LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
        Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
        if (fs.exists(snapshotDir)) {
            FileStatus[] snapshots = CommonFSUtils.listStatus(fs, snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
            if (snapshots != null) {
                LOG.error("Snapshots are present, but cleaners are not enabled.");
                checkSnapshotSupport();
            }
        }
    }
}

Example 17 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class ExportSnapshot method doWork.

/**
 * Execute the export snapshot by copying the snapshot metadata, hfiles and wals.
 * @return 0 on success, and != 0 upon failure.
 */
@Override
public int doWork() throws IOException {
    Configuration conf = getConf();
    // Check user options
    if (snapshotName == null) {
        System.err.println("Snapshot name not provided.");
        LOG.error("Use -h or --help for usage instructions.");
        return 0;
    }
    if (outputRoot == null) {
        System.err.println("Destination file-system (--" + Options.COPY_TO.getLongOpt() + ") not provided.");
        LOG.error("Use -h or --help for usage instructions.");
        return 0;
    }
    if (targetName == null) {
        targetName = snapshotName;
    }
    if (inputRoot == null) {
        inputRoot = CommonFSUtils.getRootDir(conf);
    } else {
        CommonFSUtils.setRootDir(conf, inputRoot);
    }
    Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
    srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
    FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
    Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
    destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);
    boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false) || conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null;
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot, destConf);
    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
    Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
    LOG.debug("inputFs={}, inputRoot={}", inputFs.getUri().toString(), inputRoot);
    LOG.debug("outputFs={}, outputRoot={}, skipTmp={}, initialOutputSnapshotDir={}", outputFs, outputRoot.toString(), skipTmp, initialOutputSnapshotDir);
    // Find the necessary directory which need to change owner and group
    Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);
    if (outputFs.exists(needSetOwnerDir)) {
        if (skipTmp) {
            needSetOwnerDir = outputSnapshotDir;
        } else {
            needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot, destConf);
            if (outputFs.exists(needSetOwnerDir)) {
                needSetOwnerDir = snapshotTmpDir;
            }
        }
    }
    // Check if the snapshot already exists
    if (outputFs.exists(outputSnapshotDir)) {
        if (overwrite) {
            if (!outputFs.delete(outputSnapshotDir, true)) {
                System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
                return 1;
            }
        } else {
            System.err.println("The snapshot '" + targetName + "' already exists in the destination: " + outputSnapshotDir);
            return 1;
        }
    }
    if (!skipTmp) {
        // Check if the snapshot already in-progress
        if (outputFs.exists(snapshotTmpDir)) {
            if (overwrite) {
                if (!outputFs.delete(snapshotTmpDir, true)) {
                    System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
                    return 1;
                }
            } else {
                System.err.println("A snapshot with the same name '" + targetName + "' may be in-progress");
                System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
                System.err.println("consider removing " + snapshotTmpDir + " by using the -overwrite option");
                return 1;
            }
        }
    }
    // Step 1 - Copy fs1:/.snapshot/<snapshot> to  fs2:/.snapshot/.tmp/<snapshot>
    // The snapshot references must be copied before the hfiles otherwise the cleaner
    // will remove them because they are unreferenced.
    List<Path> travesedPaths = new ArrayList<>();
    boolean copySucceeded = false;
    try {
        LOG.info("Copy Snapshot Manifest from " + snapshotDir + " to " + initialOutputSnapshotDir);
        travesedPaths = FSUtils.copyFilesParallel(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, conf, conf.getInt(CONF_COPY_MANIFEST_THREADS, DEFAULT_COPY_MANIFEST_THREADS));
        copySucceeded = true;
    } catch (IOException e) {
        throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" + snapshotDir + " to=" + initialOutputSnapshotDir, e);
    } finally {
        if (copySucceeded) {
            if (filesUser != null || filesGroup != null) {
                LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to " + filesUser) + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to " + filesGroup));
                setOwnerParallel(outputFs, filesUser, filesGroup, conf, travesedPaths);
            }
            if (filesMode > 0) {
                LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode);
                setPermissionParallel(outputFs, (short) filesMode, travesedPaths, conf);
            }
        }
    }
    // Write a new .snapshotinfo if the target name is different from the source name
    if (!targetName.equals(snapshotName)) {
        SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder().setName(targetName).build();
        SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);
        if (filesUser != null || filesGroup != null) {
            outputFs.setOwner(new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup);
        }
        if (filesMode > 0) {
            outputFs.setPermission(new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short) filesMode));
        }
    }
    // by the HFileArchiver, since they have no references.
    try {
        runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum, filesUser, filesGroup, filesMode, mappers, bandwidthMB);
        LOG.info("Finalize the Snapshot Export");
        if (!skipTmp) {
            // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
            if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
                throw new ExportSnapshotException("Unable to rename snapshot directory from=" + snapshotTmpDir + " to=" + outputSnapshotDir);
            }
        }
        // Step 4 - Verify snapshot integrity
        if (verifyTarget) {
            LOG.info("Verify snapshot integrity");
            verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);
        }
        LOG.info("Export Completed: " + targetName);
        return 0;
    } catch (Exception e) {
        LOG.error("Snapshot export failed", e);
        if (!skipTmp) {
            outputFs.delete(snapshotTmpDir, true);
        }
        outputFs.delete(outputSnapshotDir, true);
        return 1;
    } finally {
        IOUtils.closeStream(inputFs);
        IOUtils.closeStream(outputFs);
    }
}

Example 18 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class TableSnapshotInputFormatImpl method getSnapshotManifest.

public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, Path rootDir, FileSystem fs) throws IOException {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
    SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
    return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}

Example 19 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class FileArchiverNotifierImpl method computeSnapshotSizes.

/**
 * Computes the size of each snapshot against the table referenced by {@code this}.
 *
 * @param snapshots A sorted list of snapshots against {@code tn}.
 * @return A list of the size for each snapshot against {@code tn}.
 */
List<SnapshotWithSize> computeSnapshotSizes(List<String> snapshots) throws IOException {
    final List<SnapshotWithSize> snapshotSizes = new ArrayList<>(snapshots.size());
    final Path rootDir = CommonFSUtils.getRootDir(conf);
    // Get the map of store file names to store file path for this table
    final Set<String> tableReferencedStoreFiles;
    try {
        tableReferencedStoreFiles = FSUtils.getTableStoreFilePathMap(fs, rootDir).keySet();
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        return null;
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("Paths for " + tn + ": " + tableReferencedStoreFiles);
    }
    // For each snapshot on this table, get the files which the snapshot references which
    // the table does not.
    Set<String> snapshotReferencedFiles = new HashSet<>();
    for (String snapshotName : snapshots) {
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
        SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
        if (LOG.isTraceEnabled()) {
            LOG.trace("Files referenced by other snapshots: " + snapshotReferencedFiles);
        }
        // Get the set of files from the manifest that this snapshot references which are not also
        // referenced by the originating table.
        Set<StoreFileReference> unreferencedStoreFileNames = getStoreFilesFromSnapshot(manifest, (sfn) -> !tableReferencedStoreFiles.contains(sfn) && !snapshotReferencedFiles.contains(sfn));
        if (LOG.isTraceEnabled()) {
            LOG.trace("Snapshot " + snapshotName + " solely references the files: " + unreferencedStoreFileNames);
        }
        // Compute the size of the store files for this snapshot
        long size = getSizeOfStoreFiles(tn, unreferencedStoreFileNames);
        if (LOG.isTraceEnabled()) {
            LOG.trace("Computed size of " + snapshotName + " to be " + size);
        }
        // Persist this snapshot's size into the map
        snapshotSizes.add(new SnapshotWithSize(snapshotName, size));
        // Make sure that we don't double-count the same file
        for (StoreFileReference ref : unreferencedStoreFileNames) {
            for (String fileNames : ref.getFamilyToFilesMapping().values()) {
                snapshotReferencedFiles.add(fileNames);
            }
        }
    }
    return snapshotSizes;
}

Example 20 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription in project hbase by apache.

the class TestExportSnapshot method verifySnapshot.

/*
   * Verify if the files exists
   */
protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs, final Path rootDir, final TableName tableName, final String snapshotName, final RegionPredicate bypassregionPredicate) throws IOException {
    final Path exportedSnapshot = new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
    final Set<String> snapshotFiles = new HashSet<>();
    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot, new SnapshotReferenceUtil.SnapshotVisitor() {

        @Override
        public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
            if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
                return;
            }
            if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) {
                String hfile = storeFile.getName();
                snapshotFiles.add(hfile);
                verifyNonEmptyFile(new Path(exportedArchive, new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
            } else {
                Pair<String, String> referredToRegionAndFile = StoreFileInfo.getReferredToRegionAndFile(storeFile.getName());
                String region = referredToRegionAndFile.getFirst();
                String hfile = referredToRegionAndFile.getSecond();
                snapshotFiles.add(hfile);
                verifyNonEmptyFile(new Path(exportedArchive, new Path(CommonFSUtils.getTableDir(new Path("./"), tableName), new Path(region, new Path(family, hfile)))));
            }
        }

        private void verifyNonEmptyFile(final Path path) throws IOException {
            assertTrue(path + " should exists", fs.exists(path));
            assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
        }
    });
    // Verify Snapshot description
    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
    assertTrue(desc.getName().equals(snapshotName));
    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
    return snapshotFiles;
}