Examples with SnapshotDescription org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription used on opensource projects

Example 26 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class ExportSnapshot method doWork.

/**
   * Execute the export snapshot by copying the snapshot metadata, hfiles and wals.
   * @return 0 on success, and != 0 upon failure.
   */
@Override
public int doWork() throws IOException {
    Configuration conf = getConf();
    // Check user options
    if (snapshotName == null) {
        System.err.println("Snapshot name not provided.");
        LOG.error("Use -h or --help for usage instructions.");
        return 0;
    }
    if (outputRoot == null) {
        System.err.println("Destination file-system (--" + Options.COPY_TO.getLongOpt() + ") not provided.");
        LOG.error("Use -h or --help for usage instructions.");
        return 0;
    }
    if (targetName == null) {
        targetName = snapshotName;
    }
    if (inputRoot == null) {
        inputRoot = FSUtils.getRootDir(conf);
    } else {
        FSUtils.setRootDir(conf, inputRoot);
    }
    Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
    srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
    FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf);
    LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
    Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
    destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true);
    FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf);
    LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
    boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
    Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
    Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
    Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
    // Find the necessary directory which need to change owner and group
    Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot);
    if (outputFs.exists(needSetOwnerDir)) {
        if (skipTmp) {
            needSetOwnerDir = outputSnapshotDir;
        } else {
            needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot);
            if (outputFs.exists(needSetOwnerDir)) {
                needSetOwnerDir = snapshotTmpDir;
            }
        }
    }
    // Check if the snapshot already exists
    if (outputFs.exists(outputSnapshotDir)) {
        if (overwrite) {
            if (!outputFs.delete(outputSnapshotDir, true)) {
                System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
                return 1;
            }
        } else {
            System.err.println("The snapshot '" + targetName + "' already exists in the destination: " + outputSnapshotDir);
            return 1;
        }
    }
    if (!skipTmp) {
        // Check if the snapshot already in-progress
        if (outputFs.exists(snapshotTmpDir)) {
            if (overwrite) {
                if (!outputFs.delete(snapshotTmpDir, true)) {
                    System.err.println("Unable to remove existing snapshot tmp directory: " + snapshotTmpDir);
                    return 1;
                }
            } else {
                System.err.println("A snapshot with the same name '" + targetName + "' may be in-progress");
                System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
                System.err.println("consider removing " + snapshotTmpDir + " by using the -overwrite option");
                return 1;
            }
        }
    }
    // will remove them because they are unreferenced.
    try {
        LOG.info("Copy Snapshot Manifest");
        FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
    } catch (IOException e) {
        throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" + snapshotDir + " to=" + initialOutputSnapshotDir, e);
    } finally {
        if (filesUser != null || filesGroup != null) {
            LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to " + filesUser) + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to " + filesGroup));
            setOwner(outputFs, needSetOwnerDir, filesUser, filesGroup, true);
        }
        if (filesMode > 0) {
            LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode);
            setPermission(outputFs, needSetOwnerDir, (short) filesMode, true);
        }
    }
    // Write a new .snapshotinfo if the target name is different from the source name
    if (!targetName.equals(snapshotName)) {
        SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir).toBuilder().setName(targetName).build();
        SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs);
        if (filesUser != null || filesGroup != null) {
            outputFs.setOwner(new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup);
        }
        if (filesMode > 0) {
            outputFs.setPermission(new Path(initialOutputSnapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short) filesMode));
        }
    }
    // by the HFileArchiver, since they have no references.
    try {
        runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum, filesUser, filesGroup, filesMode, mappers, bandwidthMB);
        LOG.info("Finalize the Snapshot Export");
        if (!skipTmp) {
            // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
            if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
                throw new ExportSnapshotException("Unable to rename snapshot directory from=" + snapshotTmpDir + " to=" + outputSnapshotDir);
            }
        }
        // Step 4 - Verify snapshot integrity
        if (verifyTarget) {
            LOG.info("Verify snapshot integrity");
            verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir);
        }
        LOG.info("Export Completed: " + targetName);
        return 0;
    } catch (Exception e) {
        LOG.error("Snapshot export failed", e);
        if (!skipTmp) {
            outputFs.delete(snapshotTmpDir, true);
        }
        outputFs.delete(outputSnapshotDir, true);
        return 1;
    } finally {
        IOUtils.closeStream(inputFs);
        IOUtils.closeStream(outputFs);
    }
}

Example 27 with SnapshotDescription

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription in project hbase by apache.

the class SnapshotDescriptionUtils method readSnapshotInfo.

/**
   * Read in the {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} stored for the snapshot in the passed directory
   * @param fs filesystem where the snapshot was taken
   * @param snapshotDir directory where the snapshot was stored
   * @return the stored snapshot description
   * @throws CorruptedSnapshotException if the
   * snapshot cannot be read
   */
public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir) throws CorruptedSnapshotException {
    Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE);
    try {
        FSDataInputStream in = null;
        try {
            in = fs.open(snapshotInfo);
            SnapshotDescription desc = SnapshotDescription.parseFrom(in);
            return desc;
        } finally {
            if (in != null)
                in.close();
        }
    } catch (IOException e) {
        throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e);
    }
}