Examples with SnapshotDiffReport org.apache.hadoop.hdfs.protocol.SnapshotDiffReport used on opensource projects

Example 26 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class SnapshotManager method diff.

/**
   * Compute the difference between two snapshots of a directory, or between a
   * snapshot of the directory and its current tree.
   */
public SnapshotDiffReport diff(final INodesInPath iip, final String snapshotRootPath, final String from, final String to) throws IOException {
    // Find the source root directory path where the snapshots were taken.
    // All the check for path has been included in the valueOf method.
    final INodeDirectory snapshotRoot = getSnapshottableRoot(iip);
    if ((from == null || from.isEmpty()) && (to == null || to.isEmpty())) {
        // both fromSnapshot and toSnapshot indicate the current tree
        return new SnapshotDiffReport(snapshotRootPath, from, to, Collections.<DiffReportEntry>emptyList());
    }
    final SnapshotDiffInfo diffs = snapshotRoot.getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to);
    return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(snapshotRootPath, from, to, Collections.<DiffReportEntry>emptyList());
}

Example 27 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class DistCpSync method getAllDiffs.

/**
   * Get all diffs from source directory snapshot diff report, put them into an
   * EnumMap whose key is DiffType, and value is a DiffInfo list. If there is
   * no entry for a given DiffType, the associated value will be an empty list.
   */
private boolean getAllDiffs() throws IOException {
    Path ssDir = isRdiff() ? inputOptions.getTargetPath() : inputOptions.getSourcePaths().get(0);
    try {
        DistributedFileSystem fs = (DistributedFileSystem) ssDir.getFileSystem(conf);
        final String from = getSnapshotName(inputOptions.getFromSnapshot());
        final String to = getSnapshotName(inputOptions.getToSnapshot());
        SnapshotDiffReport report = fs.getSnapshotDiffReport(ssDir, from, to);
        this.diffMap = new EnumMap<>(SnapshotDiffReport.DiffType.class);
        for (SnapshotDiffReport.DiffType type : SnapshotDiffReport.DiffType.values()) {
            diffMap.put(type, new ArrayList<DiffInfo>());
        }
        for (SnapshotDiffReport.DiffReportEntry entry : report.getDiffList()) {
            // since its sourcepath is empty.
            if (entry.getSourcePath().length <= 0) {
                continue;
            }
            SnapshotDiffReport.DiffType dt = entry.getType();
            List<DiffInfo> list = diffMap.get(dt);
            if (dt == SnapshotDiffReport.DiffType.MODIFY || dt == SnapshotDiffReport.DiffType.CREATE || dt == SnapshotDiffReport.DiffType.DELETE) {
                final Path source = new Path(DFSUtilClient.bytes2String(entry.getSourcePath()));
                list.add(new DiffInfo(source, null, dt));
            } else if (dt == SnapshotDiffReport.DiffType.RENAME) {
                final Path source = new Path(DFSUtilClient.bytes2String(entry.getSourcePath()));
                final Path target = new Path(DFSUtilClient.bytes2String(entry.getTargetPath()));
                list.add(new DiffInfo(source, target, dt));
            }
        }
        return true;
    } catch (IOException e) {
        DistCp.LOG.warn("Failed to compute snapshot diff on " + ssDir, e);
    }
    this.diffMap = null;
    return false;
}

Example 28 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class DistCpSync method checkNoChange.

/**
   * Compute the snapshot diff on the given file system. Return true if the diff
   * is empty, i.e., no changes have happened in the FS.
   */
private boolean checkNoChange(DistributedFileSystem fs, Path path) {
    try {
        final String from = getSnapshotName(inputOptions.getFromSnapshot());
        SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, from, "");
        if (!targetDiff.getDiffList().isEmpty()) {
            DistCp.LOG.warn("The target has been modified since snapshot " + inputOptions.getFromSnapshot());
            return false;
        } else {
            return true;
        }
    } catch (IOException e) {
        DistCp.LOG.warn("Failed to compute snapshot diff on " + path + " at snapshot " + inputOptions.getFromSnapshot(), e);
    }
    return false;
}

Example 29 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class TestDistCpSyncReverseBase method testAndVerify.

private void testAndVerify(int numDeletedAndModified) throws Exception {
    SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1");
    System.out.println(report);
    final FsShell shell = new FsShell(conf);
    lsrSource("Before sync source: ", shell, source);
    lsr("Before sync target: ", shell, target);
    DistCpSync distCpSync = new DistCpSync(options, conf);
    // do the sync
    distCpSync.sync();
    lsr("After sync target: ", shell, target);
    // make sure the source path has been updated to the snapshot path
    final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1");
    Assert.assertEquals(spath, options.getSourcePaths().get(0));
    // build copy listing
    final Path listingPath = new Path("/tmp/META/fileList.seq");
    CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
    listing.buildListing(listingPath, options);
    Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(conf, null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> context = stubContext.getContext();
    // Enable append
    context.getConfiguration().setBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), true);
    copyMapper.setup(context);
    for (Map.Entry<Text, CopyListingFileStatus> entry : copyListing.entrySet()) {
        copyMapper.map(entry.getKey(), entry.getValue(), context);
    }
    // verify that we only list modified and created files/directories
    Assert.assertEquals(numDeletedAndModified, copyListing.size());
    lsr("After Copy target: ", shell, target);
    // verify the source and target now has the same structure
    verifyCopy(dfs.getFileStatus(spath), dfs.getFileStatus(target), false);
}

Example 30 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class TestSnapshotCommands method testSnapshotCommandsWithURI.

@Test(timeout = 60000)
public void testSnapshotCommandsWithURI() throws Exception {
    Configuration config = new HdfsConfiguration();
    //fs.defaultFS should not be used, when path is fully qualified.
    config.set("fs.defaultFS", "hdfs://127.0.0.1:1024");
    String path = fs.getUri() + "/Fully/QPath";
    DFSTestUtil.DFSAdminRun("-allowSnapshot " + path, 0, "Allowing snaphot on " + path + " succeeded", config);
    DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn1", config);
    // create file1
    DFSTestUtil.createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File1"), 1024, (short) 1, 100);
    // create file2
    DFSTestUtil.createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File2"), 1024, (short) 1, 100);
    DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn2", config);
    // verify the snapshotdiff using api and command line
    SnapshotDiffReport report = fs.getSnapshotDiffReport(new Path(path), "sn1", "sn2");
    DFSTestUtil.toolRun(new SnapshotDiff(config), path + " sn1 sn2", 0, report.toString());
    DFSTestUtil.FsShellRun("-renameSnapshot " + path + " sn2 sn3", config);
    DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn1", config);
    DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn3", config);
    DFSTestUtil.DFSAdminRun("-disallowSnapshot " + path, 0, "Disallowing snaphot on " + path + " succeeded", config);
    fs.delete(new Path("/Fully/QPath"), true);
}