Examples with SnapshotDiffReport org.apache.hadoop.hdfs.protocol.SnapshotDiffReport used on opensource projects

Example 21 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class TestDistCpSync method testSync2.

@Test
public void testSync2() throws Exception {
    initData2(source);
    initData2(target);
    enableAndCreateFirstSnapshot();
    // make changes under source
    changeData2(source);
    dfs.createSnapshot(source, "s2");
    SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
    System.out.println(report);
    syncAndVerify();
}

Example 22 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class TestDistCpSync method testSync5.

/**
   * Test a case with different delete and rename sequences.
   */
@Test
public void testSync5() throws Exception {
    initData5(source);
    initData5(target);
    enableAndCreateFirstSnapshot();
    // make changes under source
    changeData5(source);
    dfs.createSnapshot(source, "s2");
    SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
    System.out.println(report);
    syncAndVerify();
}

Example 23 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class TestDistCpSync method testAndVerify.

private void testAndVerify(int numCreatedModified) throws Exception {
    SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2");
    System.out.println(report);
    DistCpSync distCpSync = new DistCpSync(options, conf);
    // do the sync
    Assert.assertTrue(distCpSync.sync());
    // make sure the source path has been updated to the snapshot path
    final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2");
    Assert.assertEquals(spath, options.getSourcePaths().get(0));
    // build copy listing
    final Path listingPath = new Path("/tmp/META/fileList.seq");
    CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync);
    listing.buildListing(listingPath, options);
    Map<Text, CopyListingFileStatus> copyListing = getListing(listingPath);
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(conf, null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context<Text, CopyListingFileStatus, Text, Text> context = stubContext.getContext();
    // Enable append
    context.getConfiguration().setBoolean(DistCpOptionSwitch.APPEND.getConfigLabel(), true);
    copyMapper.setup(context);
    for (Map.Entry<Text, CopyListingFileStatus> entry : copyListing.entrySet()) {
        copyMapper.map(entry.getKey(), entry.getValue(), context);
    }
    // verify that we only list modified and created files/directories
    Assert.assertEquals(numCreatedModified, copyListing.size());
    // verify the source and target now has the same structure
    verifyCopy(dfs.getFileStatus(spath), dfs.getFileStatus(target), false);
}

Example 24 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class FSDirSnapshotOp method getSnapshotDiffReport.

static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd, SnapshotManager snapshotManager, String path, String fromSnapshot, String toSnapshot) throws IOException {
    SnapshotDiffReport diffs;
    final FSPermissionChecker pc = fsd.getPermissionChecker();
    fsd.readLock();
    try {
        INodesInPath iip = fsd.resolvePath(pc, path, DirOp.READ);
        if (fsd.isPermissionEnabled()) {
            checkSubtreeReadPermission(fsd, pc, path, fromSnapshot);
            checkSubtreeReadPermission(fsd, pc, path, toSnapshot);
        }
        diffs = snapshotManager.diff(iip, path, fromSnapshot, toSnapshot);
    } finally {
        fsd.readUnlock();
    }
    return diffs;
}

Example 25 with SnapshotDiffReport

use of org.apache.hadoop.hdfs.protocol.SnapshotDiffReport in project hadoop by apache.

the class FSNamesystem method getSnapshotDiffReport.

/**
   * Get the difference between two snapshots (or between a snapshot and the
   * current status) of a snapshottable directory.
   * 
   * @param path The full path of the snapshottable directory.
   * @param fromSnapshot Name of the snapshot to calculate the diff from. Null
   *          or empty string indicates the current tree.
   * @param toSnapshot Name of the snapshot to calculated the diff to. Null or
   *          empty string indicates the current tree.
   * @return A report about the difference between {@code fromSnapshot} and 
   *         {@code toSnapshot}. Modified/deleted/created/renamed files and 
   *         directories belonging to the snapshottable directories are listed 
   *         and labeled as M/-/+/R respectively. 
   * @throws IOException
   */
SnapshotDiffReport getSnapshotDiffReport(String path, String fromSnapshot, String toSnapshot) throws IOException {
    final String operationName = "computeSnapshotDiff";
    SnapshotDiffReport diffs = null;
    checkOperation(OperationCategory.READ);
    boolean success = false;
    String fromSnapshotRoot = (fromSnapshot == null || fromSnapshot.isEmpty()) ? path : Snapshot.getSnapshotPath(path, fromSnapshot);
    String toSnapshotRoot = (toSnapshot == null || toSnapshot.isEmpty()) ? path : Snapshot.getSnapshotPath(path, toSnapshot);
    readLock();
    try {
        checkOperation(OperationCategory.READ);
        diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager, path, fromSnapshot, toSnapshot);
        success = true;
    } catch (AccessControlException ace) {
        logAuditEvent(success, operationName, fromSnapshotRoot, toSnapshotRoot, null);
        throw ace;
    } finally {
        readUnlock(operationName);
    }
    logAuditEvent(success, operationName, fromSnapshotRoot, toSnapshotRoot, null);
    return diffs;
}