Examples with DirectoryDiff org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff used on opensource projects

Example 1 with DirectoryDiff

use of org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff in project hadoop by apache.

the class TestFSImageWithSnapshot method testSnapshotOnRoot.

/**
   * Test when there is snapshot taken on root
   */
@Test
public void testSnapshotOnRoot() throws Exception {
    final Path root = new Path("/");
    hdfs.allowSnapshot(root);
    hdfs.createSnapshot(root, "s1");
    cluster.shutdown();
    cluster = new MiniDFSCluster.Builder(conf).format(false).numDataNodes(NUM_DATANODES).build();
    cluster.waitActive();
    fsn = cluster.getNamesystem();
    hdfs = cluster.getFileSystem();
    // save namespace and restart cluster
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    hdfs.saveNamespace();
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    cluster.shutdown();
    cluster = new MiniDFSCluster.Builder(conf).format(false).numDataNodes(NUM_DATANODES).build();
    cluster.waitActive();
    fsn = cluster.getNamesystem();
    hdfs = cluster.getFileSystem();
    INodeDirectory rootNode = fsn.dir.getINode4Write(root.toString()).asDirectory();
    assertTrue("The children list of root should be empty", rootNode.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
    // one snapshot on root: s1
    List<DirectoryDiff> diffList = rootNode.getDiffs().asList();
    assertEquals(1, diffList.size());
    Snapshot s1 = rootNode.getSnapshot(DFSUtil.string2Bytes("s1"));
    assertEquals(s1.getId(), diffList.get(0).getSnapshotId());
    // check SnapshotManager's snapshottable directory list
    assertEquals(1, fsn.getSnapshotManager().getNumSnapshottableDirs());
    SnapshottableDirectoryStatus[] sdirs = fsn.getSnapshotManager().getSnapshottableDirListing(null);
    assertEquals(root, sdirs[0].getFullPath());
    // save namespace and restart cluster
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    hdfs.saveNamespace();
    hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    cluster.shutdown();
    cluster = new MiniDFSCluster.Builder(conf).format(false).numDataNodes(NUM_DATANODES).build();
    cluster.waitActive();
    fsn = cluster.getNamesystem();
    hdfs = cluster.getFileSystem();
}

Example 2 with DirectoryDiff

use of org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff in project hadoop by apache.

the class TestRenameWithSnapshots method testCleanDstReference.

/**
   * Make sure we clean the whole subtree under a DstReference node after 
   * deleting a snapshot.
   * see HDFS-5476.
   */
@Test
public void testCleanDstReference() throws Exception {
    final Path test = new Path("/test");
    final Path foo = new Path(test, "foo");
    final Path bar = new Path(foo, "bar");
    hdfs.mkdirs(bar);
    SnapshotTestHelper.createSnapshot(hdfs, test, "s0");
    // create file after s0 so that the file should not be included in s0
    final Path fileInBar = new Path(bar, "file");
    DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPL, SEED);
    // rename foo --> foo2
    final Path foo2 = new Path(test, "foo2");
    hdfs.rename(foo, foo2);
    // create snapshot s1, note the file is included in s1
    hdfs.createSnapshot(test, "s1");
    // delete bar and foo2
    hdfs.delete(new Path(foo2, "bar"), true);
    hdfs.delete(foo2, true);
    final Path sfileInBar = SnapshotTestHelper.getSnapshotPath(test, "s1", "foo2/bar/file");
    assertTrue(hdfs.exists(sfileInBar));
    hdfs.deleteSnapshot(test, "s1");
    assertFalse(hdfs.exists(sfileInBar));
    restartClusterAndCheckImage(true);
    // make sure the file under bar is deleted 
    final Path barInS0 = SnapshotTestHelper.getSnapshotPath(test, "s0", "foo/bar");
    INodeDirectory barNode = fsdir.getINode(barInS0.toString()).asDirectory();
    assertEquals(0, barNode.getChildrenList(Snapshot.CURRENT_STATE_ID).size());
    List<DirectoryDiff> diffList = barNode.getDiffs().asList();
    assertEquals(1, diffList.size());
    DirectoryDiff diff = diffList.get(0);
    assertEquals(0, diff.getChildrenDiff().getList(ListType.DELETED).size());
    assertEquals(0, diff.getChildrenDiff().getList(ListType.CREATED).size());
}

Example 3 with DirectoryDiff

use of org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff in project hadoop by apache.

the class TestRenameWithSnapshots method testRenameDirAndDeleteSnapshot_4.

/**
   * After the following operations:
   * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir
   * again -> delete snapshot s on dst tree
   * 
   * Make sure we only delete the snapshot s under the renamed dir.
   */
@Test
public void testRenameDirAndDeleteSnapshot_4() throws Exception {
    final Path sdir1 = new Path("/dir1");
    final Path sdir2 = new Path("/dir2");
    final Path foo = new Path(sdir1, "foo");
    final Path bar = new Path(foo, "bar");
    DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED);
    hdfs.mkdirs(sdir2);
    SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1");
    SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2");
    final Path foo2 = new Path(sdir2, "foo");
    hdfs.rename(foo, foo2);
    // create two new files under foo2
    final Path bar2 = new Path(foo2, "bar2");
    DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED);
    final Path bar3 = new Path(foo2, "bar3");
    DFSTestUtil.createFile(hdfs, bar3, BLOCKSIZE, REPL, SEED);
    // create a new snapshot on sdir2
    hdfs.createSnapshot(sdir2, "s3");
    // rename foo2 again
    hdfs.rename(foo2, foo);
    // delete snapshot s3
    hdfs.deleteSnapshot(sdir2, "s3");
    // check
    final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()).asDirectory();
    // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3)
    QuotaCounts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
    assertEquals(7, q1.getNameSpace());
    final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()).asDirectory();
    QuotaCounts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
    assertEquals(1, q2.getNameSpace());
    final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", foo.getName());
    final INode fooRef = fsdir.getINode(foo_s1.toString());
    assertTrue(fooRef instanceof INodeReference.WithName);
    INodeReference.WithCount wc = (WithCount) fooRef.asReference().getReferredINode();
    assertEquals(2, wc.getReferenceCount());
    INodeDirectory fooNode = wc.getReferredINode().asDirectory();
    ReadOnlyList<INode> children = fooNode.getChildrenList(Snapshot.CURRENT_STATE_ID);
    assertEquals(3, children.size());
    assertEquals(bar.getName(), children.get(0).getLocalName());
    assertEquals(bar2.getName(), children.get(1).getLocalName());
    assertEquals(bar3.getName(), children.get(2).getLocalName());
    List<DirectoryDiff> diffList = fooNode.getDiffs().asList();
    assertEquals(1, diffList.size());
    Snapshot s1 = dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
    assertEquals(s1.getId(), diffList.get(0).getSnapshotId());
    ChildrenDiff diff = diffList.get(0).getChildrenDiff();
    // bar2 and bar3 in the created list
    assertEquals(2, diff.getList(ListType.CREATED).size());
    assertEquals(0, diff.getList(ListType.DELETED).size());
    final INode fooRef2 = fsdir.getINode4Write(foo.toString());
    assertTrue(fooRef2 instanceof INodeReference.DstReference);
    INodeReference.WithCount wc2 = (WithCount) fooRef2.asReference().getReferredINode();
    assertSame(wc, wc2);
    assertSame(fooRef2, wc.getParentReference());
    restartClusterAndCheckImage(true);
}

Example 4 with DirectoryDiff

use of org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff in project hadoop by apache.

the class TestSetQuotaWithSnapshot method testClearQuota.

/**
   * Test clear quota of a snapshottable dir or a dir with snapshot.
   */
@Test
public void testClearQuota() throws Exception {
    final Path dir = new Path("/TestSnapshot");
    hdfs.mkdirs(dir);
    hdfs.allowSnapshot(dir);
    hdfs.setQuota(dir, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_DONT_SET);
    INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
    assertTrue(dirNode.isSnapshottable());
    assertEquals(0, dirNode.getDiffs().asList().size());
    hdfs.setQuota(dir, HdfsConstants.QUOTA_DONT_SET - 1, HdfsConstants.QUOTA_DONT_SET - 1);
    dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
    assertTrue(dirNode.isSnapshottable());
    assertEquals(0, dirNode.getDiffs().asList().size());
    hdfs.setQuota(dir, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_RESET);
    dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
    assertTrue(dirNode.isSnapshottable());
    assertEquals(0, dirNode.getDiffs().asList().size());
    // allow snapshot on dir and create snapshot s1
    SnapshotTestHelper.createSnapshot(hdfs, dir, "s1");
    // clear quota of dir
    hdfs.setQuota(dir, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_RESET);
    // dir should still be a snapshottable directory
    dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
    assertTrue(dirNode.isSnapshottable());
    assertEquals(1, dirNode.getDiffs().asList().size());
    SnapshottableDirectoryStatus[] status = hdfs.getSnapshottableDirListing();
    assertEquals(1, status.length);
    assertEquals(dir, status[0].getFullPath());
    final Path subDir = new Path(dir, "sub");
    hdfs.mkdirs(subDir);
    hdfs.createSnapshot(dir, "s2");
    final Path file = new Path(subDir, "file");
    DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, seed);
    hdfs.setQuota(dir, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_RESET);
    INode subNode = fsdir.getINode4Write(subDir.toString());
    assertTrue(subNode.asDirectory().isWithSnapshot());
    List<DirectoryDiff> diffList = subNode.asDirectory().getDiffs().asList();
    assertEquals(1, diffList.size());
    Snapshot s2 = dirNode.getSnapshot(DFSUtil.string2Bytes("s2"));
    assertEquals(s2.getId(), diffList.get(0).getSnapshotId());
    List<INode> createdList = diffList.get(0).getChildrenDiff().getList(ListType.CREATED);
    assertEquals(1, createdList.size());
    assertSame(fsdir.getINode4Write(file.toString()), createdList.get(0));
}

Example 5 with DirectoryDiff

use of org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff in project hadoop by apache.

the class TestRenameWithSnapshots method testRenameDirAndDeleteSnapshot_2.

/**
   * After rename, delete the snapshot in src
   */
@Test
public void testRenameDirAndDeleteSnapshot_2() throws Exception {
    final Path sdir1 = new Path("/dir1");
    final Path sdir2 = new Path("/dir2");
    hdfs.mkdirs(sdir1);
    hdfs.mkdirs(sdir2);
    final Path foo = new Path(sdir2, "foo");
    final Path bar = new Path(foo, "bar");
    DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPL, SEED);
    SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1");
    SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2");
    SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s3");
    final Path newfoo = new Path(sdir1, "foo");
    hdfs.rename(foo, newfoo);
    // restart the cluster and check fsimage
    restartClusterAndCheckImage(true);
    final Path bar2 = new Path(newfoo, "bar2");
    DFSTestUtil.createFile(hdfs, bar2, BLOCKSIZE, REPL, SEED);
    hdfs.createSnapshot(sdir1, "s4");
    hdfs.delete(newfoo, true);
    final Path bar2_s4 = SnapshotTestHelper.getSnapshotPath(sdir1, "s4", "foo/bar2");
    assertTrue(hdfs.exists(bar2_s4));
    final Path bar_s4 = SnapshotTestHelper.getSnapshotPath(sdir1, "s4", "foo/bar");
    assertTrue(hdfs.exists(bar_s4));
    // delete snapshot s4. The diff of s4 should be combined to s3
    hdfs.deleteSnapshot(sdir1, "s4");
    // restart the cluster and check fsimage
    restartClusterAndCheckImage(true);
    Path bar_s3 = SnapshotTestHelper.getSnapshotPath(sdir1, "s3", "foo/bar");
    assertFalse(hdfs.exists(bar_s3));
    bar_s3 = SnapshotTestHelper.getSnapshotPath(sdir2, "s3", "foo/bar");
    assertTrue(hdfs.exists(bar_s3));
    Path bar2_s3 = SnapshotTestHelper.getSnapshotPath(sdir1, "s3", "foo/bar2");
    assertFalse(hdfs.exists(bar2_s3));
    bar2_s3 = SnapshotTestHelper.getSnapshotPath(sdir2, "s3", "foo/bar2");
    assertFalse(hdfs.exists(bar2_s3));
    // delete snapshot s3
    hdfs.deleteSnapshot(sdir2, "s3");
    final Path bar_s2 = SnapshotTestHelper.getSnapshotPath(sdir2, "s2", "foo/bar");
    assertTrue(hdfs.exists(bar_s2));
    // check internal details
    INodeDirectory sdir2Node = fsdir.getINode(sdir2.toString()).asDirectory();
    Snapshot s2 = sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
    final Path foo_s2 = SnapshotTestHelper.getSnapshotPath(sdir2, "s2", "foo");
    INodeReference fooRef = fsdir.getINode(foo_s2.toString()).asReference();
    assertTrue(fooRef instanceof INodeReference.WithName);
    INodeReference.WithCount fooWC = (WithCount) fooRef.getReferredINode();
    assertEquals(1, fooWC.getReferenceCount());
    INodeDirectory fooDir = fooWC.getReferredINode().asDirectory();
    List<DirectoryDiff> diffs = fooDir.getDiffs().asList();
    assertEquals(1, diffs.size());
    assertEquals(s2.getId(), diffs.get(0).getSnapshotId());
    // restart the cluster and check fsimage
    restartClusterAndCheckImage(true);
    // delete snapshot s2.
    hdfs.deleteSnapshot(sdir2, "s2");
    assertFalse(hdfs.exists(bar_s2));
    restartClusterAndCheckImage(true);
    // make sure the whole referred subtree has been destroyed
    QuotaCounts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
    assertEquals(3, q.getNameSpace());
    assertEquals(0, q.getStorageSpace());
    hdfs.deleteSnapshot(sdir1, "s1");
    restartClusterAndCheckImage(true);
    q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
    assertEquals(3, q.getNameSpace());
    assertEquals(0, q.getStorageSpace());
}