Examples with DirectoryDeletingService org.apache.hadoop.ozone.om.DirectoryDeletingService used on opensource projects

Example 1 with DirectoryDeletingService

use of org.apache.hadoop.ozone.om.DirectoryDeletingService in project ozone by apache.

the class TestDirectoryDeletingServiceWithFSO method testDeleteWithLargeSubPathsThanBatchSize.

/**
 * Tests verifies that directories and files are getting purged in multiple
 * batches.
 */
@Test
public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception {
    Path root = new Path("/rootDir");
    Path appRoot = new Path(root, "appRoot");
    // Creates 2 parent dirs from root.
    fs.mkdirs(appRoot);
    // Total of (3 * 5) childFiles
    for (int i = 1; i <= 3; i++) {
        Path childDir = new Path(appRoot, "parentDir" + i);
        for (int j = 1; j <= 5; j++) {
            // total 5 sub-dirs + 5 sub-files = 10 items in this level.
            Path childSubDir = new Path(childDir, "childDir" + j);
            Path childSubFile = new Path(childDir, "childFile" + j);
            // create sub file
            ContractTestUtils.touch(fs, childSubFile);
            // create sub dir
            fs.mkdirs(childSubDir);
        }
    }
    Table<String, OmKeyInfo> deletedDirTable = cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
    Table<String, OmKeyInfo> keyTable = cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout());
    Table<String, OmDirectoryInfo> dirTable = cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
    DirectoryDeletingService dirDeletingService = (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager().getDirDeletingService();
    // Before delete
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(keyTable, 15);
    assertTableRowCount(dirTable, 20);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
    // Delete the appRoot
    fs.delete(appRoot, true);
    // After Delete
    checkPath(appRoot);
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(keyTable, 0);
    assertTableRowCount(dirTable, 1);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 15);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 19);
    Assert.assertTrue(dirDeletingService.getRunCount() > 1);
}

Example 2 with DirectoryDeletingService

use of org.apache.hadoop.ozone.om.DirectoryDeletingService in project ozone by apache.

the class TestDirectoryDeletingServiceWithFSO method testDeleteFilesAndSubFiles.

@Test
public void testDeleteFilesAndSubFiles() throws Exception {
    Table<String, OmKeyInfo> deletedDirTable = cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
    Table<String, OmKeyInfo> keyTable = cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout());
    Table<String, OmDirectoryInfo> dirTable = cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
    Table<String, RepeatedOmKeyInfo> deletedKeyTable = cluster.getOzoneManager().getMetadataManager().getDeletedTable();
    Path root = new Path("/rootDir2");
    // Create  parent dir from root.
    fs.mkdirs(root);
    // Added 10 sub files inside root dir
    for (int i = 0; i < 5; i++) {
        Path path = new Path(root, "testKey" + i);
        try (FSDataOutputStream stream = fs.create(path)) {
            stream.write(1);
        }
    }
    KeyDeletingService keyDeletingService = (KeyDeletingService) cluster.getOzoneManager().getKeyManager().getDeletingService();
    // Before delete
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(keyTable, 5);
    assertTableRowCount(dirTable, 1);
    long prevDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
    // Case-1) Delete 3 Files directly.
    for (int i = 0; i < 3; i++) {
        Path path = new Path(root, "testKey" + i);
        fs.delete(path, true);
    }
    DirectoryDeletingService dirDeletingService = (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager().getDirDeletingService();
    // After delete. 2 more files left out under the root dir
    assertTableRowCount(keyTable, 2);
    assertTableRowCount(dirTable, 1);
    // Eventually keys would get cleaned up from deletedTables too
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(deletedKeyTable, 0);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
    // verify whether KeyDeletingService has purged the keys
    long currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
    Assert.assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount);
    // Case-2) Delete dir, this will cleanup sub-files under the deleted dir.
    fs.delete(root, true);
    // After delete. 2 sub files to be deleted.
    assertTableRowCount(keyTable, 0);
    assertTableRowCount(dirTable, 0);
    // Eventually keys would get cleaned up from deletedTables too
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(deletedKeyTable, 0);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 2);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 1);
    // verify whether KeyDeletingService has purged the keys
    currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
    Assert.assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount);
}

Example 3 with DirectoryDeletingService

use of org.apache.hadoop.ozone.om.DirectoryDeletingService in project ozone by apache.

the class TestDirectoryDeletingServiceWithFSO method testDeleteEmptyDirectory.

@Test
public void testDeleteEmptyDirectory() throws Exception {
    Path root = new Path("/rootDir");
    Path appRoot = new Path(root, "appRoot");
    fs.mkdirs(appRoot);
    Table<String, OmKeyInfo> deletedDirTable = cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
    Table<String, OmDirectoryInfo> dirTable = cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
    DirectoryDeletingService dirDeletingService = (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager().getDirDeletingService();
    // Before delete
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(dirTable, 2);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
    // Delete the appRoot, empty dir
    fs.delete(appRoot, true);
    // After Delete
    checkPath(appRoot);
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(dirTable, 1);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 1);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
    Assert.assertTrue(dirTable.iterator().hasNext());
    Assert.assertEquals(root.getName(), dirTable.iterator().next().getValue().getName());
    Assert.assertTrue(dirDeletingService.getRunCount() > 1);
}

Example 4 with DirectoryDeletingService

use of org.apache.hadoop.ozone.om.DirectoryDeletingService in project ozone by apache.

the class TestDirectoryDeletingServiceWithFSO method testDeleteWithMultiLevels.

@Test
public void testDeleteWithMultiLevels() throws Exception {
    Path root = new Path("/rootDir");
    Path appRoot = new Path(root, "appRoot");
    for (int i = 1; i <= 3; i++) {
        Path parent = new Path(appRoot, "parentDir" + i);
        Path child = new Path(parent, "childFile");
        ContractTestUtils.touch(fs, child);
    }
    Table<String, OmKeyInfo> deletedDirTable = cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
    Table<String, OmKeyInfo> keyTable = cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout());
    Table<String, OmDirectoryInfo> dirTable = cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
    DirectoryDeletingService dirDeletingService = (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager().getDirDeletingService();
    // Before delete
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(dirTable, 5);
    assertTableRowCount(keyTable, 3);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 0);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 0);
    // Delete the rootDir, which should delete all keys.
    fs.delete(root, true);
    // After Delete
    checkPath(root);
    assertTableRowCount(deletedDirTable, 0);
    assertTableRowCount(keyTable, 0);
    assertTableRowCount(dirTable, 0);
    assertSubPathsCount(dirDeletingService.getMovedFilesCount(), 3);
    assertSubPathsCount(dirDeletingService.getDeletedDirsCount(), 5);
    Assert.assertTrue(dirDeletingService.getRunCount() > 1);
}