Examples with FsCleanableHelper org.apache.gobblin.data.management.retention.dataset.FsCleanableHelper used on opensource projects

Example 1 with FsCleanableHelper

use of org.apache.gobblin.data.management.retention.dataset.FsCleanableHelper in project incubator-gobblin by apache.

the class FsCleanableHelperTest method testDeleteEmptyDirs.

@Test
public void testDeleteEmptyDirs() throws Exception {
    Properties props = new Properties();
    props.setProperty(FsCleanableHelper.SKIP_TRASH_KEY, Boolean.toString(true));
    FsCleanableHelper fsCleanableHelper = new FsCleanableHelper(this.fs, props, ConfigFactory.empty(), log);
    FileSystemDataset fsDataset = mock(FileSystemDataset.class);
    Path datasetRoot = new Path(testTempPath, "dataset1");
    when(fsDataset.datasetRoot()).thenReturn(datasetRoot);
    // To delete
    Path deleted1 = new Path(datasetRoot, "2016/01/01/13");
    Path deleted2 = new Path(datasetRoot, "2016/01/01/14");
    Path deleted3 = new Path(datasetRoot, "2016/01/02/15");
    // Do not delete
    Path notDeleted1 = new Path(datasetRoot, "2016/01/02/16");
    this.fs.mkdirs(deleted1);
    this.fs.mkdirs(deleted2);
    this.fs.mkdirs(deleted3);
    this.fs.mkdirs(notDeleted1);
    // Make sure all paths are created
    Assert.assertTrue(this.fs.exists(deleted1));
    Assert.assertTrue(this.fs.exists(deleted2));
    Assert.assertTrue(this.fs.exists(deleted3));
    Assert.assertTrue(this.fs.exists(notDeleted1));
    List<FileSystemDatasetVersion> deletableVersions = ImmutableList.<FileSystemDatasetVersion>of(new MockFileSystemDatasetVersion(deleted1), new MockFileSystemDatasetVersion(deleted2), new MockFileSystemDatasetVersion(deleted3));
    fsCleanableHelper.clean(deletableVersions, fsDataset);
    // Verify versions are deleted
    Assert.assertFalse(this.fs.exists(deleted1));
    Assert.assertFalse(this.fs.exists(deleted2));
    Assert.assertFalse(this.fs.exists(deleted3));
    // Verify versions are not deleted
    Assert.assertTrue(this.fs.exists(notDeleted1));
    // Verify empty parent dir "2016/01/01" is deleted
    Assert.assertFalse(this.fs.exists(deleted1.getParent()));
    // Verify non empty parent dir "2016/01/02" exists
    Assert.assertTrue(this.fs.exists(notDeleted1.getParent()));
}

Example 2 with FsCleanableHelper

use of org.apache.gobblin.data.management.retention.dataset.FsCleanableHelper in project incubator-gobblin by apache.

the class FsCleanableHelperTest method testSimulateTrash.

@Test
public void testSimulateTrash() throws Exception {
    Properties props = new Properties();
    props.setProperty(FsCleanableHelper.SIMULATE_KEY, Boolean.toString(true));
    FsCleanableHelper fsCleanableHelper = new FsCleanableHelper(this.fs, props, ConfigFactory.empty(), log);
    assertThat(fsCleanableHelper.getTrash(), instanceOf(MockTrash.class));
}