Examples with FileSystemDatasetVersion org.apache.gobblin.data.management.version.FileSystemDatasetVersion used on opensource projects

Example 6 with FileSystemDatasetVersion

use of org.apache.gobblin.data.management.version.FileSystemDatasetVersion in project incubator-gobblin by apache.

the class ConfigurableCleanableDatasetTest method testConfigureWithMulitplePolicies.

@Test
public void testConfigureWithMulitplePolicies() throws Exception {
    Map<String, String> partitionConf = ImmutableMap.<String, String>of("version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "selection.policy.class", "org.apache.gobblin.data.management.policy.NewestKSelectionPolicy", "selection.newestK.versionsSelected", "2");
    Config conf = ConfigFactory.parseMap(ImmutableMap.<String, List<Map<String, String>>>of("gobblin.retention.dataset.partitions", ImmutableList.of(partitionConf, partitionConf)));
    ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class));
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class);
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class);
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(1).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class);
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(1).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class);
    Assert.assertEquals(dataset.isDatasetBlacklisted(), false);
}

Example 7 with FileSystemDatasetVersion

use of org.apache.gobblin.data.management.version.FileSystemDatasetVersion in project incubator-gobblin by apache.

the class ConfigurableCleanableDatasetTest method testConfigureWithSelectionPolicy.

@Test
public void testConfigureWithSelectionPolicy() throws Exception {
    Config conf = ConfigFactory.parseMap(ImmutableMap.<String, String>of("gobblin.retention.version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "gobblin.retention.selection.policy.class", "org.apache.gobblin.data.management.policy.NewestKSelectionPolicy", "gobblin.retention.selection.newestK.versionsSelected", "2"));
    ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class));
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), NewestKSelectionPolicy.class);
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class);
    Assert.assertEquals(dataset.isDatasetBlacklisted(), false);
}

Example 8 with FileSystemDatasetVersion

use of org.apache.gobblin.data.management.version.FileSystemDatasetVersion in project incubator-gobblin by apache.

the class ConfigurableCleanableDatasetTest method testConfigureWithRetentionPolicy.

@Test
public void testConfigureWithRetentionPolicy() throws Exception {
    Config conf = ConfigFactory.parseMap(ImmutableMap.<String, String>of("gobblin.retention.version.finder.class", "org.apache.gobblin.data.management.version.finder.WatermarkDatasetVersionFinder", "gobblin.retention.retention.policy.class", "org.apache.gobblin.data.management.retention.policy.NewestKRetentionPolicy", "gobblin.retention.newestK.versions.retained", "2"));
    ConfigurableCleanableDataset<FileSystemDatasetVersion> dataset = new ConfigurableCleanableDataset<FileSystemDatasetVersion>(FileSystem.get(new URI(ConfigurationKeys.LOCAL_FS_URI), new Configuration()), new Properties(), new Path("/someroot"), conf, LoggerFactory.getLogger(ConfigurableCleanableDatasetTest.class));
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionSelectionPolicy().getClass(), EmbeddedRetentionSelectionPolicy.class);
    Assert.assertEquals(dataset.getVersionFindersAndPolicies().get(0).getVersionFinder().getClass(), WatermarkDatasetVersionFinder.class);
    Assert.assertEquals(dataset.isDatasetBlacklisted(), false);
}

Example 9 with FileSystemDatasetVersion

use of org.apache.gobblin.data.management.version.FileSystemDatasetVersion in project incubator-gobblin by apache.

the class FsCleanableHelper method clean.

/**
 * Delete all {@link FileSystemDatasetVersion}s <code>deletableVersions</code> and also delete any empty parent directories.
 *
 * @param fsDataset to which the version belongs.
 */
public void clean(final Collection<? extends FileSystemDatasetVersion> deletableVersions, final FileSystemDataset fsDataset) throws IOException {
    if (deletableVersions.isEmpty()) {
        log.warn("No deletable dataset version can be found. Ignoring.");
        return;
    }
    Set<Path> possiblyEmptyDirectories = new HashSet<>();
    for (FileSystemDatasetVersion fsdv : deletableVersions) {
        clean(fsdv, possiblyEmptyDirectories);
    }
    cleanEmptyDirectories(possiblyEmptyDirectories, fsDataset);
}