Examples with CombineRetentionPolicy org.apache.gobblin.data.management.retention.policy.CombineRetentionPolicy used on opensource projects

Example 1 with CombineRetentionPolicy

use of org.apache.gobblin.data.management.retention.policy.CombineRetentionPolicy in project incubator-gobblin by apache.

the class CombineRetentionPolicyTest method testCommonSuperclass.

@Test
public void testCommonSuperclass() throws IOException {
    Properties props = new Properties();
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "1", ContainsARetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.DELETE_SETS_COMBINE_OPERATION, CombineRetentionPolicy.DeletableCombineOperation.INTERSECT.name());
    CombineRetentionPolicy policy = new CombineRetentionPolicy(props);
    Assert.assertEquals(policy.commonSuperclass(StringDatasetVersion.class, StringDatasetVersion.class), StringDatasetVersion.class);
    Assert.assertEquals(policy.commonSuperclass(StringDatasetVersion.class, TimestampedDatasetVersion.class), DatasetVersion.class);
    Assert.assertEquals(policy.commonSuperclass(StringDatasetVersion.class, FileStatusDatasetVersion.class), StringDatasetVersion.class);
    Assert.assertEquals(policy.commonSuperclass(FileStatusDatasetVersion.class, StringDatasetVersion.class), StringDatasetVersion.class);
    Assert.assertEquals(policy.commonSuperclass(DatasetVersion.class, StringDatasetVersion.class), DatasetVersion.class);
}

Example 2 with CombineRetentionPolicy

use of org.apache.gobblin.data.management.retention.policy.CombineRetentionPolicy in project incubator-gobblin by apache.

the class CombineRetentionPolicyTest method testIntersect.

@Test
public void testIntersect() throws IOException {
    Properties props = new Properties();
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "1", ContainsARetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "2", ContainsBRetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "3", ContainsCRetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.DELETE_SETS_COMBINE_OPERATION, CombineRetentionPolicy.DeletableCombineOperation.INTERSECT.name());
    CombineRetentionPolicy policy = new CombineRetentionPolicy(props);
    Collection<DatasetVersion> deletableVersions = policy.listDeletableVersions(Lists.<DatasetVersion>newArrayList(new StringDatasetVersion("a", new Path("/")), new StringDatasetVersion("abc", new Path("/")), new StringDatasetVersion("abcd", new Path("/")), new StringDatasetVersion("bc", new Path("/")), new StringDatasetVersion("d", new Path("/"))));
    Set<String> actualDeletableVersions = Sets.newHashSet(Iterables.transform(deletableVersions, new Function<DatasetVersion, String>() {

        @Nullable
        @Override
        public String apply(DatasetVersion input) {
            return ((StringDatasetVersion) input).getVersion();
        }
    }));
    Assert.assertEquals(policy.versionClass(), StringDatasetVersion.class);
    Assert.assertEquals(deletableVersions.size(), 2);
    Assert.assertEquals(actualDeletableVersions, Sets.newHashSet("abcd", "abc"));
}

Example 3 with CombineRetentionPolicy

use of org.apache.gobblin.data.management.retention.policy.CombineRetentionPolicy in project incubator-gobblin by apache.

the class CombineRetentionPolicyTest method testUnion.

@Test
public void testUnion() throws IOException {
    Properties props = new Properties();
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "1", ContainsARetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "2", ContainsBRetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.RETENTION_POLICIES_PREFIX + "3", ContainsCRetentionPolicy.class.getCanonicalName());
    props.setProperty(CombineRetentionPolicy.DELETE_SETS_COMBINE_OPERATION, CombineRetentionPolicy.DeletableCombineOperation.UNION.name());
    CombineRetentionPolicy policy = new CombineRetentionPolicy(props);
    Collection<DatasetVersion> deletableVersions = policy.listDeletableVersions(Lists.<DatasetVersion>newArrayList(new StringDatasetVersion("a", new Path("/")), new StringDatasetVersion("abc", new Path("/")), new StringDatasetVersion("abcd", new Path("/")), new StringDatasetVersion("bc", new Path("/")), new StringDatasetVersion("d", new Path("/"))));
    Set<String> actualDeletableVersions = Sets.newHashSet(Iterables.transform(deletableVersions, new Function<DatasetVersion, String>() {

        @Nullable
        @Override
        public String apply(DatasetVersion input) {
            return ((StringDatasetVersion) input).getVersion();
        }
    }));
    Assert.assertEquals(deletableVersions.size(), 4);
    Assert.assertEquals(actualDeletableVersions, Sets.newHashSet("abcd", "abc", "a", "bc"));
}