Examples with RecompactionConditionFactory org.apache.gobblin.compaction.conditions.RecompactionConditionFactory used on opensource projects

Example 1 with RecompactionConditionFactory

use of org.apache.gobblin.compaction.conditions.RecompactionConditionFactory in project incubator-gobblin by apache.

the class DatasetHelper method createRecompactionCondition.

private RecompactionCondition createRecompactionCondition() {
    ClassAliasResolver<RecompactionConditionFactory> conditionClassAliasResolver = new ClassAliasResolver<>(RecompactionConditionFactory.class);
    String factoryName = this.dataset.jobProps().getProp(MRCompactor.COMPACTION_RECOMPACT_CONDITION, MRCompactor.DEFAULT_COMPACTION_RECOMPACT_CONDITION);
    try {
        RecompactionConditionFactory factory = GobblinConstructorUtils.invokeFirstConstructor(conditionClassAliasResolver.resolveClass(factoryName), ImmutableList.of());
        return factory.createRecompactionCondition(dataset);
    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) {
        throw new IllegalArgumentException(e);
    }
}

Example 2 with RecompactionConditionFactory

use of org.apache.gobblin.compaction.conditions.RecompactionConditionFactory in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnFileCount.

@Test
public void testRecompactionConditionBasedOnFileCount() {
    try {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        fs.delete(outputLatePath, true);
        fs.mkdirs(outputLatePath);
        RecompactionConditionFactory factory = new RecompactionConditionBasedOnFileCount.Factory();
        RecompactionCondition conditionBasedOnFileCount = factory.createRecompactionCondition(dataset);
        DatasetHelper helper = new DatasetHelper(dataset, fs, Lists.newArrayList("avro"));
        fs.createNewFile(new Path(outputLatePath, new Path("1.avro")));
        fs.createNewFile(new Path(outputLatePath, new Path("2.avro")));
        Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), false);
        fs.createNewFile(new Path(outputLatePath, new Path("3.avro")));
        Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), true);
        fs.delete(outputLatePath, true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 3 with RecompactionConditionFactory

use of org.apache.gobblin.compaction.conditions.RecompactionConditionFactory in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnRatio.

@Test
public void testRecompactionConditionBasedOnRatio() {
    RecompactionConditionFactory factory = new RecompactionConditionBasedOnRatio.Factory();
    RecompactionCondition conditionBasedOnRatio = factory.createRecompactionCondition(dataset);
    DatasetHelper helper = mock(DatasetHelper.class);
    when(helper.getLateOutputRecordCount()).thenReturn(6L);
    when(helper.getOutputRecordCount()).thenReturn(94L);
    Assert.assertEquals(conditionBasedOnRatio.isRecompactionNeeded(helper), false);
    when(helper.getLateOutputRecordCount()).thenReturn(21L);
    when(helper.getOutputRecordCount()).thenReturn(79L);
    Assert.assertEquals(conditionBasedOnRatio.isRecompactionNeeded(helper), true);
}

Example 4 with RecompactionConditionFactory

use of org.apache.gobblin.compaction.conditions.RecompactionConditionFactory in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnDuration.

@Test
public void testRecompactionConditionBasedOnDuration() {
    RecompactionConditionFactory factory = new RecompactionConditionBasedOnDuration.Factory();
    RecompactionCondition conditionBasedOnDuration = factory.createRecompactionCondition(dataset);
    DatasetHelper helper = mock(DatasetHelper.class);
    when(helper.getDataset()).thenReturn(dataset);
    PeriodFormatter periodFormatter = new PeriodFormatterBuilder().appendMonths().appendSuffix("m").appendDays().appendSuffix("d").appendHours().appendSuffix("h").appendMinutes().appendSuffix("min").toFormatter();
    DateTime currentTime = getCurrentTime();
    Period period_A = periodFormatter.parsePeriod("11h59min");
    DateTime earliest_A = currentTime.minus(period_A);
    when(helper.getEarliestLateFileModificationTime()).thenReturn(Optional.of(earliest_A));
    when(helper.getCurrentTime()).thenReturn(currentTime);
    Assert.assertEquals(conditionBasedOnDuration.isRecompactionNeeded(helper), false);
    Period period_B = periodFormatter.parsePeriod("12h01min");
    DateTime earliest_B = currentTime.minus(period_B);
    when(helper.getEarliestLateFileModificationTime()).thenReturn(Optional.of(earliest_B));
    when(helper.getCurrentTime()).thenReturn(currentTime);
    Assert.assertEquals(conditionBasedOnDuration.isRecompactionNeeded(helper), true);
}