Examples with RecompactionCondition org.apache.gobblin.compaction.conditions.RecompactionCondition used on opensource projects

Example 1 with RecompactionCondition

use of org.apache.gobblin.compaction.conditions.RecompactionCondition in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnFileCount.

@Test
public void testRecompactionConditionBasedOnFileCount() {
    try {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);
        fs.delete(outputLatePath, true);
        fs.mkdirs(outputLatePath);
        RecompactionConditionFactory factory = new RecompactionConditionBasedOnFileCount.Factory();
        RecompactionCondition conditionBasedOnFileCount = factory.createRecompactionCondition(dataset);
        DatasetHelper helper = new DatasetHelper(dataset, fs, Lists.newArrayList("avro"));
        fs.createNewFile(new Path(outputLatePath, new Path("1.avro")));
        fs.createNewFile(new Path(outputLatePath, new Path("2.avro")));
        Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), false);
        fs.createNewFile(new Path(outputLatePath, new Path("3.avro")));
        Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), true);
        fs.delete(outputLatePath, true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with RecompactionCondition

use of org.apache.gobblin.compaction.conditions.RecompactionCondition in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnRatio.

@Test
public void testRecompactionConditionBasedOnRatio() {
    RecompactionConditionFactory factory = new RecompactionConditionBasedOnRatio.Factory();
    RecompactionCondition conditionBasedOnRatio = factory.createRecompactionCondition(dataset);
    DatasetHelper helper = mock(DatasetHelper.class);
    when(helper.getLateOutputRecordCount()).thenReturn(6L);
    when(helper.getOutputRecordCount()).thenReturn(94L);
    Assert.assertEquals(conditionBasedOnRatio.isRecompactionNeeded(helper), false);
    when(helper.getLateOutputRecordCount()).thenReturn(21L);
    when(helper.getOutputRecordCount()).thenReturn(79L);
    Assert.assertEquals(conditionBasedOnRatio.isRecompactionNeeded(helper), true);
}

Example 3 with RecompactionCondition

use of org.apache.gobblin.compaction.conditions.RecompactionCondition in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionCombineCondition.

@Test
public void testRecompactionCombineCondition() {
    DatasetHelper helper = mock(DatasetHelper.class);
    RecompactionCondition cond1 = mock(RecompactionConditionBasedOnRatio.class);
    RecompactionCondition cond2 = mock(RecompactionConditionBasedOnFileCount.class);
    RecompactionCondition cond3 = mock(RecompactionConditionBasedOnDuration.class);
    RecompactionCombineCondition combineConditionOr = new RecompactionCombineCondition(Arrays.asList(cond1, cond2, cond3), RecompactionCombineCondition.CombineOperation.OR);
    when(cond1.isRecompactionNeeded(helper)).thenReturn(false);
    when(cond2.isRecompactionNeeded(helper)).thenReturn(false);
    when(cond3.isRecompactionNeeded(helper)).thenReturn(false);
    Assert.assertEquals(combineConditionOr.isRecompactionNeeded(helper), false);
    when(cond1.isRecompactionNeeded(helper)).thenReturn(false);
    when(cond2.isRecompactionNeeded(helper)).thenReturn(true);
    when(cond3.isRecompactionNeeded(helper)).thenReturn(false);
    Assert.assertEquals(combineConditionOr.isRecompactionNeeded(helper), true);
    RecompactionCombineCondition combineConditionAnd = new RecompactionCombineCondition(Arrays.asList(cond1, cond2, cond3), RecompactionCombineCondition.CombineOperation.AND);
    when(cond1.isRecompactionNeeded(helper)).thenReturn(true);
    when(cond2.isRecompactionNeeded(helper)).thenReturn(true);
    when(cond3.isRecompactionNeeded(helper)).thenReturn(false);
    Assert.assertEquals(combineConditionAnd.isRecompactionNeeded(helper), false);
    when(cond1.isRecompactionNeeded(helper)).thenReturn(true);
    when(cond2.isRecompactionNeeded(helper)).thenReturn(true);
    when(cond3.isRecompactionNeeded(helper)).thenReturn(true);
    Assert.assertEquals(combineConditionAnd.isRecompactionNeeded(helper), true);
}

Example 4 with RecompactionCondition

use of org.apache.gobblin.compaction.conditions.RecompactionCondition in project incubator-gobblin by apache.

the class RecompactionConditionTest method testRecompactionConditionBasedOnDuration.

@Test
public void testRecompactionConditionBasedOnDuration() {
    RecompactionConditionFactory factory = new RecompactionConditionBasedOnDuration.Factory();
    RecompactionCondition conditionBasedOnDuration = factory.createRecompactionCondition(dataset);
    DatasetHelper helper = mock(DatasetHelper.class);
    when(helper.getDataset()).thenReturn(dataset);
    PeriodFormatter periodFormatter = new PeriodFormatterBuilder().appendMonths().appendSuffix("m").appendDays().appendSuffix("d").appendHours().appendSuffix("h").appendMinutes().appendSuffix("min").toFormatter();
    DateTime currentTime = getCurrentTime();
    Period period_A = periodFormatter.parsePeriod("11h59min");
    DateTime earliest_A = currentTime.minus(period_A);
    when(helper.getEarliestLateFileModificationTime()).thenReturn(Optional.of(earliest_A));
    when(helper.getCurrentTime()).thenReturn(currentTime);
    Assert.assertEquals(conditionBasedOnDuration.isRecompactionNeeded(helper), false);
    Period period_B = periodFormatter.parsePeriod("12h01min");
    DateTime earliest_B = currentTime.minus(period_B);
    when(helper.getEarliestLateFileModificationTime()).thenReturn(Optional.of(earliest_B));
    when(helper.getCurrentTime()).thenReturn(currentTime);
    Assert.assertEquals(conditionBasedOnDuration.isRecompactionNeeded(helper), true);
}