Examples with RetentionManager com.linkedin.pinot.controller.helix.core.retention.RetentionManager used on opensource projects

Example 1 with RetentionManager

use of com.linkedin.pinot.controller.helix.core.retention.RetentionManager in project pinot by linkedin.

the class RetentionManagerTest method testRealtimeLLCCleanup.

@Test
public void testRealtimeLLCCleanup() throws Exception {
    final int initialNumSegments = 8;
    final long now = System.currentTimeMillis();
    Set<String> remainingSegments = setupRealtimeTable(initialNumSegments, now);
    // At least one segment should be deleted, otherwise we don't have a test
    Assert.assertTrue(initialNumSegments - remainingSegments.size() > 0);
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    // Do not check external view when validating because the segments that are OFFLINE in Idealstate to begin with
    // never show up in Externalview
    validate(initialNumSegments, _realtimeTableName, remainingSegments.size(), false);
    // Ensure that the segments that should be present are indeed present.
    IdealState idealState = HelixHelper.getTableIdealState(_helixZkManager, _realtimeTableName);
    for (final String segmentId : remainingSegments) {
        Assert.assertTrue(idealState.getPartitionSet().contains(segmentId));
        Assert.assertNotNull(ZKMetadataProvider.getRealtimeSegmentZKMetadata(_propertyStore, _realtimeTableName, segmentId));
    }
    cleanupSegments(_realtimeTableName);
}

Example 2 with RetentionManager

use of com.linkedin.pinot.controller.helix.core.retention.RetentionManager in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithMillsTimeUnit.

/**
   * Test with millseconds time unit.
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithMillsTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long millsSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60 * 60 * 1000;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("1343001600000", "1343001600000", TimeUnit.MILLISECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(millsSinceEpochTimeStamp + "", millsSinceEpochTimeStamp + "", TimeUnit.MILLISECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 3 with RetentionManager

use of com.linkedin.pinot.controller.helix.core.retention.RetentionManager in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithSecondsTimeUnit.

/**
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithSecondsTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long secondsSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60 * 60;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("1343001600", "1343001600", TimeUnit.SECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(secondsSinceEpochTimeStamp + "", secondsSinceEpochTimeStamp + "", TimeUnit.SECONDS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 4 with RetentionManager

use of com.linkedin.pinot.controller.helix.core.retention.RetentionManager in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithMinutesTimeUnit.

/**
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithMinutesTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long minutesSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24 * 60;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("22383360", "22383360", TimeUnit.MINUTES.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(minutesSinceEpochTimeStamp + "", minutesSinceEpochTimeStamp + "", TimeUnit.MINUTES.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}

Example 5 with RetentionManager

use of com.linkedin.pinot.controller.helix.core.retention.RetentionManager in project pinot by linkedin.

the class RetentionManagerTest method testRetentionWithHoursTimeUnit.

/**
   * @throws JSONException
   * @throws UnsupportedEncodingException
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void testRetentionWithHoursTimeUnit() throws JSONException, UnsupportedEncodingException, IOException, InterruptedException {
    _retentionManager = new RetentionManager(_pinotHelixResourceManager, 5);
    _retentionManager.start();
    long theDayAfterTomorrowSinceEpoch = System.currentTimeMillis() / 1000 / 60 / 60 / 24 + 2;
    long hoursSinceEpochTimeStamp = theDayAfterTomorrowSinceEpoch * 24;
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl("373056", "373056", TimeUnit.HOURS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    for (int i = 0; i < 10; ++i) {
        SegmentMetadata segmentMetadata = getTimeSegmentMetadataImpl(hoursSinceEpochTimeStamp + "", hoursSinceEpochTimeStamp + "", TimeUnit.HOURS.toString());
        registerSegmentMetadata(segmentMetadata);
        Thread.sleep(100);
    }
    validate(20, _offlineTableName, 10, true);
    cleanupSegments(_offlineTableName);
}