Examples with TimeRetentionStrategy com.linkedin.pinot.controller.helix.core.retention.strategy.TimeRetentionStrategy used on opensource projects

Example 1 with TimeRetentionStrategy

use of com.linkedin.pinot.controller.helix.core.retention.strategy.TimeRetentionStrategy in project pinot by linkedin.

the class RetentionManager method updateDeletionStrategyForRealtimeTable.

/**
   * Update deletion strategy for realtime table.
   * <ul>
   *   <li>Keep the current deletion strategy when failed to get a valid retention time</li>
   *   <li>Update the deletion strategy when valid retention time is set.</li>
   * </ul>
   * The reason for this is that we don't allow realtime table without deletion strategy.
   */
private void updateDeletionStrategyForRealtimeTable(String realtimeTableName) {
    try {
        AbstractTableConfig realtimeTableConfig = ZKMetadataProvider.getRealtimeTableConfig(_pinotHelixResourceManager.getPropertyStore(), realtimeTableName);
        assert realtimeTableConfig != null;
        SegmentsValidationAndRetentionConfig validationConfig = realtimeTableConfig.getValidationConfig();
        TimeRetentionStrategy timeRetentionStrategy = new TimeRetentionStrategy(validationConfig.getRetentionTimeUnit(), validationConfig.getRetentionTimeValue());
        _tableDeletionStrategy.put(realtimeTableName, timeRetentionStrategy);
    } catch (Exception e) {
        LOGGER.error("Caught exception while updating deletion strategy, skip updating deletion strategy for table: {}.", realtimeTableName, e);
    }
}

Example 2 with TimeRetentionStrategy

use of com.linkedin.pinot.controller.helix.core.retention.strategy.TimeRetentionStrategy in project pinot by linkedin.

the class RetentionManager method updateDeletionStrategyForOfflineTable.

/**
   * Update deletion strategy for offline table.
   * <ul>
   *   <li>Keep the current deletion strategy when one of the followings happened:
   *     <ul>
   *       <li>Failed to fetch the retention config.</li>
   *       <li>Push type is not valid (neither 'APPEND' nor 'REFRESH').</li>
   *     </ul>
   *   <li>
   *     Remove the deletion strategy when one of the followings happened:
   *     <ul>
   *       <li>Push type is set to 'REFRESH'.</li>
   *       <li>No valid retention time is set.</li>
   *     </ul>
   *   </li>
   *   <li>Update the deletion strategy when push type is set to 'APPEND' and valid retention time is set.</li>
   * </ul>
   */
private void updateDeletionStrategyForOfflineTable(String offlineTableName) {
    // Fetch table config.
    AbstractTableConfig offlineTableConfig;
    try {
        offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_pinotHelixResourceManager.getPropertyStore(), offlineTableName);
        if (offlineTableConfig == null) {
            LOGGER.error("Table config is null, skip updating deletion strategy for table: {}.", offlineTableName);
            return;
        }
    } catch (Exception e) {
        LOGGER.error("Caught exception while fetching table config, skip updating deletion strategy for table: {}.", offlineTableName, e);
        return;
    }
    // Fetch validation config.
    SegmentsValidationAndRetentionConfig validationConfig = offlineTableConfig.getValidationConfig();
    if (validationConfig == null) {
        LOGGER.error("Validation config is null, skip updating deletion strategy for table: {}.", offlineTableName);
        return;
    }
    // Fetch push type.
    String segmentPushType = validationConfig.getSegmentPushType();
    if ((segmentPushType == null) || (!segmentPushType.equalsIgnoreCase("APPEND") && !segmentPushType.equalsIgnoreCase("REFRESH"))) {
        LOGGER.error("Segment push type: {} is not valid ('APPEND' or 'REFRESH'), skip updating deletion strategy for table: {}.", segmentPushType, offlineTableName);
        return;
    }
    if (segmentPushType.equalsIgnoreCase("REFRESH")) {
        LOGGER.info("Segment push type is set to 'REFRESH', remove deletion strategy for table: {}.", offlineTableName);
        _tableDeletionStrategy.remove(offlineTableName);
        return;
    }
    // Fetch retention time unit and value.
    String retentionTimeUnit = validationConfig.getRetentionTimeUnit();
    String retentionTimeValue = validationConfig.getRetentionTimeValue();
    if (((retentionTimeUnit == null) || retentionTimeUnit.isEmpty()) || ((retentionTimeValue == null) || retentionTimeValue.isEmpty())) {
        LOGGER.info("Retention time unit/value is not set, remove deletion strategy for table: {}.", offlineTableName);
        _tableDeletionStrategy.remove(offlineTableName);
        return;
    }
    // Update time retention strategy.
    try {
        TimeRetentionStrategy timeRetentionStrategy = new TimeRetentionStrategy(retentionTimeUnit, retentionTimeValue);
        _tableDeletionStrategy.put(offlineTableName, timeRetentionStrategy);
        LOGGER.info("Updated deletion strategy for table: {} using retention time: {} {}.", offlineTableName, retentionTimeValue, retentionTimeUnit);
    } catch (Exception e) {
        LOGGER.error("Caught exception while building deletion strategy with retention time: {} {], remove deletion strategy for table: {}.", retentionTimeValue, retentionTimeUnit, offlineTableName);
        _tableDeletionStrategy.remove(offlineTableName);
    }
}