Examples with RetentionStrategy com.linkedin.pinot.controller.helix.core.retention.strategy.RetentionStrategy used on opensource projects

Example 1 with RetentionStrategy

use of com.linkedin.pinot.controller.helix.core.retention.strategy.RetentionStrategy in project pinot by linkedin.

the class RetentionManager method scanSegmentMetadataAndPurge.

private void scanSegmentMetadataAndPurge() {
    for (String tableName : _segmentMetadataMap.keySet()) {
        List<SegmentZKMetadata> segmentZKMetadataList = _segmentMetadataMap.get(tableName);
        List<String> segmentsToDelete = new ArrayList<>(128);
        IdealState idealState = null;
        try {
            if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(TableType.REALTIME)) {
                idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableName);
            }
        } catch (Exception e) {
            LOGGER.warn("Could not get idealstate for {}", tableName, e);
        // Ignore, worst case we have some old inactive segments in place.
        }
        for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
            RetentionStrategy deletionStrategy;
            deletionStrategy = _tableDeletionStrategy.get(tableName);
            if (deletionStrategy == null) {
                LOGGER.info("No Retention strategy found for segment: {}", segmentZKMetadata.getSegmentName());
                continue;
            }
            if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
                final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = (RealtimeSegmentZKMetadata) segmentZKMetadata;
                if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                    final String segmentId = realtimeSegmentZKMetadata.getSegmentName();
                    if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
                        continue;
                    }
                    // auto-create LLC segments.
                    if (shouldDeleteInProgressLLCSegment(segmentId, idealState, realtimeSegmentZKMetadata)) {
                        segmentsToDelete.add(segmentId);
                    }
                    continue;
                }
            }
            if (deletionStrategy.isPurgeable(segmentZKMetadata)) {
                LOGGER.info("Marking segment to delete: {}", segmentZKMetadata.getSegmentName());
                segmentsToDelete.add(segmentZKMetadata.getSegmentName());
            }
        }
        if (segmentsToDelete.size() > 0) {
            LOGGER.info("Trying to delete {} segments for table {}", segmentsToDelete.size(), tableName);
            _pinotHelixResourceManager.deleteSegments(tableName, segmentsToDelete);
        }
    }
}