Examples with SegmentZKMetadata com.linkedin.pinot.common.metadata.segment.SegmentZKMetadata used on opensource projects

Example 1 with SegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.SegmentZKMetadata in project pinot by linkedin.

the class RetentionManager method retrieveSegmentMetadataForTable.

private List<SegmentZKMetadata> retrieveSegmentMetadataForTable(String tableName) {
    List<SegmentZKMetadata> segmentMetadataList = new ArrayList<>();
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
    assert tableType != null;
    switch(tableType) {
        case OFFLINE:
            List<OfflineSegmentZKMetadata> offlineSegmentZKMetadatas = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(propertyStore, tableName);
            for (OfflineSegmentZKMetadata offlineSegmentZKMetadata : offlineSegmentZKMetadatas) {
                segmentMetadataList.add(offlineSegmentZKMetadata);
            }
            break;
        case REALTIME:
            List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadatas = ZKMetadataProvider.getRealtimeSegmentZKMetadataListForTable(propertyStore, tableName);
            for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadatas) {
                segmentMetadataList.add(realtimeSegmentZKMetadata);
            }
            break;
        default:
            throw new IllegalArgumentException("No table type matches table name: " + tableName);
    }
    return segmentMetadataList;
}

Example 2 with SegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.SegmentZKMetadata in project pinot by linkedin.

the class RetentionManager method scanSegmentMetadataAndPurge.

private void scanSegmentMetadataAndPurge() {
    for (String tableName : _segmentMetadataMap.keySet()) {
        List<SegmentZKMetadata> segmentZKMetadataList = _segmentMetadataMap.get(tableName);
        List<String> segmentsToDelete = new ArrayList<>(128);
        IdealState idealState = null;
        try {
            if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(TableType.REALTIME)) {
                idealState = HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), tableName);
            }
        } catch (Exception e) {
            LOGGER.warn("Could not get idealstate for {}", tableName, e);
        // Ignore, worst case we have some old inactive segments in place.
        }
        for (SegmentZKMetadata segmentZKMetadata : segmentZKMetadataList) {
            RetentionStrategy deletionStrategy;
            deletionStrategy = _tableDeletionStrategy.get(tableName);
            if (deletionStrategy == null) {
                LOGGER.info("No Retention strategy found for segment: {}", segmentZKMetadata.getSegmentName());
                continue;
            }
            if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
                final RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = (RealtimeSegmentZKMetadata) segmentZKMetadata;
                if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                    final String segmentId = realtimeSegmentZKMetadata.getSegmentName();
                    if (SegmentName.isHighLevelConsumerSegmentName(segmentId)) {
                        continue;
                    }
                    // auto-create LLC segments.
                    if (shouldDeleteInProgressLLCSegment(segmentId, idealState, realtimeSegmentZKMetadata)) {
                        segmentsToDelete.add(segmentId);
                    }
                    continue;
                }
            }
            if (deletionStrategy.isPurgeable(segmentZKMetadata)) {
                LOGGER.info("Marking segment to delete: {}", segmentZKMetadata.getSegmentName());
                segmentsToDelete.add(segmentZKMetadata.getSegmentName());
            }
        }
        if (segmentsToDelete.size() > 0) {
            LOGGER.info("Trying to delete {} segments for table {}", segmentsToDelete.size(), tableName);
            _pinotHelixResourceManager.deleteSegments(tableName, segmentsToDelete);
        }
    }
}

Example 3 with SegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.SegmentZKMetadata in project pinot by linkedin.

the class TimeRetentionStrategyTest method testTimeRetention.

@Test
public void testTimeRetention() throws Exception {
    TimeRetentionStrategy retentionStrategy = new TimeRetentionStrategy("DAYS", "30");
    SegmentZKMetadata metadata = new OfflineSegmentZKMetadata();
    metadata.setTimeUnit(TimeUnit.DAYS);
    // Set end time to Jan 2nd, 1970 (not purgeable due to bogus timestamp)
    metadata.setEndTime(1L);
    assertFalse(retentionStrategy.isPurgeable(metadata));
    // Set end time to today
    final long today = TimeUnit.MILLISECONDS.toDays(System.currentTimeMillis());
    metadata.setEndTime(today);
    assertFalse(retentionStrategy.isPurgeable(metadata));
    // Set end time to two weeks ago
    metadata.setEndTime(today - 14);
    assertFalse(retentionStrategy.isPurgeable(metadata));
    // Set end time to two months ago (purgeable due to being past the retention period)
    metadata.setEndTime(today - 60);
    assertTrue(retentionStrategy.isPurgeable(metadata));
    // Set end time to 200 years in the future (not purgeable due to bogus timestamp)
    metadata.setEndTime(today + (365 * 200));
    assertFalse(retentionStrategy.isPurgeable(metadata));
}