Examples with CompactionStatistics org.apache.druid.server.coordinator.CompactionStatistics used on opensource projects

Example 1 with CompactionStatistics

use of org.apache.druid.server.coordinator.CompactionStatistics in project druid by druid-io.

the class NewestSegmentFirstIterator method collectSegmentStatistics.

private void collectSegmentStatistics(Map<String, CompactionStatistics> statisticsMap, String dataSourceName, SegmentsToCompact segments) {
    CompactionStatistics statistics = statisticsMap.computeIfAbsent(dataSourceName, v -> CompactionStatistics.initializeCompactionStatistics());
    statistics.incrementCompactedByte(segments.getTotalSize());
    statistics.incrementCompactedIntervals(segments.getNumberOfIntervals());
    statistics.incrementCompactedSegments(segments.getNumberOfSegments());
}

Example 2 with CompactionStatistics

use of org.apache.druid.server.coordinator.CompactionStatistics in project druid by druid-io.

the class CompactSegments method makeStats.

private CoordinatorStats makeStats(Map<String, AutoCompactionSnapshot.Builder> currentRunAutoCompactionSnapshotBuilders, int numCompactionTasks, CompactionSegmentIterator iterator) {
    final Map<String, AutoCompactionSnapshot> currentAutoCompactionSnapshotPerDataSource = new HashMap<>();
    final CoordinatorStats stats = new CoordinatorStats();
    stats.addToGlobalStat(COMPACTION_TASK_COUNT, numCompactionTasks);
    // the statistic to the AwaitingCompaction statistics
    while (iterator.hasNext()) {
        final List<DataSegment> segmentsToCompact = iterator.next();
        if (!segmentsToCompact.isEmpty()) {
            final String dataSourceName = segmentsToCompact.get(0).getDataSource();
            AutoCompactionSnapshot.Builder snapshotBuilder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSourceName, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
            snapshotBuilder.incrementBytesAwaitingCompaction(segmentsToCompact.stream().mapToLong(DataSegment::getSize).sum());
            snapshotBuilder.incrementIntervalCountAwaitingCompaction(segmentsToCompact.stream().map(DataSegment::getInterval).distinct().count());
            snapshotBuilder.incrementSegmentCountAwaitingCompaction(segmentsToCompact.size());
        }
    }
    // Statistics of all segments considered compacted after this run
    Map<String, CompactionStatistics> allCompactedStatistics = iterator.totalCompactedStatistics();
    for (Map.Entry<String, CompactionStatistics> compactionStatisticsEntry : allCompactedStatistics.entrySet()) {
        final String dataSource = compactionStatisticsEntry.getKey();
        final CompactionStatistics dataSourceCompactedStatistics = compactionStatisticsEntry.getValue();
        AutoCompactionSnapshot.Builder builder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSource, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
        builder.incrementBytesCompacted(dataSourceCompactedStatistics.getByteSum());
        builder.incrementSegmentCountCompacted(dataSourceCompactedStatistics.getSegmentNumberCountSum());
        builder.incrementIntervalCountCompacted(dataSourceCompactedStatistics.getSegmentIntervalCountSum());
    }
    // Statistics of all segments considered skipped after this run
    Map<String, CompactionStatistics> allSkippedStatistics = iterator.totalSkippedStatistics();
    for (Map.Entry<String, CompactionStatistics> compactionStatisticsEntry : allSkippedStatistics.entrySet()) {
        final String dataSource = compactionStatisticsEntry.getKey();
        final CompactionStatistics dataSourceSkippedStatistics = compactionStatisticsEntry.getValue();
        AutoCompactionSnapshot.Builder builder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSource, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
        builder.incrementBytesSkipped(dataSourceSkippedStatistics.getByteSum());
        builder.incrementSegmentCountSkipped(dataSourceSkippedStatistics.getSegmentNumberCountSum());
        builder.incrementIntervalCountSkipped(dataSourceSkippedStatistics.getSegmentIntervalCountSum());
    }
    for (Map.Entry<String, AutoCompactionSnapshot.Builder> autoCompactionSnapshotBuilderEntry : currentRunAutoCompactionSnapshotBuilders.entrySet()) {
        final String dataSource = autoCompactionSnapshotBuilderEntry.getKey();
        final AutoCompactionSnapshot.Builder builder = autoCompactionSnapshotBuilderEntry.getValue();
        // Build the complete snapshot for the datasource
        AutoCompactionSnapshot autoCompactionSnapshot = builder.build();
        currentAutoCompactionSnapshotPerDataSource.put(dataSource, autoCompactionSnapshot);
        // Use the complete snapshot to emits metrics
        stats.addToDataSourceStat(TOTAL_SIZE_OF_SEGMENTS_AWAITING, dataSource, autoCompactionSnapshot.getBytesAwaitingCompaction());
        stats.addToDataSourceStat(TOTAL_COUNT_OF_SEGMENTS_AWAITING, dataSource, autoCompactionSnapshot.getSegmentCountAwaitingCompaction());
        stats.addToDataSourceStat(TOTAL_INTERVAL_OF_SEGMENTS_AWAITING, dataSource, autoCompactionSnapshot.getIntervalCountAwaitingCompaction());
        stats.addToDataSourceStat(TOTAL_SIZE_OF_SEGMENTS_COMPACTED, dataSource, autoCompactionSnapshot.getBytesCompacted());
        stats.addToDataSourceStat(TOTAL_COUNT_OF_SEGMENTS_COMPACTED, dataSource, autoCompactionSnapshot.getSegmentCountCompacted());
        stats.addToDataSourceStat(TOTAL_INTERVAL_OF_SEGMENTS_COMPACTED, dataSource, autoCompactionSnapshot.getIntervalCountCompacted());
        stats.addToDataSourceStat(TOTAL_SIZE_OF_SEGMENTS_SKIPPED, dataSource, autoCompactionSnapshot.getBytesSkipped());
        stats.addToDataSourceStat(TOTAL_COUNT_OF_SEGMENTS_SKIPPED, dataSource, autoCompactionSnapshot.getSegmentCountSkipped());
        stats.addToDataSourceStat(TOTAL_INTERVAL_OF_SEGMENTS_SKIPPED, dataSource, autoCompactionSnapshot.getIntervalCountSkipped());
    }
    // Atomic update of autoCompactionSnapshotPerDataSource with the latest from this coordinator run
    autoCompactionSnapshotPerDataSource.set(currentAutoCompactionSnapshotPerDataSource);
    return stats;
}