Examples with ClientCompactionTaskTransformSpec org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec used on opensource projects

Example 11 with ClientCompactionTaskTransformSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec in project druid by druid-io.

the class CompactSegments method doRun.

private CoordinatorStats doRun(Map<String, DataSourceCompactionConfig> compactionConfigs, Map<String, AutoCompactionSnapshot.Builder> currentRunAutoCompactionSnapshotBuilders, int numAvailableCompactionTaskSlots, CompactionSegmentIterator iterator) {
    int numSubmittedTasks = 0;
    int numCompactionTasksAndSubtasks = 0;
    while (iterator.hasNext() && numCompactionTasksAndSubtasks < numAvailableCompactionTaskSlots) {
        final List<DataSegment> segmentsToCompact = iterator.next();
        if (!segmentsToCompact.isEmpty()) {
            final String dataSourceName = segmentsToCompact.get(0).getDataSource();
            // As these segments will be compacted, we will aggregates the statistic to the Compacted statistics
            AutoCompactionSnapshot.Builder snapshotBuilder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSourceName, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
            snapshotBuilder.incrementBytesCompacted(segmentsToCompact.stream().mapToLong(DataSegment::getSize).sum());
            snapshotBuilder.incrementIntervalCountCompacted(segmentsToCompact.stream().map(DataSegment::getInterval).distinct().count());
            snapshotBuilder.incrementSegmentCountCompacted(segmentsToCompact.size());
            final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName);
            // Create granularitySpec to send to compaction task
            ClientCompactionTaskGranularitySpec granularitySpec;
            Granularity segmentGranularityToUse = null;
            if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) {
                // Determines segmentGranularity from the segmentsToCompact
                // Each batch of segmentToCompact from CompactionSegmentIterator will contains the same interval as
                // segmentGranularity is not set in the compaction config
                Interval interval = segmentsToCompact.get(0).getInterval();
                if (segmentsToCompact.stream().allMatch(segment -> interval.overlaps(segment.getInterval()))) {
                    try {
                        segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity();
                    } catch (IAE iae) {
                        // This case can happen if the existing segment interval result in complicated periods.
                        // Fall back to setting segmentGranularity as null
                        LOG.warn("Cannot determine segmentGranularity from interval [%s]", interval);
                    }
                } else {
                    LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity for auto compaction task");
                }
            } else {
                segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity();
            }
            granularitySpec = new ClientCompactionTaskGranularitySpec(segmentGranularityToUse, config.getGranularitySpec() != null ? config.getGranularitySpec().getQueryGranularity() : null, config.getGranularitySpec() != null ? config.getGranularitySpec().isRollup() : null);
            // Create dimensionsSpec to send to compaction task
            ClientCompactionTaskDimensionsSpec dimensionsSpec;
            if (config.getDimensionsSpec() != null) {
                dimensionsSpec = new ClientCompactionTaskDimensionsSpec(config.getDimensionsSpec().getDimensions());
            } else {
                dimensionsSpec = null;
            }
            // Create transformSpec to send to compaction task
            ClientCompactionTaskTransformSpec transformSpec = null;
            if (config.getTransformSpec() != null) {
                transformSpec = new ClientCompactionTaskTransformSpec(config.getTransformSpec().getFilter());
            }
            Boolean dropExisting = null;
            if (config.getIoConfig() != null) {
                dropExisting = config.getIoConfig().isDropExisting();
            }
            // make tuningConfig
            final String taskId = indexingServiceClient.compactSegments("coordinator-issued", segmentsToCompact, config.getTaskPriority(), ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment()), granularitySpec, dimensionsSpec, config.getMetricsSpec(), transformSpec, dropExisting, newAutoCompactionContext(config.getTaskContext()));
            LOG.info("Submitted a compactionTask[%s] for %s segments", taskId, segmentsToCompact.size());
            LOG.infoSegments(segmentsToCompact, "Compacting segments");
            // Count the compaction task itself + its sub tasks
            numSubmittedTasks++;
            numCompactionTasksAndSubtasks += findMaxNumTaskSlotsUsedByOneCompactionTask(config.getTuningConfig());
        } else {
            throw new ISE("segmentsToCompact is empty?");
        }
    }
    return makeStats(currentRunAutoCompactionSnapshotBuilders, numSubmittedTasks, iterator);
}