Examples with DataSourceCompactionConfig org.apache.druid.server.coordinator.DataSourceCompactionConfig used on opensource projects

Example 21 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactSegments method doRun.

private CoordinatorStats doRun(Map<String, DataSourceCompactionConfig> compactionConfigs, Map<String, AutoCompactionSnapshot.Builder> currentRunAutoCompactionSnapshotBuilders, int numAvailableCompactionTaskSlots, CompactionSegmentIterator iterator) {
    int numSubmittedTasks = 0;
    int numCompactionTasksAndSubtasks = 0;
    while (iterator.hasNext() && numCompactionTasksAndSubtasks < numAvailableCompactionTaskSlots) {
        final List<DataSegment> segmentsToCompact = iterator.next();
        if (!segmentsToCompact.isEmpty()) {
            final String dataSourceName = segmentsToCompact.get(0).getDataSource();
            // As these segments will be compacted, we will aggregates the statistic to the Compacted statistics
            AutoCompactionSnapshot.Builder snapshotBuilder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSourceName, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
            snapshotBuilder.incrementBytesCompacted(segmentsToCompact.stream().mapToLong(DataSegment::getSize).sum());
            snapshotBuilder.incrementIntervalCountCompacted(segmentsToCompact.stream().map(DataSegment::getInterval).distinct().count());
            snapshotBuilder.incrementSegmentCountCompacted(segmentsToCompact.size());
            final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName);
            // Create granularitySpec to send to compaction task
            ClientCompactionTaskGranularitySpec granularitySpec;
            Granularity segmentGranularityToUse = null;
            if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) {
                // Determines segmentGranularity from the segmentsToCompact
                // Each batch of segmentToCompact from CompactionSegmentIterator will contains the same interval as
                // segmentGranularity is not set in the compaction config
                Interval interval = segmentsToCompact.get(0).getInterval();
                if (segmentsToCompact.stream().allMatch(segment -> interval.overlaps(segment.getInterval()))) {
                    try {
                        segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity();
                    } catch (IAE iae) {
                        // This case can happen if the existing segment interval result in complicated periods.
                        // Fall back to setting segmentGranularity as null
                        LOG.warn("Cannot determine segmentGranularity from interval [%s]", interval);
                    }
                } else {
                    LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity for auto compaction task");
                }
            } else {
                segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity();
            }
            granularitySpec = new ClientCompactionTaskGranularitySpec(segmentGranularityToUse, config.getGranularitySpec() != null ? config.getGranularitySpec().getQueryGranularity() : null, config.getGranularitySpec() != null ? config.getGranularitySpec().isRollup() : null);
            // Create dimensionsSpec to send to compaction task
            ClientCompactionTaskDimensionsSpec dimensionsSpec;
            if (config.getDimensionsSpec() != null) {
                dimensionsSpec = new ClientCompactionTaskDimensionsSpec(config.getDimensionsSpec().getDimensions());
            } else {
                dimensionsSpec = null;
            }
            // Create transformSpec to send to compaction task
            ClientCompactionTaskTransformSpec transformSpec = null;
            if (config.getTransformSpec() != null) {
                transformSpec = new ClientCompactionTaskTransformSpec(config.getTransformSpec().getFilter());
            }
            Boolean dropExisting = null;
            if (config.getIoConfig() != null) {
                dropExisting = config.getIoConfig().isDropExisting();
            }
            // make tuningConfig
            final String taskId = indexingServiceClient.compactSegments("coordinator-issued", segmentsToCompact, config.getTaskPriority(), ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment()), granularitySpec, dimensionsSpec, config.getMetricsSpec(), transformSpec, dropExisting, newAutoCompactionContext(config.getTaskContext()));
            LOG.info("Submitted a compactionTask[%s] for %s segments", taskId, segmentsToCompact.size());
            LOG.infoSegments(segmentsToCompact, "Compacting segments");
            // Count the compaction task itself + its sub tasks
            numSubmittedTasks++;
            numCompactionTasksAndSubtasks += findMaxNumTaskSlotsUsedByOneCompactionTask(config.getTuningConfig());
        } else {
            throw new ISE("segmentsToCompact is empty?");
        }
    }
    return makeStats(currentRunAutoCompactionSnapshotBuilders, numSubmittedTasks, iterator);
}

Example 22 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithRangePartitionsSpec.

@Test
public void testFindPartitionsSpecFromConfigWithRangePartitionsSpec() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new SingleDimensionPartitionsSpec(10000, null, "dim", false), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new SingleDimensionPartitionsSpec(10000, null, "dim", false), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 23 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithNonNullMaxTotalRowsReturnGivenValue.

@Test
public void testFindPartitionsSpecFromConfigWithNonNullMaxTotalRowsReturnGivenValue() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new DynamicPartitionsSpec(null, 1000L), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, 1000L), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 24 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithHashPartitionsSpec.

@Test
public void testFindPartitionsSpecFromConfigWithHashPartitionsSpec() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new HashedPartitionsSpec(null, 10, ImmutableList.of("dim")), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new HashedPartitionsSpec(null, 10, ImmutableList.of("dim")), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 25 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithNonNullMaxRowsPerSegmentReturnGivenValue.

@Test
public void testFindPartitionsSpecFromConfigWithNonNullMaxRowsPerSegmentReturnGivenValue() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new DynamicPartitionsSpec(100, 1000L), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(100, 1000L), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}