Examples with ClientCompactionTaskDimensionsSpec org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec used on opensource projects

Example 1 with ClientCompactionTaskDimensionsSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec in project druid by druid-io.

the class ClientCompactionTaskQuerySerdeTest method testCompactionTaskToClientCompactionTaskQuery.

@Test
public void testCompactionTaskToClientCompactionTaskQuery() throws IOException {
    final ObjectMapper mapper = setupInjectablesInObjectMapper(new DefaultObjectMapper());
    final CompactionTask.Builder builder = new CompactionTask.Builder("datasource", new SegmentCacheManagerFactory(mapper), new RetryPolicyFactory(new RetryPolicyConfig()));
    final CompactionTask task = builder.inputSpec(new CompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true).tuningConfig(new ParallelIndexTuningConfig(null, null, null, 40000, 2000L, null, null, null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.UNCOMPRESSED, LongEncodingStrategy.AUTO), 2, null, null, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), null, 100, 5, 1000L, new Duration(3000L), 7, 1000, 100, null, null, null, null, null, null)).granularitySpec(new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true)).dimensionsSpec(DimensionsSpec.builder().setDimensions(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))).setDimensionExclusions(ImmutableList.of("__time", "val")).build()).metricsSpec(new AggregatorFactory[] { new CountAggregatorFactory("cnt") }).transformSpec(new ClientCompactionTaskTransformSpec(new SelectorDimFilter("dim1", "foo", null))).build();
    final ClientCompactionTaskQuery expected = new ClientCompactionTaskQuery(task.getId(), "datasource", new ClientCompactionIOConfig(new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true), new ClientCompactionTaskQueryTuningConfig(100, 40000, 2000L, 30000L, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.UNCOMPRESSED, LongEncodingStrategy.AUTO), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), 100, 5, 1000L, new Duration(3000L), 7, 1000, 100), new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true), new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), new AggregatorFactory[] { new CountAggregatorFactory("cnt") }, new ClientCompactionTaskTransformSpec(new SelectorDimFilter("dim1", "foo", null)), new HashMap<>());
    final byte[] json = mapper.writeValueAsBytes(task);
    final ClientCompactionTaskQuery actual = (ClientCompactionTaskQuery) mapper.readValue(json, ClientTaskQuery.class);
    Assert.assertEquals(expected, actual);
}

Example 2 with ClientCompactionTaskDimensionsSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec in project druid by druid-io.

the class ClientCompactionTaskQuerySerdeTest method testClientCompactionTaskQueryToCompactionTask.

@Test
public void testClientCompactionTaskQueryToCompactionTask() throws IOException {
    final ObjectMapper mapper = setupInjectablesInObjectMapper(new DefaultObjectMapper());
    final ClientCompactionTaskQuery query = new ClientCompactionTaskQuery("id", "datasource", new ClientCompactionIOConfig(new ClientCompactionIntervalSpec(Intervals.of("2019/2020"), "testSha256OfSortedSegmentIds"), true), new ClientCompactionTaskQueryTuningConfig(null, 40000, 2000L, null, new SegmentsSplitHintSpec(new HumanReadableBytes(100000L), 10), new DynamicPartitionsSpec(100, 30000L), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.LZF, LongEncodingStrategy.LONGS), new IndexSpec(new DefaultBitmapSerdeFactory(), CompressionStrategy.LZ4, CompressionStrategy.UNCOMPRESSED, LongEncodingStrategy.AUTO), 2, 1000L, TmpFileSegmentWriteOutMediumFactory.instance(), 100, 5, 1000L, new Duration(3000L), 7, 1000, 100), new ClientCompactionTaskGranularitySpec(Granularities.DAY, Granularities.HOUR, true), new ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), new AggregatorFactory[] { new CountAggregatorFactory("cnt") }, new ClientCompactionTaskTransformSpec(new SelectorDimFilter("dim1", "foo", null)), ImmutableMap.of("key", "value"));
    final byte[] json = mapper.writeValueAsBytes(query);
    final CompactionTask task = (CompactionTask) mapper.readValue(json, Task.class);
    Assert.assertEquals(query.getId(), task.getId());
    Assert.assertEquals(query.getDataSource(), task.getDataSource());
    Assert.assertTrue(task.getIoConfig().getInputSpec() instanceof CompactionIntervalSpec);
    Assert.assertEquals(query.getIoConfig().getInputSpec().getInterval(), ((CompactionIntervalSpec) task.getIoConfig().getInputSpec()).getInterval());
    Assert.assertEquals(query.getIoConfig().getInputSpec().getSha256OfSortedSegmentIds(), ((CompactionIntervalSpec) task.getIoConfig().getInputSpec()).getSha256OfSortedSegmentIds());
    Assert.assertEquals(query.getTuningConfig().getMaxRowsInMemory().intValue(), task.getTuningConfig().getMaxRowsInMemory());
    Assert.assertEquals(query.getTuningConfig().getMaxBytesInMemory().longValue(), task.getTuningConfig().getMaxBytesInMemory());
    Assert.assertEquals(query.getTuningConfig().getSplitHintSpec(), task.getTuningConfig().getSplitHintSpec());
    Assert.assertEquals(query.getTuningConfig().getPartitionsSpec(), task.getTuningConfig().getPartitionsSpec());
    Assert.assertEquals(query.getTuningConfig().getIndexSpec(), task.getTuningConfig().getIndexSpec());
    Assert.assertEquals(query.getTuningConfig().getIndexSpecForIntermediatePersists(), task.getTuningConfig().getIndexSpecForIntermediatePersists());
    Assert.assertEquals(query.getTuningConfig().getPushTimeout().longValue(), task.getTuningConfig().getPushTimeout());
    Assert.assertEquals(query.getTuningConfig().getSegmentWriteOutMediumFactory(), task.getTuningConfig().getSegmentWriteOutMediumFactory());
    Assert.assertEquals(query.getTuningConfig().getMaxNumConcurrentSubTasks().intValue(), task.getTuningConfig().getMaxNumConcurrentSubTasks());
    Assert.assertEquals(query.getTuningConfig().getMaxRetry().intValue(), task.getTuningConfig().getMaxRetry());
    Assert.assertEquals(query.getTuningConfig().getTaskStatusCheckPeriodMs().longValue(), task.getTuningConfig().getTaskStatusCheckPeriodMs());
    Assert.assertEquals(query.getTuningConfig().getChatHandlerTimeout(), task.getTuningConfig().getChatHandlerTimeout());
    Assert.assertEquals(query.getTuningConfig().getMaxNumSegmentsToMerge().intValue(), task.getTuningConfig().getMaxNumSegmentsToMerge());
    Assert.assertEquals(query.getTuningConfig().getTotalNumMergeTasks().intValue(), task.getTuningConfig().getTotalNumMergeTasks());
    Assert.assertEquals(query.getGranularitySpec(), task.getGranularitySpec());
    Assert.assertEquals(query.getGranularitySpec().getQueryGranularity(), task.getGranularitySpec().getQueryGranularity());
    Assert.assertEquals(query.getGranularitySpec().getSegmentGranularity(), task.getGranularitySpec().getSegmentGranularity());
    Assert.assertEquals(query.getGranularitySpec().isRollup(), task.getGranularitySpec().isRollup());
    Assert.assertEquals(query.getIoConfig().isDropExisting(), task.getIoConfig().isDropExisting());
    Assert.assertEquals(query.getContext(), task.getContext());
    Assert.assertEquals(query.getDimensionsSpec().getDimensions(), task.getDimensionsSpec().getDimensions());
    Assert.assertEquals(query.getTransformSpec().getFilter(), task.getTransformSpec().getFilter());
    Assert.assertArrayEquals(query.getMetricsSpec(), task.getMetricsSpec());
}

Example 3 with ClientCompactionTaskDimensionsSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec in project druid by druid-io.

the class CompactSegments method doRun.

private CoordinatorStats doRun(Map<String, DataSourceCompactionConfig> compactionConfigs, Map<String, AutoCompactionSnapshot.Builder> currentRunAutoCompactionSnapshotBuilders, int numAvailableCompactionTaskSlots, CompactionSegmentIterator iterator) {
    int numSubmittedTasks = 0;
    int numCompactionTasksAndSubtasks = 0;
    while (iterator.hasNext() && numCompactionTasksAndSubtasks < numAvailableCompactionTaskSlots) {
        final List<DataSegment> segmentsToCompact = iterator.next();
        if (!segmentsToCompact.isEmpty()) {
            final String dataSourceName = segmentsToCompact.get(0).getDataSource();
            // As these segments will be compacted, we will aggregates the statistic to the Compacted statistics
            AutoCompactionSnapshot.Builder snapshotBuilder = currentRunAutoCompactionSnapshotBuilders.computeIfAbsent(dataSourceName, k -> new AutoCompactionSnapshot.Builder(k, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING));
            snapshotBuilder.incrementBytesCompacted(segmentsToCompact.stream().mapToLong(DataSegment::getSize).sum());
            snapshotBuilder.incrementIntervalCountCompacted(segmentsToCompact.stream().map(DataSegment::getInterval).distinct().count());
            snapshotBuilder.incrementSegmentCountCompacted(segmentsToCompact.size());
            final DataSourceCompactionConfig config = compactionConfigs.get(dataSourceName);
            // Create granularitySpec to send to compaction task
            ClientCompactionTaskGranularitySpec granularitySpec;
            Granularity segmentGranularityToUse = null;
            if (config.getGranularitySpec() == null || config.getGranularitySpec().getSegmentGranularity() == null) {
                // Determines segmentGranularity from the segmentsToCompact
                // Each batch of segmentToCompact from CompactionSegmentIterator will contains the same interval as
                // segmentGranularity is not set in the compaction config
                Interval interval = segmentsToCompact.get(0).getInterval();
                if (segmentsToCompact.stream().allMatch(segment -> interval.overlaps(segment.getInterval()))) {
                    try {
                        segmentGranularityToUse = GranularityType.fromPeriod(interval.toPeriod()).getDefaultGranularity();
                    } catch (IAE iae) {
                        // This case can happen if the existing segment interval result in complicated periods.
                        // Fall back to setting segmentGranularity as null
                        LOG.warn("Cannot determine segmentGranularity from interval [%s]", interval);
                    }
                } else {
                    LOG.warn("segmentsToCompact does not have the same interval. Fallback to not setting segmentGranularity for auto compaction task");
                }
            } else {
                segmentGranularityToUse = config.getGranularitySpec().getSegmentGranularity();
            }
            granularitySpec = new ClientCompactionTaskGranularitySpec(segmentGranularityToUse, config.getGranularitySpec() != null ? config.getGranularitySpec().getQueryGranularity() : null, config.getGranularitySpec() != null ? config.getGranularitySpec().isRollup() : null);
            // Create dimensionsSpec to send to compaction task
            ClientCompactionTaskDimensionsSpec dimensionsSpec;
            if (config.getDimensionsSpec() != null) {
                dimensionsSpec = new ClientCompactionTaskDimensionsSpec(config.getDimensionsSpec().getDimensions());
            } else {
                dimensionsSpec = null;
            }
            // Create transformSpec to send to compaction task
            ClientCompactionTaskTransformSpec transformSpec = null;
            if (config.getTransformSpec() != null) {
                transformSpec = new ClientCompactionTaskTransformSpec(config.getTransformSpec().getFilter());
            }
            Boolean dropExisting = null;
            if (config.getIoConfig() != null) {
                dropExisting = config.getIoConfig().isDropExisting();
            }
            // make tuningConfig
            final String taskId = indexingServiceClient.compactSegments("coordinator-issued", segmentsToCompact, config.getTaskPriority(), ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment()), granularitySpec, dimensionsSpec, config.getMetricsSpec(), transformSpec, dropExisting, newAutoCompactionContext(config.getTaskContext()));
            LOG.info("Submitted a compactionTask[%s] for %s segments", taskId, segmentsToCompact.size());
            LOG.infoSegments(segmentsToCompact, "Compacting segments");
            // Count the compaction task itself + its sub tasks
            numSubmittedTasks++;
            numCompactionTasksAndSubtasks += findMaxNumTaskSlotsUsedByOneCompactionTask(config.getTuningConfig());
        } else {
            throw new ISE("segmentsToCompact is empty?");
        }
    }
    return makeStats(currentRunAutoCompactionSnapshotBuilders, numSubmittedTasks, iterator);
}

Example 4 with ClientCompactionTaskDimensionsSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithDimensionSpec.

@Test
public void testCompactWithDimensionSpec() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo"))), null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskDimensionsSpec> dimensionsSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskDimensionsSpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), dimensionsSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskDimensionsSpec actual = dimensionsSpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    Assert.assertEquals(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), actual.getDimensions());
}

Example 5 with ClientCompactionTaskDimensionsSpec

use of org.apache.druid.client.indexing.ClientCompactionTaskDimensionsSpec in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithoutDimensionSpec.

@Test
public void testCompactWithoutDimensionSpec() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskDimensionsSpec> dimensionsSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskDimensionsSpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), dimensionsSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskDimensionsSpec actual = dimensionsSpecArgumentCaptor.getValue();
    Assert.assertNull(actual);
}