Examples with UserCompactionTaskQueryTuningConfig org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig used on opensource projects

Example 6 with UserCompactionTaskQueryTuningConfig

use of org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithMetricsSpec.

@Test
public void testCompactWithMetricsSpec() {
    NullHandling.initializeForTests();
    AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] { new CountAggregatorFactory("cnt") };
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, aggregatorFactories, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<AggregatorFactory[]> metricsSpecArgumentCaptor = ArgumentCaptor.forClass(AggregatorFactory[].class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), metricsSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    AggregatorFactory[] actual = metricsSpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    Assert.assertArrayEquals(aggregatorFactories, actual);
}

Example 7 with UserCompactionTaskQueryTuningConfig

use of org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithDeprecatedMaxTotalRowsAndPartitionsSpecIgnoreDeprecatedOne.

@Test
public void testFindPartitionsSpecFromConfigWithDeprecatedMaxTotalRowsAndPartitionsSpecIgnoreDeprecatedOne() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, 1000L, null, new DynamicPartitionsSpec(null, null), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, Long.MAX_VALUE), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 8 with UserCompactionTaskQueryTuningConfig

use of org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithNullMaxTotalRowsReturnLongMaxValue.

@Test
public void testFindPartitionsSpecFromConfigWithNullMaxTotalRowsReturnLongMaxValue() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new DynamicPartitionsSpec(null, null), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, Long.MAX_VALUE), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 9 with UserCompactionTaskQueryTuningConfig

use of org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig in project druid by druid-io.

the class ITAutoCompactionTest method submitCompactionConfig.

private void submitCompactionConfig(PartitionsSpec partitionsSpec, Period skipOffsetFromLatest, int maxNumConcurrentSubTasks, UserCompactionTaskGranularityConfig granularitySpec, UserCompactionTaskDimensionsConfig dimensionsSpec, UserCompactionTaskTransformConfig transformSpec, AggregatorFactory[] metricsSpec, boolean dropExisting) throws Exception {
    DataSourceCompactionConfig compactionConfig = new DataSourceCompactionConfig(fullDatasourceName, null, null, null, skipOffsetFromLatest, new UserCompactionTaskQueryTuningConfig(null, null, null, new MaxSizeSplitHintSpec(null, 1), partitionsSpec, null, null, null, null, null, maxNumConcurrentSubTasks, null, null, null, null, null, 1), granularitySpec, dimensionsSpec, metricsSpec, transformSpec, !dropExisting ? null : new UserCompactionTaskIOConfig(true), null);
    compactionResource.submitCompactionConfig(compactionConfig);
    // Wait for compaction config to persist
    Thread.sleep(2000);
    // Verify that the compaction config is updated correctly.
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(fullDatasourceName)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), partitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), skipOffsetFromLatest);
    foundDataSourceCompactionConfig = compactionResource.getDataSourceCompactionConfig(fullDatasourceName);
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), partitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), skipOffsetFromLatest);
}

Example 10 with UserCompactionTaskQueryTuningConfig

use of org.apache.druid.server.coordinator.UserCompactionTaskQueryTuningConfig in project druid by druid-io.

the class ITAutoCompactionUpgradeTest method testUpgradeAutoCompactionConfigurationWhenConfigurationFromOlderVersionAlreadyExist.

@Test
public void testUpgradeAutoCompactionConfigurationWhenConfigurationFromOlderVersionAlreadyExist() throws Exception {
    // Verify that compaction config already exist. This config was inserted manually into the database using SQL script.
    // This auto compaction configuration payload is from Druid 0.21.0
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(UPGRADE_DATASOURCE_NAME)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    // Now submit a new auto compaction configuration
    PartitionsSpec newPartitionsSpec = new DynamicPartitionsSpec(4000, null);
    Period newSkipOffset = Period.seconds(0);
    DataSourceCompactionConfig compactionConfig = new DataSourceCompactionConfig(UPGRADE_DATASOURCE_NAME, null, null, null, newSkipOffset, new UserCompactionTaskQueryTuningConfig(null, null, null, new MaxSizeSplitHintSpec(null, 1), newPartitionsSpec, null, null, null, null, null, 1, null, null, null, null, null, 1), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null), null, null, null, new UserCompactionTaskIOConfig(true), null);
    compactionResource.submitCompactionConfig(compactionConfig);
    // Wait for compaction config to persist
    Thread.sleep(2000);
    // Verify that compaction was successfully updated
    coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(UPGRADE_DATASOURCE_NAME)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), newPartitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), newSkipOffset);
}