Examples with DataSourceCompactionConfig org.apache.druid.server.coordinator.DataSourceCompactionConfig used on opensource projects

Example 16 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class ITAutoCompactionTest method submitCompactionConfig.

private void submitCompactionConfig(PartitionsSpec partitionsSpec, Period skipOffsetFromLatest, int maxNumConcurrentSubTasks, UserCompactionTaskGranularityConfig granularitySpec, UserCompactionTaskDimensionsConfig dimensionsSpec, UserCompactionTaskTransformConfig transformSpec, AggregatorFactory[] metricsSpec, boolean dropExisting) throws Exception {
    DataSourceCompactionConfig compactionConfig = new DataSourceCompactionConfig(fullDatasourceName, null, null, null, skipOffsetFromLatest, new UserCompactionTaskQueryTuningConfig(null, null, null, new MaxSizeSplitHintSpec(null, 1), partitionsSpec, null, null, null, null, null, maxNumConcurrentSubTasks, null, null, null, null, null, 1), granularitySpec, dimensionsSpec, metricsSpec, transformSpec, !dropExisting ? null : new UserCompactionTaskIOConfig(true), null);
    compactionResource.submitCompactionConfig(compactionConfig);
    // Wait for compaction config to persist
    Thread.sleep(2000);
    // Verify that the compaction config is updated correctly.
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(fullDatasourceName)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), partitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), skipOffsetFromLatest);
    foundDataSourceCompactionConfig = compactionResource.getDataSourceCompactionConfig(fullDatasourceName);
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), partitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), skipOffsetFromLatest);
}

Example 17 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class ITAutoCompactionTest method deleteCompactionConfig.

private void deleteCompactionConfig() throws Exception {
    compactionResource.deleteCompactionConfig(fullDatasourceName);
    // Verify that the compaction config is updated correctly.
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(fullDatasourceName)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNull(foundDataSourceCompactionConfig);
}

Example 18 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class ITAutoCompactionLockContentionTest method submitAndVerifyCompactionConfig.

/**
 * Submits a compaction config for the current datasource.
 */
private void submitAndVerifyCompactionConfig() throws Exception {
    final DataSourceCompactionConfig compactionConfig = CompactionUtil.createCompactionConfig(fullDatasourceName, Specs.MAX_ROWS_PER_SEGMENT, Period.ZERO);
    compactionResource.updateCompactionTaskSlot(0.5, 10, null);
    compactionResource.submitCompactionConfig(compactionConfig);
    // Wait for compaction config to persist
    Thread.sleep(2000);
    // Verify that the compaction config is updated correctly.
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig observedCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(fullDatasourceName)) {
            observedCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertEquals(observedCompactionConfig, compactionConfig);
    observedCompactionConfig = compactionResource.getDataSourceCompactionConfig(fullDatasourceName);
    Assert.assertEquals(observedCompactionConfig, compactionConfig);
}

Example 19 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class ITAutoCompactionUpgradeTest method testUpgradeAutoCompactionConfigurationWhenConfigurationFromOlderVersionAlreadyExist.

@Test
public void testUpgradeAutoCompactionConfigurationWhenConfigurationFromOlderVersionAlreadyExist() throws Exception {
    // Verify that compaction config already exist. This config was inserted manually into the database using SQL script.
    // This auto compaction configuration payload is from Druid 0.21.0
    CoordinatorCompactionConfig coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    DataSourceCompactionConfig foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(UPGRADE_DATASOURCE_NAME)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    // Now submit a new auto compaction configuration
    PartitionsSpec newPartitionsSpec = new DynamicPartitionsSpec(4000, null);
    Period newSkipOffset = Period.seconds(0);
    DataSourceCompactionConfig compactionConfig = new DataSourceCompactionConfig(UPGRADE_DATASOURCE_NAME, null, null, null, newSkipOffset, new UserCompactionTaskQueryTuningConfig(null, null, null, new MaxSizeSplitHintSpec(null, 1), newPartitionsSpec, null, null, null, null, null, 1, null, null, null, null, null, 1), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, null), null, null, null, new UserCompactionTaskIOConfig(true), null);
    compactionResource.submitCompactionConfig(compactionConfig);
    // Wait for compaction config to persist
    Thread.sleep(2000);
    // Verify that compaction was successfully updated
    coordinatorCompactionConfig = compactionResource.getCoordinatorCompactionConfigs();
    foundDataSourceCompactionConfig = null;
    for (DataSourceCompactionConfig dataSourceCompactionConfig : coordinatorCompactionConfig.getCompactionConfigs()) {
        if (dataSourceCompactionConfig.getDataSource().equals(UPGRADE_DATASOURCE_NAME)) {
            foundDataSourceCompactionConfig = dataSourceCompactionConfig;
        }
    }
    Assert.assertNotNull(foundDataSourceCompactionConfig);
    Assert.assertNotNull(foundDataSourceCompactionConfig.getTuningConfig());
    Assert.assertEquals(foundDataSourceCompactionConfig.getTuningConfig().getPartitionsSpec(), newPartitionsSpec);
    Assert.assertEquals(foundDataSourceCompactionConfig.getSkipOffsetFromLatest(), newSkipOffset);
}

Example 20 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CoordinatorCompactionConfigsResource method getCompactionConfig.

@GET
@Path("/{dataSource}")
@Produces(MediaType.APPLICATION_JSON)
public Response getCompactionConfig(@PathParam("dataSource") String dataSource) {
    final CoordinatorCompactionConfig current = CoordinatorCompactionConfig.current(manager);
    final Map<String, DataSourceCompactionConfig> configs = current.getCompactionConfigs().stream().collect(Collectors.toMap(DataSourceCompactionConfig::getDataSource, Function.identity()));
    final DataSourceCompactionConfig config = configs.get(dataSource);
    if (config == null) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    return Response.ok().entity(config).build();
}