Examples with DataSourceCompactionConfig org.apache.druid.server.coordinator.DataSourceCompactionConfig used on opensource projects

Example 11 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithNullTuningConfigReturnDynamicPartitinosSpecWithMaxTotalRowsOfLongMax.

@Test
public void testFindPartitionsSpecFromConfigWithNullTuningConfigReturnDynamicPartitinosSpecWithMaxTotalRowsOfLongMax() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, null, null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, Long.MAX_VALUE), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 12 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithDeprecatedMaxTotalRowsAndPartitionsSpecIgnoreDeprecatedOne.

@Test
public void testFindPartitionsSpecFromConfigWithDeprecatedMaxTotalRowsAndPartitionsSpecIgnoreDeprecatedOne() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, 1000L, null, new DynamicPartitionsSpec(null, null), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, Long.MAX_VALUE), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 13 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class NewestSegmentFirstIteratorTest method testFindPartitionsSpecFromConfigWithNullMaxTotalRowsReturnLongMaxValue.

@Test
public void testFindPartitionsSpecFromConfigWithNullMaxTotalRowsReturnLongMaxValue() {
    final DataSourceCompactionConfig config = new DataSourceCompactionConfig("datasource", null, null, null, null, new UserCompactionTaskQueryTuningConfig(null, null, null, null, new DynamicPartitionsSpec(null, null), null, null, null, null, null, null, null, null, null, null, null, null), null, null, null, null, null, null);
    Assert.assertEquals(new DynamicPartitionsSpec(null, Long.MAX_VALUE), NewestSegmentFirstIterator.findPartitionsSpecFromConfig(ClientCompactionTaskQueryTuningConfig.from(config.getTuningConfig(), config.getMaxRowsPerSegment())));
}

Example 14 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class KillCompactionConfig method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    long currentTimeMillis = System.currentTimeMillis();
    if ((lastKillTime + period) < currentTimeMillis) {
        lastKillTime = currentTimeMillis;
        try {
            RetryUtils.retry(() -> {
                final byte[] currentBytes = CoordinatorCompactionConfig.getConfigInByteFromDb(connector, connectorConfig);
                final CoordinatorCompactionConfig current = CoordinatorCompactionConfig.convertByteToConfig(jacksonConfigManager, currentBytes);
                // If current compaction config is empty then there is nothing to do
                if (CoordinatorCompactionConfig.empty().equals(current)) {
                    log.info("Finished running KillCompactionConfig duty. Nothing to do as compaction config is already empty.");
                    emitMetric(params.getEmitter(), 0);
                    return ConfigManager.SetResult.ok();
                }
                // Get all active datasources
                // Note that we get all active datasources after getting compaction config to prevent race condition if new
                // datasource and config are added.
                Set<String> activeDatasources = sqlSegmentsMetadataManager.retrieveAllDataSourceNames();
                final Map<String, DataSourceCompactionConfig> updated = current.getCompactionConfigs().stream().filter(dataSourceCompactionConfig -> activeDatasources.contains(dataSourceCompactionConfig.getDataSource())).collect(Collectors.toMap(DataSourceCompactionConfig::getDataSource, Function.identity()));
                // Calculate number of compaction configs to remove for logging
                int compactionConfigRemoved = current.getCompactionConfigs().size() - updated.size();
                ConfigManager.SetResult result = jacksonConfigManager.set(CoordinatorCompactionConfig.CONFIG_KEY, currentBytes, CoordinatorCompactionConfig.from(current, ImmutableList.copyOf(updated.values())), new AuditInfo("KillCompactionConfig", "CoordinatorDuty for automatic deletion of compaction config", ""));
                if (result.isOk()) {
                    log.info("Finished running KillCompactionConfig duty. Removed %,d compaction configs", compactionConfigRemoved);
                    emitMetric(params.getEmitter(), compactionConfigRemoved);
                } else if (result.isRetryable()) {
                    // Failed but is retryable
                    log.debug("Retrying KillCompactionConfig duty");
                    throw new RetryableException(result.getException());
                } else {
                    // Failed and not retryable
                    log.error(result.getException(), "Failed to kill compaction configurations");
                    emitMetric(params.getEmitter(), 0);
                }
                return result;
            }, e -> e instanceof RetryableException, UPDATE_NUM_RETRY);
        } catch (Exception e) {
            log.error(e, "Failed to kill compaction configurations");
            emitMetric(params.getEmitter(), 0);
        }
    }
    return params;
}

Example 15 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactionResourceTestClient method getDataSourceCompactionConfig.

public DataSourceCompactionConfig getDataSourceCompactionConfig(String dataSource) throws Exception {
    String url = StringUtils.format("%sconfig/compaction/%s", getCoordinatorURL(), StringUtils.urlEncode(dataSource));
    StatusResponseHolder response = httpClient.go(new Request(HttpMethod.GET, new URL(url)), responseHandler).get();
    if (!response.getStatus().equals(HttpResponseStatus.OK)) {
        throw new ISE("Error while getting compaction config status[%s] content[%s]", response.getStatus(), response.getContent());
    }
    return jsonMapper.readValue(response.getContent(), new TypeReference<DataSourceCompactionConfig>() {
    });
}