Examples with DataSourceCompactionConfig org.apache.druid.server.coordinator.DataSourceCompactionConfig used on opensource projects

Example 6 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithRollupInGranularitySpec.

@Test
public void testCompactWithRollupInGranularitySpec() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, true), null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<List<DataSegment>> segmentsCaptor = ArgumentCaptor.forClass(List.class);
    ArgumentCaptor<ClientCompactionTaskGranularitySpec> granularitySpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskGranularitySpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), segmentsCaptor.capture(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), granularitySpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    Assert.assertEquals(datasourceToSegments.get(dataSource).size(), segmentsCaptor.getValue().size());
    ClientCompactionTaskGranularitySpec actual = granularitySpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    ClientCompactionTaskGranularitySpec expected = new ClientCompactionTaskGranularitySpec(Granularities.YEAR, null, true);
    Assert.assertEquals(expected, actual);
}

Example 7 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithoutCustomSpecs.

@Test
public void testCompactWithoutCustomSpecs() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskTransformSpec> transformSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskTransformSpec.class);
    ArgumentCaptor<AggregatorFactory[]> metricsSpecArgumentCaptor = ArgumentCaptor.forClass(AggregatorFactory[].class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), metricsSpecArgumentCaptor.capture(), transformSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskTransformSpec actualTransformSpec = transformSpecArgumentCaptor.getValue();
    Assert.assertNull(actualTransformSpec);
    AggregatorFactory[] actualMetricsSpec = metricsSpecArgumentCaptor.getValue();
    Assert.assertNull(actualMetricsSpec);
}

Example 8 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithTransformSpec.

@Test
public void testCompactWithTransformSpec() {
    NullHandling.initializeForTests();
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, new UserCompactionTaskTransformConfig(new SelectorDimFilter("dim1", "foo", null)), null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskTransformSpec> transformSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskTransformSpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), transformSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskTransformSpec actual = transformSpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    Assert.assertEquals(new SelectorDimFilter("dim1", "foo", null), actual.getFilter());
}

Example 9 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithMetricsSpec.

@Test
public void testCompactWithMetricsSpec() {
    NullHandling.initializeForTests();
    AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] { new CountAggregatorFactory("cnt") };
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, aggregatorFactories, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<AggregatorFactory[]> metricsSpecArgumentCaptor = ArgumentCaptor.forClass(AggregatorFactory[].class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), metricsSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    AggregatorFactory[] actual = metricsSpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    Assert.assertArrayEquals(aggregatorFactories, actual);
}

Example 10 with DataSourceCompactionConfig

use of org.apache.druid.server.coordinator.DataSourceCompactionConfig in project druid by druid-io.

the class KillCompactionConfigTest method testRunRemoveInactiveDatasourceCompactionConfig.

@Test
public void testRunRemoveInactiveDatasourceCompactionConfig() {
    String inactiveDatasourceName = "inactive_datasource";
    String activeDatasourceName = "active_datasource";
    DataSourceCompactionConfig inactiveDatasourceConfig = new DataSourceCompactionConfig(inactiveDatasourceName, null, 500L, null, new Period(3600), null, new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null), null, null, null, null, ImmutableMap.of("key", "val"));
    DataSourceCompactionConfig activeDatasourceConfig = new DataSourceCompactionConfig(activeDatasourceName, null, 500L, null, new Period(3600), null, new UserCompactionTaskGranularityConfig(Granularities.HOUR, null, null), null, null, null, null, ImmutableMap.of("key", "val"));
    CoordinatorCompactionConfig originalCurrentConfig = CoordinatorCompactionConfig.from(ImmutableList.of(inactiveDatasourceConfig, activeDatasourceConfig));
    byte[] originalCurrentConfigBytes = { 1, 2, 3 };
    Mockito.when(mockConnector.lookup(ArgumentMatchers.anyString(), ArgumentMatchers.eq("name"), ArgumentMatchers.eq("payload"), ArgumentMatchers.eq(CoordinatorCompactionConfig.CONFIG_KEY))).thenReturn(originalCurrentConfigBytes);
    Mockito.when(mockJacksonConfigManager.convertByteToConfig(ArgumentMatchers.eq(originalCurrentConfigBytes), ArgumentMatchers.eq(CoordinatorCompactionConfig.class), ArgumentMatchers.eq(CoordinatorCompactionConfig.empty()))).thenReturn(originalCurrentConfig);
    Mockito.when(mockDruidCoordinatorRuntimeParams.getEmitter()).thenReturn(mockServiceEmitter);
    Mockito.when(mockSqlSegmentsMetadataManager.retrieveAllDataSourceNames()).thenReturn(ImmutableSet.of(activeDatasourceName));
    final ArgumentCaptor<byte[]> oldConfigCaptor = ArgumentCaptor.forClass(byte[].class);
    final ArgumentCaptor<CoordinatorCompactionConfig> newConfigCaptor = ArgumentCaptor.forClass(CoordinatorCompactionConfig.class);
    Mockito.when(mockJacksonConfigManager.set(ArgumentMatchers.eq(CoordinatorCompactionConfig.CONFIG_KEY), oldConfigCaptor.capture(), newConfigCaptor.capture(), ArgumentMatchers.any())).thenReturn(ConfigManager.SetResult.ok());
    TestDruidCoordinatorConfig druidCoordinatorConfig = new TestDruidCoordinatorConfig(null, null, null, new Duration("PT5S"), null, null, null, null, null, null, null, new Duration("PT6S"), null, null, null, null, 10, null);
    killCompactionConfig = new KillCompactionConfig(druidCoordinatorConfig, mockSqlSegmentsMetadataManager, mockJacksonConfigManager, mockConnector, mockConnectorConfig);
    killCompactionConfig.run(mockDruidCoordinatorRuntimeParams);
    // Verify and Assert
    Assert.assertNotNull(oldConfigCaptor.getValue());
    Assert.assertEquals(oldConfigCaptor.getValue(), originalCurrentConfigBytes);
    Assert.assertNotNull(newConfigCaptor.getValue());
    // The updated config should only contains one compaction config for the active datasource
    Assert.assertEquals(1, newConfigCaptor.getValue().getCompactionConfigs().size());
    Assert.assertEquals(activeDatasourceConfig, newConfigCaptor.getValue().getCompactionConfigs().get(0));
    final ArgumentCaptor<ServiceEventBuilder> emittedEventCaptor = ArgumentCaptor.forClass(ServiceEventBuilder.class);
    Mockito.verify(mockServiceEmitter).emit(emittedEventCaptor.capture());
    Assert.assertEquals(KillCompactionConfig.COUNT_METRIC, emittedEventCaptor.getValue().build(ImmutableMap.of()).toMap().get("metric"));
    // Should delete 1 config
    Assert.assertEquals(1, emittedEventCaptor.getValue().build(ImmutableMap.of()).toMap().get("value"));
    Mockito.verify(mockJacksonConfigManager).convertByteToConfig(ArgumentMatchers.eq(originalCurrentConfigBytes), ArgumentMatchers.eq(CoordinatorCompactionConfig.class), ArgumentMatchers.eq(CoordinatorCompactionConfig.empty()));
    Mockito.verify(mockConnector).lookup(ArgumentMatchers.anyString(), ArgumentMatchers.eq("name"), ArgumentMatchers.eq("payload"), ArgumentMatchers.eq(CoordinatorCompactionConfig.CONFIG_KEY));
    Mockito.verify(mockJacksonConfigManager).set(ArgumentMatchers.eq(CoordinatorCompactionConfig.CONFIG_KEY), ArgumentMatchers.any(byte[].class), ArgumentMatchers.any(CoordinatorCompactionConfig.class), ArgumentMatchers.any());
    Mockito.verifyNoMoreInteractions(mockJacksonConfigManager);
    Mockito.verify(mockSqlSegmentsMetadataManager).retrieveAllDataSourceNames();
    Mockito.verifyNoMoreInteractions(mockSqlSegmentsMetadataManager);
}