Examples with HttpIndexingServiceClient org.apache.druid.client.indexing.HttpIndexingServiceClient used on opensource projects

Example 1 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testRunMultipleCompactionTaskSlotsWithUseAutoScaleSlotsUnderMaxSlot.

@Test
public void testRunMultipleCompactionTaskSlotsWithUseAutoScaleSlotsUnderMaxSlot() {
    int maxCompactionSlot = 100;
    Assert.assertFalse(maxCompactionSlot < MAXIMUM_CAPACITY_WITH_AUTO_SCALE);
    final TestDruidLeaderClient leaderClient = new TestDruidLeaderClient(JSON_MAPPER);
    leaderClient.start();
    final HttpIndexingServiceClient indexingServiceClient = new HttpIndexingServiceClient(JSON_MAPPER, leaderClient);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, indexingServiceClient);
    final CoordinatorStats stats = doCompactSegments(compactSegments, createCompactionConfigs(), maxCompactionSlot, true);
    Assert.assertEquals(MAXIMUM_CAPACITY_WITH_AUTO_SCALE, stats.getGlobalStat(CompactSegments.AVAILABLE_COMPACTION_TASK_SLOT));
    Assert.assertEquals(MAXIMUM_CAPACITY_WITH_AUTO_SCALE, stats.getGlobalStat(CompactSegments.MAX_COMPACTION_TASK_SLOT));
    Assert.assertEquals(MAXIMUM_CAPACITY_WITH_AUTO_SCALE, stats.getGlobalStat(CompactSegments.COMPACTION_TASK_COUNT));
}

Example 2 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithNotNullIOConfig.

@Test
public void testCompactWithNotNullIOConfig() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, null, new UserCompactionTaskIOConfig(true), null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<Boolean> dropExistingCapture = ArgumentCaptor.forClass(Boolean.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), dropExistingCapture.capture(), ArgumentMatchers.any());
    Assert.assertEquals(true, dropExistingCapture.getValue());
}

Example 3 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testSerde.

@Test
public void testSerde() throws Exception {
    final TestDruidLeaderClient leaderClient = new TestDruidLeaderClient(JSON_MAPPER);
    final HttpIndexingServiceClient indexingServiceClient = new HttpIndexingServiceClient(JSON_MAPPER, leaderClient);
    JSON_MAPPER.setInjectableValues(new InjectableValues.Std().addValue(DruidCoordinatorConfig.class, COORDINATOR_CONFIG).addValue(ObjectMapper.class, JSON_MAPPER).addValue(IndexingServiceClient.class, indexingServiceClient));
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, indexingServiceClient);
    String compactSegmentString = JSON_MAPPER.writeValueAsString(compactSegments);
    CompactSegments serdeCompactSegments = JSON_MAPPER.readValue(compactSegmentString, CompactSegments.class);
    Assert.assertNotNull(serdeCompactSegments);
    Assert.assertEquals(COORDINATOR_CONFIG.getCompactionSkipLockedIntervals(), serdeCompactSegments.isSkipLockedIntervals());
    Assert.assertEquals(indexingServiceClient, serdeCompactSegments.getIndexingServiceClient());
}

Example 4 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithNullIOConfig.

@Test
public void testCompactWithNullIOConfig() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<Boolean> dropExistingCapture = ArgumentCaptor.forClass(Boolean.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), dropExistingCapture.capture(), ArgumentMatchers.any());
    Assert.assertNull(dropExistingCapture.getValue());
}

Example 5 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testMakeStatsForDataSourceWithSkipped.

@Test
public void testMakeStatsForDataSourceWithSkipped() {
    // Only test and validate for one datasource for simplicity.
    // This dataSource has three intervals skipped (3 intervals, 1200 byte, 12 segments skipped by auto compaction)
    // Note that these segment used to be 10 bytes each in other tests, we are increasing it to 100 bytes each here
    // so that they will be skipped by the auto compaction.
    String dataSourceName = DATA_SOURCE_PREFIX + 1;
    List<DataSegment> segments = new ArrayList<>();
    for (int j : new int[] { 0, 1, 2, 3, 7, 8 }) {
        for (int k = 0; k < 4; k++) {
            DataSegment beforeNoon = createSegment(dataSourceName, j, true, k);
            DataSegment afterNoon = createSegment(dataSourceName, j, false, k);
            if (j == 3) {
                // Make two intervals on this day skipped (two skipped intervals back-to-back)
                beforeNoon = beforeNoon.withSize(100);
                afterNoon = afterNoon.withSize(100);
            }
            if (j == 1) {
                // Make one interval on this day skipped
                afterNoon = afterNoon.withSize(100);
            }
            segments.add(beforeNoon);
            segments.add(afterNoon);
        }
    }
    dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of()).getUsedSegmentsTimelinesPerDataSource();
    final TestDruidLeaderClient leaderClient = new TestDruidLeaderClient(JSON_MAPPER);
    leaderClient.start();
    final HttpIndexingServiceClient indexingServiceClient = new HttpIndexingServiceClient(JSON_MAPPER, leaderClient);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, indexingServiceClient);
    // Before any compaction, we do not have any snapshot of compactions
    Map<String, AutoCompactionSnapshot> autoCompactionSnapshots = compactSegments.getAutoCompactionSnapshot();
    Assert.assertEquals(0, autoCompactionSnapshots.size());
    // 3 intervals, 1200 byte (each segment is 100 bytes), 12 segments will be skipped by auto compaction
    for (int compactionRunCount = 0; compactionRunCount < 8; compactionRunCount++) {
        // Do a cycle of auto compaction which creates one compaction task
        final CoordinatorStats stats = doCompactSegments(compactSegments);
        Assert.assertEquals(1, stats.getGlobalStat(CompactSegments.COMPACTION_TASK_COUNT));
        verifySnapshot(compactSegments, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, dataSourceName, // Minus 120 bytes accounting for the three skipped segments' original size
        TOTAL_BYTE_PER_DATASOURCE - 120 - 40 * (compactionRunCount + 1), 40 * (compactionRunCount + 1), 1200, TOTAL_INTERVAL_PER_DATASOURCE - 3 - (compactionRunCount + 1), (compactionRunCount + 1), 3, TOTAL_SEGMENT_PER_DATASOURCE - 12 - 4 * (compactionRunCount + 1), 4 + 2 * (compactionRunCount), 12);
    }
    // Test that stats does not change (and is still correct) when auto compaction runs with everything is fully compacted
    final CoordinatorStats stats = doCompactSegments(compactSegments);
    Assert.assertEquals(0, stats.getGlobalStat(CompactSegments.COMPACTION_TASK_COUNT));
    verifySnapshot(compactSegments, AutoCompactionSnapshot.AutoCompactionScheduleStatus.RUNNING, dataSourceName, 0, // Minus 120 bytes accounting for the three skipped segments' original size
    TOTAL_BYTE_PER_DATASOURCE - 120, 1200, 0, TOTAL_INTERVAL_PER_DATASOURCE - 3, 3, 0, 16, 12);
}