Examples with HttpIndexingServiceClient org.apache.druid.client.indexing.HttpIndexingServiceClient used on opensource projects

Example 6 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithRollupInGranularitySpec.

@Test
public void testCompactWithRollupInGranularitySpec() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), new UserCompactionTaskGranularityConfig(Granularities.YEAR, null, true), null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<List<DataSegment>> segmentsCaptor = ArgumentCaptor.forClass(List.class);
    ArgumentCaptor<ClientCompactionTaskGranularitySpec> granularitySpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskGranularitySpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), segmentsCaptor.capture(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), granularitySpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any());
    Assert.assertEquals(datasourceToSegments.get(dataSource).size(), segmentsCaptor.getValue().size());
    ClientCompactionTaskGranularitySpec actual = granularitySpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    ClientCompactionTaskGranularitySpec expected = new ClientCompactionTaskGranularitySpec(Granularities.YEAR, null, true);
    Assert.assertEquals(expected, actual);
}

Example 7 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testRunWithLockedIntervalsNoSkip.

@Test
public void testRunWithLockedIntervalsNoSkip() {
    Mockito.when(COORDINATOR_CONFIG.getCompactionSkipLockedIntervals()).thenReturn(false);
    final TestDruidLeaderClient leaderClient = new TestDruidLeaderClient(JSON_MAPPER);
    leaderClient.start();
    HttpIndexingServiceClient indexingServiceClient = new HttpIndexingServiceClient(JSON_MAPPER, leaderClient);
    // Lock all intervals for all the dataSources
    final String datasource0 = DATA_SOURCE_PREFIX + 0;
    leaderClient.lockedIntervals.computeIfAbsent(datasource0, k -> new ArrayList<>()).add(Intervals.of("2017/2018"));
    final String datasource1 = DATA_SOURCE_PREFIX + 1;
    leaderClient.lockedIntervals.computeIfAbsent(datasource1, k -> new ArrayList<>()).add(Intervals.of("2017/2018"));
    final String datasource2 = DATA_SOURCE_PREFIX + 2;
    leaderClient.lockedIntervals.computeIfAbsent(datasource2, k -> new ArrayList<>()).add(Intervals.of("2017/2018"));
    // Verify that no locked intervals are skipped
    CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, indexingServiceClient);
    int maxTaskSlots = partitionsSpec instanceof SingleDimensionPartitionsSpec ? 5 : 3;
    final CoordinatorStats stats = doCompactSegments(compactSegments, createCompactionConfigs(1), maxTaskSlots);
    Assert.assertEquals(3, stats.getGlobalStat(CompactSegments.COMPACTION_TASK_COUNT));
    Assert.assertEquals(3, leaderClient.submittedCompactionTasks.size());
    leaderClient.submittedCompactionTasks.forEach(task -> {
        System.out.println(task.getDataSource() + " : " + task.getIoConfig().getInputSpec().getInterval());
    });
    // Verify that tasks are submitted for the latest interval of each dataSource
    final Map<String, Interval> datasourceToInterval = new HashMap<>();
    leaderClient.submittedCompactionTasks.forEach(task -> datasourceToInterval.put(task.getDataSource(), task.getIoConfig().getInputSpec().getInterval()));
    Assert.assertEquals(Intervals.of("2017-01-09T00:00:00Z/2017-01-09T12:00:00Z"), datasourceToInterval.get(datasource0));
    Assert.assertEquals(Intervals.of("2017-01-09T00:00:00Z/2017-01-09T12:00:00Z"), datasourceToInterval.get(datasource1));
    Assert.assertEquals(Intervals.of("2017-01-09T00:00:00Z/2017-01-09T12:00:00Z"), datasourceToInterval.get(datasource2));
}

Example 8 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithoutCustomSpecs.

@Test
public void testCompactWithoutCustomSpecs() {
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, null, null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskTransformSpec> transformSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskTransformSpec.class);
    ArgumentCaptor<AggregatorFactory[]> metricsSpecArgumentCaptor = ArgumentCaptor.forClass(AggregatorFactory[].class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), metricsSpecArgumentCaptor.capture(), transformSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskTransformSpec actualTransformSpec = transformSpecArgumentCaptor.getValue();
    Assert.assertNull(actualTransformSpec);
    AggregatorFactory[] actualMetricsSpec = metricsSpecArgumentCaptor.getValue();
    Assert.assertNull(actualMetricsSpec);
}

Example 9 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testRun.

@Test
public void testRun() {
    final TestDruidLeaderClient leaderClient = new TestDruidLeaderClient(JSON_MAPPER);
    leaderClient.start();
    final HttpIndexingServiceClient indexingServiceClient = new HttpIndexingServiceClient(JSON_MAPPER, leaderClient);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, indexingServiceClient);
    final Supplier<String> expectedVersionSupplier = new Supplier<String>() {

        private int i = 0;

        @Override
        public String get() {
            return "newVersion_" + i++;
        }
    };
    int expectedCompactTaskCount = 1;
    int expectedRemainingSegments = 400;
    // compact for 2017-01-08T12:00:00.000Z/2017-01-09T12:00:00.000Z
    assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT00:00:00/2017-01-%02dT12:00:00", 9, 9), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
    expectedRemainingSegments -= 40;
    assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT12:00:00/2017-01-%02dT00:00:00", 8, 9), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
    // compact for 2017-01-07T12:00:00.000Z/2017-01-08T12:00:00.000Z
    expectedRemainingSegments -= 40;
    assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT00:00:00/2017-01-%02dT12:00:00", 8, 8), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
    expectedRemainingSegments -= 40;
    assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT12:00:00/2017-01-%02dT00:00:00", 4, 5), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
    for (int endDay = 4; endDay > 1; endDay -= 1) {
        expectedRemainingSegments -= 40;
        assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT00:00:00/2017-01-%02dT12:00:00", endDay, endDay), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
        expectedRemainingSegments -= 40;
        assertCompactSegments(compactSegments, Intervals.of("2017-01-%02dT12:00:00/2017-01-%02dT00:00:00", endDay - 1, endDay), expectedRemainingSegments, expectedCompactTaskCount, expectedVersionSupplier);
    }
    assertLastSegmentNotCompacted(compactSegments);
}

Example 10 with HttpIndexingServiceClient

use of org.apache.druid.client.indexing.HttpIndexingServiceClient in project druid by druid-io.

the class CompactSegmentsTest method testCompactWithTransformSpec.

@Test
public void testCompactWithTransformSpec() {
    NullHandling.initializeForTests();
    final HttpIndexingServiceClient mockIndexingServiceClient = Mockito.mock(HttpIndexingServiceClient.class);
    final CompactSegments compactSegments = new CompactSegments(COORDINATOR_CONFIG, JSON_MAPPER, mockIndexingServiceClient);
    final List<DataSourceCompactionConfig> compactionConfigs = new ArrayList<>();
    final String dataSource = DATA_SOURCE_PREFIX + 0;
    compactionConfigs.add(new DataSourceCompactionConfig(dataSource, 0, 500L, null, // smaller than segment interval
    new Period("PT0H"), new UserCompactionTaskQueryTuningConfig(null, null, null, null, partitionsSpec, null, null, null, null, null, 3, null, null, null, null, null, null), null, null, null, new UserCompactionTaskTransformConfig(new SelectorDimFilter("dim1", "foo", null)), null, null));
    doCompactSegments(compactSegments, compactionConfigs);
    ArgumentCaptor<ClientCompactionTaskTransformSpec> transformSpecArgumentCaptor = ArgumentCaptor.forClass(ClientCompactionTaskTransformSpec.class);
    Mockito.verify(mockIndexingServiceClient).compactSegments(ArgumentMatchers.anyString(), ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), transformSpecArgumentCaptor.capture(), ArgumentMatchers.any(), ArgumentMatchers.any());
    ClientCompactionTaskTransformSpec actual = transformSpecArgumentCaptor.getValue();
    Assert.assertNotNull(actual);
    Assert.assertEquals(new SelectorDimFilter("dim1", "foo", null), actual.getFilter());
}