Examples with TaskResponseObject org.apache.druid.testing.clients.TaskResponseObject used on opensource projects

Example 6 with TaskResponseObject

use of org.apache.druid.testing.clients.TaskResponseObject in project druid by druid-io.

the class ITAutoCompactionTest method testAutoCompactionDutyWithDimensionsSpec.

@Test
public void testAutoCompactionDutyWithDimensionsSpec() throws Exception {
    // Index data with dimensions "page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous",
    // "namespace", "continent", "country", "region", "city"
    loadData(INDEX_TASK_WITH_DIMENSION_SPEC);
    try (final Closeable ignored = unloader(fullDatasourceName)) {
        final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
        intervalsBeforeCompaction.sort(null);
        // 4 segments across 2 days (4 total)...
        verifySegmentsCount(4);
        // Result is not rollup
        Map<String, Object> expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        // Compact and change dimension to only "language"
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, null, new UserCompactionTaskDimensionsConfig(DimensionsSpec.getDefaultSchemas(ImmutableList.of("language"))), null, null, false);
        forceTriggerAutoCompaction(2);
        // Result should rollup on language dimension
        expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 1, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        List<TaskResponseObject> compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        // Verify compacted segments does not get compacted again
        forceTriggerAutoCompaction(2);
        List<TaskResponseObject> compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size());
    }
}

Example 7 with TaskResponseObject

use of org.apache.druid.testing.clients.TaskResponseObject in project druid by druid-io.

the class ITAutoCompactionTest method testAutoCompactionDutyWithMetricsSpec.

@Test
public void testAutoCompactionDutyWithMetricsSpec() throws Exception {
    loadData(INDEX_TASK_WITH_DIMENSION_SPEC);
    try (final Closeable ignored = unloader(fullDatasourceName)) {
        final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
        intervalsBeforeCompaction.sort(null);
        // 4 segments across 2 days (4 total)...
        verifySegmentsCount(4);
        // For dim "page", result has values "Gypsy Danger" and "Striker Eureka"
        Map<String, Object> expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        // Compact and add longSum and doubleSum metrics
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, null, null, null, new AggregatorFactory[] { new DoubleSumAggregatorFactory("double_sum_added", "added"), new LongSumAggregatorFactory("long_sum_added", "added") }, false);
        forceTriggerAutoCompaction(2);
        // Result should be the same with the addition of new metrics, "double_sum_added" and "long_sum_added".
        // These new metrics should have the same value as the input field "added"
        expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "double_sum_added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "long_sum_added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57), ImmutableList.of(459)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        List<TaskResponseObject> compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        // Verify compacted segments does not get compacted again
        forceTriggerAutoCompaction(2);
        List<TaskResponseObject> compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size());
    }
}

Example 8 with TaskResponseObject

use of org.apache.druid.testing.clients.TaskResponseObject in project druid by druid-io.

the class ITAutoCompactionTest method testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity.

@Test
public void testAutoCompactionDutyWithSegmentGranularityAndExistingCompactedSegmentsHaveSameSegmentGranularity() throws Exception {
    loadData(INDEX_TASK);
    try (final Closeable ignored = unloader(fullDatasourceName)) {
        final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
        intervalsBeforeCompaction.sort(null);
        // 4 segments across 2 days (4 total)...
        verifySegmentsCount(4);
        verifyQuery(INDEX_QUERIES_RESOURCE);
        // Compacted without SegmentGranularity in auto compaction config
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET);
        forceTriggerAutoCompaction(2);
        verifyQuery(INDEX_QUERIES_RESOURCE);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        List<TaskResponseObject> compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        // Segments were compacted and already has DAY granularity since it was initially ingested with DAY granularity.
        // Now set auto compaction with DAY granularity in the granularitySpec
        Granularity newGranularity = Granularities.DAY;
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(newGranularity, null, null));
        forceTriggerAutoCompaction(2);
        verifyQuery(INDEX_QUERIES_RESOURCE);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        // should be no new compaction task as segmentGranularity is already DAY
        List<TaskResponseObject> compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size());
    }
}

Example 9 with TaskResponseObject

use of org.apache.druid.testing.clients.TaskResponseObject in project druid by druid-io.

the class ITAutoCompactionTest method testAutoCompactionDutyWithFilter.

@Test
public void testAutoCompactionDutyWithFilter() throws Exception {
    loadData(INDEX_TASK_WITH_DIMENSION_SPEC);
    try (final Closeable ignored = unloader(fullDatasourceName)) {
        final List<String> intervalsBeforeCompaction = coordinator.getSegmentIntervals(fullDatasourceName);
        intervalsBeforeCompaction.sort(null);
        // 4 segments across 2 days (4 total)...
        verifySegmentsCount(4);
        // Result is not rollup
        // For dim "page", result has values "Gypsy Danger" and "Striker Eureka"
        Map<String, Object> expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        // Compact and filter with selector on dim "page" and value "Striker Eureka"
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, null, null, new UserCompactionTaskTransformConfig(new SelectorDimFilter("page", "Striker Eureka", null)), null, false);
        forceTriggerAutoCompaction(2);
        // For dim "page", result should only contain value "Striker Eureka"
        expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 1, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        List<TaskResponseObject> compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        // Verify compacted segments does not get compacted again
        forceTriggerAutoCompaction(2);
        List<TaskResponseObject> compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size());
    }
}

Example 10 with TaskResponseObject

use of org.apache.druid.testing.clients.TaskResponseObject in project druid by druid-io.

the class ITAutoCompactionTest method testAutoCompactionDutyWithRollup.

@Test
public void testAutoCompactionDutyWithRollup() throws Exception {
    final ISOChronology chrono = ISOChronology.getInstance(DateTimes.inferTzFromString("America/Los_Angeles"));
    Map<String, Object> specs = ImmutableMap.of("%%GRANULARITYSPEC%%", new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, false, ImmutableList.of(new Interval("2013-08-31/2013-09-02", chrono))));
    loadData(INDEX_TASK_WITH_GRANULARITY_SPEC, specs);
    try (final Closeable ignored = unloader(fullDatasourceName)) {
        Map<String, Object> expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 2, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(57.0), ImmutableList.of(459.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        submitCompactionConfig(MAX_ROWS_PER_SEGMENT_COMPACTED, NO_SKIP_OFFSET, new UserCompactionTaskGranularityConfig(null, null, true), false);
        forceTriggerAutoCompaction(2);
        expectedResult = ImmutableMap.of("%%FIELD_TO_QUERY%%", "added", "%%EXPECTED_COUNT_RESULT%%", 1, "%%EXPECTED_SCAN_RESULT%%", ImmutableList.of(ImmutableMap.of("events", ImmutableList.of(ImmutableList.of(516.0)))));
        verifyQuery(INDEX_ROLLUP_QUERIES_RESOURCE, expectedResult);
        verifySegmentsCompacted(2, MAX_ROWS_PER_SEGMENT_COMPACTED);
        List<TaskResponseObject> compactTasksBefore = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        // Verify rollup segments does not get compacted again
        forceTriggerAutoCompaction(2);
        List<TaskResponseObject> compactTasksAfter = indexer.getCompleteTasksForDataSource(fullDatasourceName);
        Assert.assertEquals(compactTasksAfter.size(), compactTasksBefore.size());
    }
}