Examples with SingleDimensionPartitionsSpec org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec used on opensource projects

Example 6 with SingleDimensionPartitionsSpec

use of org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec in project druid by druid-io.

the class CompactionTaskParallelRunTest method testRunParallelWithRangePartitioningWithSingleTask.

@Test
public void testRunParallelWithRangePartitioningWithSingleTask() throws Exception {
    // Range partitioning is not supported with segment lock yet
    Assume.assumeFalse(lockGranularity == LockGranularity.SEGMENT);
    runIndexTask(null, true);
    final Builder builder = new Builder(DATA_SOURCE, getSegmentCacheManagerFactory(), RETRY_POLICY_FACTORY);
    final CompactionTask compactionTask = builder.inputSpec(new CompactionIntervalSpec(INTERVAL_TO_INDEX, null)).tuningConfig(newTuningConfig(new SingleDimensionPartitionsSpec(7, null, "dim", false), 1, true)).build();
    final Set<DataSegment> compactedSegments = runTask(compactionTask);
    for (DataSegment segment : compactedSegments) {
        // Expect compaction state to exist as store compaction state by default
        Map<String, String> expectedLongSumMetric = new HashMap<>();
        expectedLongSumMetric.put("type", "longSum");
        expectedLongSumMetric.put("name", "val");
        expectedLongSumMetric.put("fieldName", "val");
        expectedLongSumMetric.put("expression", null);
        Assert.assertSame(SingleDimensionShardSpec.class, segment.getShardSpec().getClass());
        CompactionState expectedState = new CompactionState(new SingleDimensionPartitionsSpec(7, null, "dim", false), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts", "dim"))), ImmutableList.of(expectedLongSumMetric), null, compactionTask.getTuningConfig().getIndexSpec().asMap(getObjectMapper()), getObjectMapper().readValue(getObjectMapper().writeValueAsString(new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, true, ImmutableList.of(segment.getInterval()))), Map.class));
        Assert.assertEquals(expectedState, segment.getLastCompactionState());
    }
}

Example 7 with SingleDimensionPartitionsSpec

use of org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec in project druid by druid-io.

the class RangePartitionTaskKillTest method failsFirstPhase.

@Test(timeout = 5000L)
public void failsFirstPhase() throws Exception {
    int targetRowsPerSegment = NUM_ROW * 2 / DIM_FILE_CARDINALITY / NUM_PARTITION;
    final ParallelIndexSupervisorTask task = newTask(TIMESTAMP_SPEC, DIMENSIONS_SPEC, INPUT_FORMAT, null, INTERVAL_TO_INDEX, inputDir, TEST_FILE_NAME_PREFIX + "*", new SingleDimensionPartitionsSpec(targetRowsPerSegment, null, DIM1, false), 2, false, 0);
    final TaskActionClient actionClient = createActionClient(task);
    final TaskToolbox toolbox = createTaskToolbox(task, actionClient);
    prepareTaskForLocking(task);
    Assert.assertTrue(task.isReady(actionClient));
    task.stopGracefully(null);
    TaskStatus taskStatus = task.runRangePartitionMultiPhaseParallel(toolbox);
    Assert.assertTrue(taskStatus.isFailure());
    Assert.assertEquals("Failed in phase[PHASE-1]. See task logs for details.", taskStatus.getErrorMsg());
}

Example 8 with SingleDimensionPartitionsSpec

use of org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec in project druid by druid-io.

the class RangePartitionTaskKillTest method failsThirdPhase.

@Test(timeout = 5000L)
public void failsThirdPhase() throws Exception {
    int targetRowsPerSegment = NUM_ROW * 2 / DIM_FILE_CARDINALITY / NUM_PARTITION;
    final ParallelIndexSupervisorTask task = newTask(TIMESTAMP_SPEC, DIMENSIONS_SPEC, INPUT_FORMAT, null, INTERVAL_TO_INDEX, inputDir, TEST_FILE_NAME_PREFIX + "*", new SingleDimensionPartitionsSpec(targetRowsPerSegment, null, DIM1, false), 2, false, 2);
    final TaskActionClient actionClient = createActionClient(task);
    final TaskToolbox toolbox = createTaskToolbox(task, actionClient);
    prepareTaskForLocking(task);
    Assert.assertTrue(task.isReady(actionClient));
    task.stopGracefully(null);
    TaskStatus taskStatus = task.runRangePartitionMultiPhaseParallel(toolbox);
    Assert.assertTrue(taskStatus.isFailure());
    Assert.assertEquals("Failed in phase[PHASE-3]. See task logs for details.", taskStatus.getErrorMsg());
}

Example 9 with SingleDimensionPartitionsSpec

use of org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec in project druid by druid-io.

the class RangePartitionTaskKillTest method failsSecondPhase.

@Test(timeout = 5000L)
public void failsSecondPhase() throws Exception {
    int targetRowsPerSegment = NUM_ROW * 2 / DIM_FILE_CARDINALITY / NUM_PARTITION;
    final ParallelIndexSupervisorTask task = newTask(TIMESTAMP_SPEC, DIMENSIONS_SPEC, INPUT_FORMAT, null, INTERVAL_TO_INDEX, inputDir, TEST_FILE_NAME_PREFIX + "*", new SingleDimensionPartitionsSpec(targetRowsPerSegment, null, DIM1, false), 2, false, 1);
    final TaskActionClient actionClient = createActionClient(task);
    final TaskToolbox toolbox = createTaskToolbox(task, actionClient);
    prepareTaskForLocking(task);
    Assert.assertTrue(task.isReady(actionClient));
    task.stopGracefully(null);
    TaskStatus taskStatus = task.runRangePartitionMultiPhaseParallel(toolbox);
    Assert.assertTrue(taskStatus.isFailure());
    Assert.assertEquals("Failed in phase[PHASE-2]. See task logs for details.", taskStatus.getErrorMsg());
}

Example 10 with SingleDimensionPartitionsSpec

use of org.apache.druid.indexer.partitions.SingleDimensionPartitionsSpec in project druid by druid-io.

the class RangePartitionMultiPhaseParallelIndexingTest method testAppendLinearlyPartitionedSegmentsToHashPartitionedDatasourceSuccessfullyAppend.

@Test
public void testAppendLinearlyPartitionedSegmentsToHashPartitionedDatasourceSuccessfullyAppend() {
    if (useMultivalueDim) {
        return;
    }
    final int targetRowsPerSegment = NUM_ROW / DIM_FILE_CARDINALITY / NUM_PARTITION;
    final Set<DataSegment> publishedSegments = new HashSet<>();
    publishedSegments.addAll(runTestTask(new SingleDimensionPartitionsSpec(targetRowsPerSegment, null, DIM1, false), TaskState.SUCCESS, false));
    // Append
    publishedSegments.addAll(runTestTask(new DynamicPartitionsSpec(5, null), TaskState.SUCCESS, true));
    // And append again
    publishedSegments.addAll(runTestTask(new DynamicPartitionsSpec(10, null), TaskState.SUCCESS, true));
    final Map<Interval, List<DataSegment>> intervalToSegments = new HashMap<>();
    publishedSegments.forEach(segment -> intervalToSegments.computeIfAbsent(segment.getInterval(), k -> new ArrayList<>()).add(segment));
    for (Entry<Interval, List<DataSegment>> entry : intervalToSegments.entrySet()) {
        final List<DataSegment> segments = entry.getValue();
        final List<DataSegment> rangedSegments = segments.stream().filter(segment -> segment.getShardSpec().getClass() == SingleDimensionShardSpec.class).collect(Collectors.toList());
        final List<DataSegment> linearSegments = segments.stream().filter(segment -> segment.getShardSpec().getClass() == NumberedShardSpec.class).collect(Collectors.toList());
        for (DataSegment rangedSegment : rangedSegments) {
            final SingleDimensionShardSpec rangeShardSpec = (SingleDimensionShardSpec) rangedSegment.getShardSpec();
            for (DataSegment linearSegment : linearSegments) {
                Assert.assertEquals(rangedSegment.getInterval(), linearSegment.getInterval());
                Assert.assertEquals(rangedSegment.getVersion(), linearSegment.getVersion());
                final NumberedShardSpec numberedShardSpec = (NumberedShardSpec) linearSegment.getShardSpec();
                Assert.assertEquals(rangeShardSpec.getNumCorePartitions(), numberedShardSpec.getNumCorePartitions());
                Assert.assertTrue(rangeShardSpec.getPartitionNum() < numberedShardSpec.getPartitionNum());
            }
        }
    }
}