Examples with NumberedOverwriteShardSpec org.apache.druid.timeline.partition.NumberedOverwriteShardSpec used on opensource projects

Example 6 with NumberedOverwriteShardSpec

use of org.apache.druid.timeline.partition.NumberedOverwriteShardSpec in project druid by druid-io.

the class CompactionTaskRunTest method testRunIndexAndCompactAtTheSameTimeForDifferentInterval.

@Test
public void testRunIndexAndCompactAtTheSameTimeForDifferentInterval() throws Exception {
    runIndexTask();
    final Builder builder = new Builder(DATA_SOURCE, segmentCacheManagerFactory, RETRY_POLICY_FACTORY);
    final CompactionTask compactionTask = builder.interval(Intervals.of("2014-01-01T00:00:00/2014-01-02T03:00:00")).build();
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
        writer.write("2014-01-01T03:00:10Z,a,1\n");
        writer.write("2014-01-01T03:00:10Z,b,2\n");
        writer.write("2014-01-01T03:00:10Z,c,3\n");
        writer.write("2014-01-01T04:00:20Z,a,1\n");
        writer.write("2014-01-01T04:00:20Z,b,2\n");
        writer.write("2014-01-01T04:00:20Z,c,3\n");
        writer.write("2014-01-01T05:00:30Z,a,1\n");
        writer.write("2014-01-01T05:00:30Z,b,2\n");
        writer.write("2014-01-01T05:00:30Z,c,3\n");
    }
    IndexTask indexTask = new IndexTask(null, null, IndexTaskTest.createIngestionSpec(getObjectMapper(), tmpDir, DEFAULT_PARSE_SPEC, null, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), IndexTaskTest.createTuningConfig(2, 2, null, 2L, null, false, true), false, false), null);
    final Future<Pair<TaskStatus, List<DataSegment>>> compactionFuture = exec.submit(() -> runTask(compactionTask));
    final Future<Pair<TaskStatus, List<DataSegment>>> indexFuture = exec.submit(() -> runTask(indexTask));
    Assert.assertTrue(indexFuture.get().lhs.isSuccess());
    List<DataSegment> segments = indexFuture.get().rhs;
    Assert.assertEquals(6, segments.size());
    for (int i = 0; i < 6; i++) {
        Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", 3 + i / 2, 3 + i / 2 + 1), segments.get(i).getInterval());
        if (lockGranularity == LockGranularity.SEGMENT) {
            Assert.assertEquals(new NumberedShardSpec(i % 2, 0), segments.get(i).getShardSpec());
        } else {
            Assert.assertEquals(new NumberedShardSpec(i % 2, 2), segments.get(i).getShardSpec());
        }
    }
    Assert.assertTrue(compactionFuture.get().lhs.isSuccess());
    segments = compactionFuture.get().rhs;
    Assert.assertEquals(3, segments.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1), segments.get(i).getInterval());
        Assert.assertEquals(getDefaultCompactionState(Granularities.HOUR, Granularities.MINUTE, ImmutableList.of(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i, i + 1))), segments.get(i).getLastCompactionState());
        if (lockGranularity == LockGranularity.SEGMENT) {
            Assert.assertEquals(new NumberedOverwriteShardSpec(PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, 0, 2, (short) 1, (short) 1), segments.get(i).getShardSpec());
        } else {
            Assert.assertEquals(new NumberedShardSpec(0, 1), segments.get(i).getShardSpec());
        }
    }
}

Example 7 with NumberedOverwriteShardSpec

use of org.apache.druid.timeline.partition.NumberedOverwriteShardSpec in project druid by druid-io.

the class CompactionTaskRunTest method testRunIndexAndCompactForSameSegmentAtTheSameTime.

@Test
public void testRunIndexAndCompactForSameSegmentAtTheSameTime() throws Exception {
    runIndexTask();
    // make sure that indexTask becomes ready first, then compactionTask becomes ready, then indexTask runs
    final CountDownLatch compactionTaskReadyLatch = new CountDownLatch(1);
    final CountDownLatch indexTaskStartLatch = new CountDownLatch(1);
    final Future<Pair<TaskStatus, List<DataSegment>>> indexFuture = exec.submit(() -> runIndexTask(compactionTaskReadyLatch, indexTaskStartLatch, false));
    final Builder builder = new Builder(DATA_SOURCE, segmentCacheManagerFactory, RETRY_POLICY_FACTORY);
    final CompactionTask compactionTask = builder.interval(Intervals.of("2014-01-01T00:00:00/2014-01-02T03:00:00")).build();
    final Future<Pair<TaskStatus, List<DataSegment>>> compactionFuture = exec.submit(() -> {
        compactionTaskReadyLatch.await();
        return runTask(compactionTask, indexTaskStartLatch, null);
    });
    Assert.assertTrue(indexFuture.get().lhs.isSuccess());
    List<DataSegment> segments = indexFuture.get().rhs;
    Assert.assertEquals(6, segments.size());
    for (int i = 0; i < 6; i++) {
        Assert.assertEquals(Intervals.of("2014-01-01T0%d:00:00/2014-01-01T0%d:00:00", i / 2, i / 2 + 1), segments.get(i).getInterval());
        if (lockGranularity == LockGranularity.SEGMENT) {
            Assert.assertEquals(new NumberedOverwriteShardSpec(PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + i % 2, 0, 2, (short) 1, (short) 2), segments.get(i).getShardSpec());
        } else {
            Assert.assertEquals(new NumberedShardSpec(i % 2, 2), segments.get(i).getShardSpec());
        }
    }
    final Pair<TaskStatus, List<DataSegment>> compactionResult = compactionFuture.get();
    Assert.assertEquals(TaskState.FAILED, compactionResult.lhs.getStatusCode());
}

Example 8 with NumberedOverwriteShardSpec

use of org.apache.druid.timeline.partition.NumberedOverwriteShardSpec in project druid by druid-io.

the class IndexTaskTest method testOverwriteWithSameSegmentGranularity.

@Test
public void testOverwriteWithSameSegmentGranularity() throws Exception {
    final File tmpDir = temporaryFolder.newFolder();
    final File tmpFile = File.createTempFile("druid", "index", tmpDir);
    populateRollupTestData(tmpFile);
    for (int i = 0; i < 2; i++) {
        final IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.DAY, Granularities.DAY, true, null), null, createTuningConfig(3, 2, null, 2L, null, false, true), false, false), null);
        final List<DataSegment> segments = runTask(indexTask).rhs;
        Assert.assertEquals(5, segments.size());
        final Interval expectedInterval = Intervals.of("2014-01-01T00:00:00.000Z/2014-01-02T00:00:00.000Z");
        for (int j = 0; j < 5; j++) {
            final DataSegment segment = segments.get(j);
            Assert.assertEquals(DATASOURCE, segment.getDataSource());
            Assert.assertEquals(expectedInterval, segment.getInterval());
            if (i == 0) {
                Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass());
                Assert.assertEquals(j, segment.getShardSpec().getPartitionNum());
            } else {
                if (lockGranularity == LockGranularity.SEGMENT) {
                    Assert.assertEquals(NumberedOverwriteShardSpec.class, segment.getShardSpec().getClass());
                    final NumberedOverwriteShardSpec numberedOverwriteShardSpec = (NumberedOverwriteShardSpec) segment.getShardSpec();
                    Assert.assertEquals(j + PartitionIds.NON_ROOT_GEN_START_PARTITION_ID, numberedOverwriteShardSpec.getPartitionNum());
                    Assert.assertEquals(1, numberedOverwriteShardSpec.getMinorVersion());
                    Assert.assertEquals(5, numberedOverwriteShardSpec.getAtomicUpdateGroupSize());
                    Assert.assertEquals(0, numberedOverwriteShardSpec.getStartRootPartitionId());
                    Assert.assertEquals(5, numberedOverwriteShardSpec.getEndRootPartitionId());
                } else {
                    Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass());
                    final NumberedShardSpec numberedShardSpec = (NumberedShardSpec) segment.getShardSpec();
                    Assert.assertEquals(j, numberedShardSpec.getPartitionNum());
                }
            }
        }
    }
}

Example 9 with NumberedOverwriteShardSpec

use of org.apache.druid.timeline.partition.NumberedOverwriteShardSpec in project druid by druid-io.

the class VersionedIntervalTimelineBenchmark method setup.

@Setup
public void setup() {
    final int numNewRootGenSegmentsAfterCompaction = (int) (numInitialRootGenSegmentsPerInterval * NEW_ROOT_GEN_SEGMENTS_RATIO_AFTER_COMPACTION);
    final int numCompactedSegments = (int) (numInitialRootGenSegmentsPerInterval * COMPACTED_SEGMENTS_RATIO_TO_INITIAL_SEGMENTS);
    intervals = Lists.newArrayList(segmentGranularity.getDefaultGranularity().getIterable(TOTAL_INTERVAL));
    segments = new ArrayList<>(intervals.size() * numInitialRootGenSegmentsPerInterval);
    Map<Interval, Integer> nextRootGenPartitionIds = Maps.newHashMapWithExpectedSize(intervals.size());
    Map<Interval, Integer> nextNonRootGenPartitionIds = Maps.newHashMapWithExpectedSize(intervals.size());
    Map<Interval, Short> nextMinorVersions = Maps.newHashMapWithExpectedSize(intervals.size());
    DateTime majorVersion = DateTimes.nowUtc();
    for (Interval interval : intervals) {
        majorVersion = majorVersion.plus(1);
        int nextRootGenPartitionId = 0;
        int nextNonRootGenPartitionId = PartitionIds.NON_ROOT_GEN_START_PARTITION_ID;
        // Generate root generation segments
        for (int i = 0; i < numInitialRootGenSegmentsPerInterval; i++) {
            segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
        }
        for (int i = 0; i < numNonRootGenerations; i++) {
            if (!useSegmentLock) {
                majorVersion = majorVersion.plus(1);
                nextRootGenPartitionId = 0;
            }
            // Compacted segments
            for (int j = 0; j < numCompactedSegments; j++) {
                if (useSegmentLock) {
                    segments.add(newSegment(interval, majorVersion.toString(), new NumberedOverwriteShardSpec(nextNonRootGenPartitionId++, 0, nextRootGenPartitionId, (short) (i + 1), (short) numCompactedSegments)));
                } else {
                    segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
                }
            }
            // New segments
            for (int j = 0; j < numNewRootGenSegmentsAfterCompaction; j++) {
                segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
            }
        }
        nextRootGenPartitionIds.put(interval, nextRootGenPartitionId);
        nextNonRootGenPartitionIds.put(interval, nextNonRootGenPartitionId);
        nextMinorVersions.put(interval, (short) (numNonRootGenerations + 1));
    }
    timeline = VersionedIntervalTimeline.forSegments(segments);
    newSegments = new ArrayList<>(200);
    // Generate new appending segments
    for (int i = 0; i < 100; i++) {
        final Interval interval = intervals.get(ThreadLocalRandom.current().nextInt(intervals.size()));
        final int rootPartitionId = nextRootGenPartitionIds.get(interval);
        newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(rootPartitionId, 0)));
        nextRootGenPartitionIds.put(interval, rootPartitionId + 1);
    }
    // Generate overwriting segments
    if (!useSegmentLock) {
        majorVersion = majorVersion.plus(1);
        nextRootGenPartitionIds.keySet().forEach(interval -> nextRootGenPartitionIds.put(interval, 0));
    }
    final List<Interval> intervalCopies = new ArrayList<>(intervals);
    for (int i = 0; i < 100 && !intervalCopies.isEmpty(); i++) {
        final Interval interval = intervalCopies.remove(ThreadLocalRandom.current().nextInt(intervalCopies.size()));
        int rootPartitionId = nextRootGenPartitionIds.remove(interval);
        int nonRootPartitionId = nextNonRootGenPartitionIds.remove(interval);
        final short minorVersion = nextMinorVersions.remove(interval);
        for (int j = 0; j < numCompactedSegments; j++) {
            if (useSegmentLock) {
                newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedOverwriteShardSpec(nonRootPartitionId++, 0, rootPartitionId, minorVersion, (short) numCompactedSegments)));
            } else {
                newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(rootPartitionId++, 0)));
            }
        }
    }
}

Example 10 with NumberedOverwriteShardSpec

use of org.apache.druid.timeline.partition.NumberedOverwriteShardSpec in project druid by druid-io.

the class SinglePhaseParallelIndexingTest method assertShardSpecAfterOverwrite.

private void assertShardSpecAfterOverwrite(ParallelIndexSupervisorTask task, LockGranularity actualLockGranularity) {
    final Collection<DataSegment> segments = getIndexingServiceClient().getPublishedSegments(task);
    final Map<Interval, List<DataSegment>> intervalToSegments = SegmentUtils.groupSegmentsByInterval(segments);
    if (actualLockGranularity != LockGranularity.SEGMENT) {
        // Check the core partition set in the shardSpec
        for (List<DataSegment> segmentsPerInterval : intervalToSegments.values()) {
            for (DataSegment segment : segmentsPerInterval) {
                Assert.assertSame(NumberedShardSpec.class, segment.getShardSpec().getClass());
                final NumberedShardSpec shardSpec = (NumberedShardSpec) segment.getShardSpec();
                Assert.assertEquals(segmentsPerInterval.size(), shardSpec.getNumCorePartitions());
            }
        }
    } else {
        for (List<DataSegment> segmentsPerInterval : intervalToSegments.values()) {
            for (DataSegment segment : segmentsPerInterval) {
                Assert.assertSame(NumberedOverwriteShardSpec.class, segment.getShardSpec().getClass());
                final NumberedOverwriteShardSpec shardSpec = (NumberedOverwriteShardSpec) segment.getShardSpec();
                Assert.assertEquals(segmentsPerInterval.size(), shardSpec.getAtomicUpdateGroupSize());
            }
        }
    }
}