Examples with NumberedShardSpec org.apache.druid.timeline.partition.NumberedShardSpec used on opensource projects

Example 31 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class NewestSegmentFirstPolicyBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    compactionConfigs = new HashMap<>();
    for (int i = 0; i < numDataSources; i++) {
        final String dataSource = DATA_SOURCE_PREFIX + i;
        compactionConfigs.put(dataSource, new DataSourceCompactionConfig(dataSource, 0, inputSegmentSizeBytes, null, null, null, null, null, null, null, null, null));
    }
    List<DataSegment> segments = new ArrayList<>();
    for (int i = 0; i < numDataSources; i++) {
        final String dataSource = DATA_SOURCE_PREFIX + i;
        final int startYear = ThreadLocalRandom.current().nextInt(2000, 2040);
        DateTime date = DateTimes.of(startYear, 1, 1, 0, 0);
        for (int j = 0; j < numDayIntervalsPerDataSource; j++, date = date.plusDays(1)) {
            for (int k = 0; k < numPartitionsPerDayInterval; k++) {
                final ShardSpec shardSpec = new NumberedShardSpec(numPartitionsPerDayInterval, k);
                final DataSegment segment = new DataSegment(dataSource, new Interval(date, date.plusDays(1)), "version", null, ImmutableList.of(), ImmutableList.of(), shardSpec, 0, segmentSizeBytes);
                segments.add(segment);
            }
        }
    }
    dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of()).getUsedSegmentsTimelinesPerDataSource();
}

Example 32 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class VersionedIntervalTimelineBenchmark method setup.

@Setup
public void setup() {
    final int numNewRootGenSegmentsAfterCompaction = (int) (numInitialRootGenSegmentsPerInterval * NEW_ROOT_GEN_SEGMENTS_RATIO_AFTER_COMPACTION);
    final int numCompactedSegments = (int) (numInitialRootGenSegmentsPerInterval * COMPACTED_SEGMENTS_RATIO_TO_INITIAL_SEGMENTS);
    intervals = Lists.newArrayList(segmentGranularity.getDefaultGranularity().getIterable(TOTAL_INTERVAL));
    segments = new ArrayList<>(intervals.size() * numInitialRootGenSegmentsPerInterval);
    Map<Interval, Integer> nextRootGenPartitionIds = Maps.newHashMapWithExpectedSize(intervals.size());
    Map<Interval, Integer> nextNonRootGenPartitionIds = Maps.newHashMapWithExpectedSize(intervals.size());
    Map<Interval, Short> nextMinorVersions = Maps.newHashMapWithExpectedSize(intervals.size());
    DateTime majorVersion = DateTimes.nowUtc();
    for (Interval interval : intervals) {
        majorVersion = majorVersion.plus(1);
        int nextRootGenPartitionId = 0;
        int nextNonRootGenPartitionId = PartitionIds.NON_ROOT_GEN_START_PARTITION_ID;
        // Generate root generation segments
        for (int i = 0; i < numInitialRootGenSegmentsPerInterval; i++) {
            segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
        }
        for (int i = 0; i < numNonRootGenerations; i++) {
            if (!useSegmentLock) {
                majorVersion = majorVersion.plus(1);
                nextRootGenPartitionId = 0;
            }
            // Compacted segments
            for (int j = 0; j < numCompactedSegments; j++) {
                if (useSegmentLock) {
                    segments.add(newSegment(interval, majorVersion.toString(), new NumberedOverwriteShardSpec(nextNonRootGenPartitionId++, 0, nextRootGenPartitionId, (short) (i + 1), (short) numCompactedSegments)));
                } else {
                    segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
                }
            }
            // New segments
            for (int j = 0; j < numNewRootGenSegmentsAfterCompaction; j++) {
                segments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(nextRootGenPartitionId++, 0)));
            }
        }
        nextRootGenPartitionIds.put(interval, nextRootGenPartitionId);
        nextNonRootGenPartitionIds.put(interval, nextNonRootGenPartitionId);
        nextMinorVersions.put(interval, (short) (numNonRootGenerations + 1));
    }
    timeline = VersionedIntervalTimeline.forSegments(segments);
    newSegments = new ArrayList<>(200);
    // Generate new appending segments
    for (int i = 0; i < 100; i++) {
        final Interval interval = intervals.get(ThreadLocalRandom.current().nextInt(intervals.size()));
        final int rootPartitionId = nextRootGenPartitionIds.get(interval);
        newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(rootPartitionId, 0)));
        nextRootGenPartitionIds.put(interval, rootPartitionId + 1);
    }
    // Generate overwriting segments
    if (!useSegmentLock) {
        majorVersion = majorVersion.plus(1);
        nextRootGenPartitionIds.keySet().forEach(interval -> nextRootGenPartitionIds.put(interval, 0));
    }
    final List<Interval> intervalCopies = new ArrayList<>(intervals);
    for (int i = 0; i < 100 && !intervalCopies.isEmpty(); i++) {
        final Interval interval = intervalCopies.remove(ThreadLocalRandom.current().nextInt(intervalCopies.size()));
        int rootPartitionId = nextRootGenPartitionIds.remove(interval);
        int nonRootPartitionId = nextNonRootGenPartitionIds.remove(interval);
        final short minorVersion = nextMinorVersions.remove(interval);
        for (int j = 0; j < numCompactedSegments; j++) {
            if (useSegmentLock) {
                newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedOverwriteShardSpec(nonRootPartitionId++, 0, rootPartitionId, minorVersion, (short) numCompactedSegments)));
            } else {
                newSegments.add(newSegment(interval, majorVersion.toString(), new NumberedShardSpec(rootPartitionId++, 0)));
            }
        }
    }
}

Example 33 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTest method testGetUniqueDimensionsAndMetrics.

@Test
public void testGetUniqueDimensionsAndMetrics() {
    final int numSegmentsPerPartitionChunk = 5;
    final int numPartitionChunksPerTimelineObject = 10;
    final int numSegments = numSegmentsPerPartitionChunk * numPartitionChunksPerTimelineObject;
    final Interval interval = Intervals.of("2017-01-01/2017-01-02");
    final String version = "1";
    final List<TimelineObjectHolder<String, DataSegment>> timelineSegments = new ArrayList<>();
    for (int i = 0; i < numPartitionChunksPerTimelineObject; i++) {
        final List<PartitionChunk<DataSegment>> chunks = new ArrayList<>();
        for (int j = 0; j < numSegmentsPerPartitionChunk; j++) {
            final List<String> dims = IntStream.range(i, i + numSegmentsPerPartitionChunk).mapToObj(suffix -> "dim" + suffix).collect(Collectors.toList());
            final List<String> metrics = IntStream.range(i, i + numSegmentsPerPartitionChunk).mapToObj(suffix -> "met" + suffix).collect(Collectors.toList());
            final DataSegment segment = new DataSegment("ds", interval, version, ImmutableMap.of(), dims, metrics, new NumberedShardSpec(numPartitionChunksPerTimelineObject, i), 1, 1);
            final PartitionChunk<DataSegment> partitionChunk = new NumberedPartitionChunk<>(i, numPartitionChunksPerTimelineObject, segment);
            chunks.add(partitionChunk);
        }
        final TimelineObjectHolder<String, DataSegment> timelineHolder = new TimelineObjectHolder<>(interval, version, new PartitionHolder<>(chunks));
        timelineSegments.add(timelineHolder);
    }
    final String[] expectedDims = new String[] { "dim9", "dim10", "dim11", "dim12", "dim13", "dim8", "dim7", "dim6", "dim5", "dim4", "dim3", "dim2", "dim1", "dim0" };
    final String[] expectedMetrics = new String[] { "met9", "met10", "met11", "met12", "met13", "met8", "met7", "met6", "met5", "met4", "met3", "met2", "met1", "met0" };
    Assert.assertEquals(Arrays.asList(expectedDims), ReingestionTimelineUtils.getUniqueDimensions(timelineSegments, null));
    Assert.assertEquals(Arrays.asList(expectedMetrics), ReingestionTimelineUtils.getUniqueMetrics(timelineSegments));
}

Example 34 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SinglePhaseParallelIndexingTest method assertShardSpec.

private void assertShardSpec(ParallelIndexSupervisorTask task, LockGranularity actualLockGranularity, boolean appendToExisting, Collection<DataSegment> originalSegmentsIfAppend) {
    final Collection<DataSegment> segments = getIndexingServiceClient().getPublishedSegments(task);
    if (!appendToExisting && actualLockGranularity == LockGranularity.TIME_CHUNK) {
        // Initial write
        final Map<Interval, List<DataSegment>> intervalToSegments = SegmentUtils.groupSegmentsByInterval(segments);
        for (List<DataSegment> segmentsPerInterval : intervalToSegments.values()) {
            for (DataSegment segment : segmentsPerInterval) {
                Assert.assertSame(NumberedShardSpec.class, segment.getShardSpec().getClass());
                final NumberedShardSpec shardSpec = (NumberedShardSpec) segment.getShardSpec();
                Assert.assertEquals(segmentsPerInterval.size(), shardSpec.getNumCorePartitions());
            }
        }
    } else {
        // Append or initial write with segment lock
        final Map<Interval, List<DataSegment>> intervalToOriginalSegments = SegmentUtils.groupSegmentsByInterval(originalSegmentsIfAppend);
        for (DataSegment segment : segments) {
            Assert.assertSame(NumberedShardSpec.class, segment.getShardSpec().getClass());
            final NumberedShardSpec shardSpec = (NumberedShardSpec) segment.getShardSpec();
            final List<DataSegment> originalSegmentsInInterval = intervalToOriginalSegments.get(segment.getInterval());
            final int expectedNumCorePartitions = originalSegmentsInInterval == null || originalSegmentsInInterval.isEmpty() ? 0 : originalSegmentsInInterval.get(0).getShardSpec().getNumCorePartitions();
            Assert.assertEquals(expectedNumCorePartitions, shardSpec.getNumCorePartitions());
        }
    }
}

Example 35 with NumberedShardSpec

use of org.apache.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SinglePhaseParallelIndexingTest method assertShardSpecAfterOverwrite.

private void assertShardSpecAfterOverwrite(ParallelIndexSupervisorTask task, LockGranularity actualLockGranularity) {
    final Collection<DataSegment> segments = getIndexingServiceClient().getPublishedSegments(task);
    final Map<Interval, List<DataSegment>> intervalToSegments = SegmentUtils.groupSegmentsByInterval(segments);
    if (actualLockGranularity != LockGranularity.SEGMENT) {
        // Check the core partition set in the shardSpec
        for (List<DataSegment> segmentsPerInterval : intervalToSegments.values()) {
            for (DataSegment segment : segmentsPerInterval) {
                Assert.assertSame(NumberedShardSpec.class, segment.getShardSpec().getClass());
                final NumberedShardSpec shardSpec = (NumberedShardSpec) segment.getShardSpec();
                Assert.assertEquals(segmentsPerInterval.size(), shardSpec.getNumCorePartitions());
            }
        }
    } else {
        for (List<DataSegment> segmentsPerInterval : intervalToSegments.values()) {
            for (DataSegment segment : segmentsPerInterval) {
                Assert.assertSame(NumberedOverwriteShardSpec.class, segment.getShardSpec().getClass());
                final NumberedOverwriteShardSpec shardSpec = (NumberedOverwriteShardSpec) segment.getShardSpec();
                Assert.assertEquals(segmentsPerInterval.size(), shardSpec.getAtomicUpdateGroupSize());
            }
        }
    }
}