Examples with SegmentId org.apache.druid.timeline.SegmentId used on opensource projects

Example 26 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class CoordinatorServerView method serverRemovedSegment.

private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) {
    SegmentId segmentId = segment.getId();
    synchronized (lock) {
        log.debug("Removing segment[%s] from server[%s].", segmentId, server);
        final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId);
        if (segmentLoadInfo == null) {
            log.warn("Told to remove non-existant segment[%s]", segmentId);
            return;
        }
        segmentLoadInfo.removeServer(server);
        if (segmentLoadInfo.isEmpty()) {
            VersionedIntervalTimeline<String, SegmentLoadInfo> timeline = timelines.get(segment.getDataSource());
            segmentLoadInfos.remove(segmentId);
            final PartitionChunk<SegmentLoadInfo> removedPartition = timeline.remove(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(new SegmentLoadInfo(segment)));
            if (removedPartition == null) {
                log.warn("Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", segment.getInterval(), segment.getVersion());
            }
        }
    }
}

Example 27 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class BrokerServerView method serverAddedSegment.

private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) {
    SegmentId segmentId = segment.getId();
    synchronized (lock) {
        // loop...
        if (!server.getType().equals(ServerType.BROKER)) {
            log.debug("Adding segment[%s] for server[%s]", segment, server);
            ServerSelector selector = selectors.get(segmentId);
            if (selector == null) {
                selector = new ServerSelector(segment, tierSelectorStrategy);
                VersionedIntervalTimeline<String, ServerSelector> timeline = timelines.get(segment.getDataSource());
                if (timeline == null) {
                    timeline = new VersionedIntervalTimeline<>(Ordering.natural());
                    timelines.put(segment.getDataSource(), timeline);
                }
                timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector));
                selectors.put(segmentId, selector);
            }
            QueryableDruidServer queryableDruidServer = clients.get(server.getName());
            if (queryableDruidServer == null) {
                queryableDruidServer = addServer(baseView.getInventoryValue(server.getName()));
            }
            selector.addServerAndUpdateSegment(queryableDruidServer, segment);
        }
        // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it
        runTimelineCallbacks(callback -> callback.segmentAdded(server, segment));
    }
}

Example 28 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class BrokerServerView method serverRemovedSegment.

private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) {
    SegmentId segmentId = segment.getId();
    final ServerSelector selector;
    synchronized (lock) {
        log.debug("Removing segment[%s] from server[%s].", segmentId, server);
        // since the broker segments are not stored on the timeline, do not fire segmentRemoved event
        if (server.getType().equals(ServerType.BROKER)) {
            runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment));
            return;
        }
        selector = selectors.get(segmentId);
        if (selector == null) {
            log.warn("Told to remove non-existant segment[%s]", segmentId);
            return;
        }
        QueryableDruidServer queryableDruidServer = clients.get(server.getName());
        if (!selector.removeServer(queryableDruidServer)) {
            log.warn("Asked to disassociate non-existant association between server[%s] and segment[%s]", server, segmentId);
        } else {
            runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment));
        }
        if (selector.isEmpty()) {
            VersionedIntervalTimeline<String, ServerSelector> timeline = timelines.get(segment.getDataSource());
            selectors.remove(segmentId);
            final PartitionChunk<ServerSelector> removedPartition = timeline.remove(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector));
            if (removedPartition == null) {
                log.warn("Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", segment.getInterval(), segment.getVersion());
            } else {
                runTimelineCallbacks(callback -> callback.segmentRemoved(segment));
            }
        }
    }
}

Example 29 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class LoggerTest method testLogSegmentsMany.

@Test
public void testLogSegmentsMany() {
    final int numSegments = 100000;
    final MutableInt msgCount = new MutableInt();
    final Stream<SegmentId> segments = makeDataSegments(numSegments).map(DataSegment::getId);
    final Logger.LogFunction logger = (msg, format) -> {
        String message = StringUtils.format(msg, format);
        Assert.assertTrue(message.startsWith("Many segments: ["));
        Assert.assertTrue(message.endsWith("]"));
        msgCount.increment();
    };
    Logger.logSegmentIds(logger, segments, "Many segments");
    final int expected = (int) Math.ceil((double) numSegments / Logger.SEGMENTS_PER_LOG_MESSAGE);
    Assert.assertEquals(expected, msgCount.getValue());
}

Example 30 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class Logger method logSegmentIds.

/**
 * Logs all the segment ids you could ever want, {@link #SEGMENTS_PER_LOG_MESSAGE} at a time, as a comma separated
 * list.
 */
@VisibleForTesting
static void logSegmentIds(Logger.LogFunction logger, @Nullable final Stream<SegmentId> stream, @Nullable String preamble) {
    Preconditions.checkNotNull(preamble);
    if (stream == null) {
        return;
    }
    final Iterator<SegmentId> iterator = stream.iterator();
    if (!iterator.hasNext()) {
        return;
    }
    final String logFormat = preamble + ": %s";
    int counter = 0;
    StringBuilder sb = null;
    while (iterator.hasNext()) {
        SegmentId nextId = iterator.next();
        if (counter == 0) {
            // use segmentId string length of first as estimate for total size of builder for this batch
            sb = new StringBuilder(SEGMENTS_PER_LOG_MESSAGE * (2 + nextId.safeUpperLimitOfStringSize())).append("[");
        }
        sb.append(nextId);
        if (++counter < SEGMENTS_PER_LOG_MESSAGE && iterator.hasNext()) {
            sb.append(", ");
        }
        counter = counter % SEGMENTS_PER_LOG_MESSAGE;
        if (counter == 0) {
            // flush
            sb.append("]");
            logger.log(logFormat, sb.toString());
        }
    }
    // check for stragglers
    if (counter > 0) {
        sb.append("]");
        logger.log(logFormat, sb.toString());
    }
}